01HPDDYW2QXFDJ6Z1DBW41FQY4: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HPDDZ4TJ9DJT1EG520H20A7W

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-12T01:31:29.160Zjob dependencies complete; ready to run (waiting for 18 m 49 s)
22024-02-12T01:32:26.507Zjob assigned to worker 01HPDF1PT09E316WV7AH9CP7NB (queued for 57 s)
32024-02-12T01:32:27.079Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-12T01:32:56.970Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-12T01:32:56.970Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-12T01:33:18.347Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-12T01:33:18.348Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-12T01:33:40.423Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-12T01:33:40.423Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-12T01:33:47.574Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-12T01:33:47.579Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-12T01:33:47.580Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-12T01:33:47.580Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-12T01:33:47.580Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-12T01:33:47.580Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-12T01:33:47.580Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-12T01:33:47.581Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-12T01:33:47.581Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-12T01:33:47.581Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-12T01:33:47.581Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-12T01:33:47.581Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-12T01:33:47.582Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-12T01:33:47.582Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-12T01:33:47.582Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-12T01:33:47.582Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-12T01:33:58.940Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-12T01:33:58.940Zstarting task 0: "setup"
282024-02-12T01:33:59.136Z++ uname -s
292024-02-12T01:33:59.139Z+ kern=SunOS
302024-02-12T01:33:59.139Z+ case "$kern" in
312024-02-12T01:33:59.139Z+ groupadd -g 12345 build
322024-02-12T01:33:59.142Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-12T01:34:01.151Z+ zfs create -o mountpoint=/work rpool/work
342024-02-12T01:34:01.219Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-12T01:34:01.223Z+ home_fs=zfs
362024-02-12T01:34:01.223Z+ [[ zfs == autofs ]]
372024-02-12T01:34:01.223Z+ mkdir -p /home/build
382024-02-12T01:34:01.223Z+ chown build:build /home/build /work
392024-02-12T01:34:03.226Z+ chmod 0700 /home/build /work
402024-02-12T01:34:03.230Zprocess exited: duration 4303 ms, exit code 0
 
412024-02-12T01:34:03.240Zstarting task 1: "authentication"
422024-02-12T01:34:03.258Zprocess exited: duration 17 ms, exit code 0
 
432024-02-12T01:34:03.266Zstarting task 2: "build"
442024-02-12T01:34:03.276Z+ banner cores
452024-02-12T01:34:03.303Z
462024-02-12T01:34:03.304Z #### #### ##### ###### ####
472024-02-12T01:34:03.304Z # # # # # # # #
482024-02-12T01:34:03.304Z # # # # # ##### ####
492024-02-12T01:34:03.304Z # # # ##### # #
502024-02-12T01:34:03.304Z # # # # # # # # #
512024-02-12T01:34:03.304Z #### #### # # ###### ####
522024-02-12T01:34:03.304Z
532024-02-12T01:34:03.309Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-12T01:34:03.312Z+ echo 'input bins dir contains:'
552024-02-12T01:34:03.317Zinput bins dir contains:
562024-02-12T01:34:03.317Z+ ls -ltr /input/build/work/bins
572024-02-12T01:34:03.318Ztotal 761236
582024-02-12T01:34:03.318Z-rw-r--r-- 1 root root 121611746 Feb 12 01:32 crucible-downstairs.gz
592024-02-12T01:34:03.318Z-rw-r--r-- 1 root root 91139552 Feb 12 01:33 crucible-hammer.gz
602024-02-12T01:34:03.318Z-rw-r--r-- 1 root root 104810185 Feb 12 01:33 crutest.gz
612024-02-12T01:34:03.318Z-rw-r--r-- 1 root root 71941606 Feb 12 01:33 dsc.gz
622024-02-12T01:34:03.318Z+ echo 'input script dir contains:'
632024-02-12T01:34:03.318Zinput script dir contains:
642024-02-12T01:34:03.318Z+ ls -ltr /input/build/work/scripts
652024-02-12T01:34:03.321Ztotal 39
662024-02-12T01:34:03.321Z-rw-r--r-- 1 root root 3056 Feb 12 01:33 perf-downstairs-tick.d
672024-02-12T01:34:03.321Z-rw-r--r-- 1 root root 1738 Feb 12 01:33 test_ds.sh
682024-02-12T01:34:03.322Z-rw-r--r-- 1 root root 3746 Feb 12 01:33 test_live_repair.sh
692024-02-12T01:34:03.322Z-rw-r--r-- 1 root root 6416 Feb 12 01:33 test_repair.sh
702024-02-12T01:34:03.322Z-rw-r--r-- 1 root root 2969 Feb 12 01:33 test_replay.sh
712024-02-12T01:34:03.322Z-rw-r--r-- 1 root root 9465 Feb 12 01:33 test_up.sh
722024-02-12T01:34:03.322Z-rw-r--r-- 1 root root 2824 Feb 12 01:33 upstairs_info.d
732024-02-12T01:34:03.322Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-12T01:34:03.322Z+ echo ' chmod input script dir contains:'
752024-02-12T01:34:03.322Z chmod input script dir contains:
762024-02-12T01:34:03.322Z+ ls -ltr /input/build/work/scripts
772024-02-12T01:34:03.325Ztotal 39
782024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 3056 Feb 12 01:33 perf-downstairs-tick.d
792024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 1738 Feb 12 01:33 test_ds.sh
802024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 3746 Feb 12 01:33 test_live_repair.sh
812024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 6416 Feb 12 01:33 test_repair.sh
822024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 2969 Feb 12 01:33 test_replay.sh
832024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 9465 Feb 12 01:33 test_up.sh
842024-02-12T01:34:03.326Z-rwxr-xr-x 1 root root 2824 Feb 12 01:33 upstairs_info.d
852024-02-12T01:34:03.326Z+ banner unpack
862024-02-12T01:34:03.326Z
872024-02-12T01:34:03.326Z # # # # ##### ## #### # #
882024-02-12T01:34:03.326Z # # ## # # # # # # # # #
892024-02-12T01:34:03.326Z # # # # # # # # # # ####
902024-02-12T01:34:03.327Z # # # # # ##### ###### # # #
912024-02-12T01:34:03.327Z # # # ## # # # # # # #
922024-02-12T01:34:03.327Z #### # # # # # #### # #
932024-02-12T01:34:03.327Z
942024-02-12T01:34:03.327Z+ mkdir -p /var/tmp/bins
952024-02-12T01:34:03.330Z+ for t in "$input/bins/"*.gz
962024-02-12T01:34:03.331Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-12T01:34:03.331Z+ b=crucible-downstairs.gz
982024-02-12T01:34:03.331Z+ b=crucible-downstairs
992024-02-12T01:34:03.331Z+ gunzip
1002024-02-12T01:34:06.171Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-12T01:34:06.174Z+ for t in "$input/bins/"*.gz
1022024-02-12T01:34:06.174Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-12T01:34:06.179Z+ b=crucible-hammer.gz
1042024-02-12T01:34:06.179Z+ b=crucible-hammer
1052024-02-12T01:34:06.179Z+ gunzip
1062024-02-12T01:34:08.236Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-12T01:34:08.253Z+ for t in "$input/bins/"*.gz
1082024-02-12T01:34:08.254Z++ basename /input/build/work/bins/crutest.gz
1092024-02-12T01:34:08.254Z+ b=crutest.gz
1102024-02-12T01:34:08.254Z+ b=crutest
1112024-02-12T01:34:08.254Z+ gunzip
1122024-02-12T01:34:10.688Z+ chmod +x /var/tmp/bins/crutest
1132024-02-12T01:34:10.690Z+ for t in "$input/bins/"*.gz
1142024-02-12T01:34:10.691Z++ basename /input/build/work/bins/dsc.gz
1152024-02-12T01:34:10.694Z+ b=dsc.gz
1162024-02-12T01:34:10.694Z+ b=dsc
1172024-02-12T01:34:10.694Z+ gunzip
1182024-02-12T01:34:12.302Z+ chmod +x /var/tmp/bins/dsc
1192024-02-12T01:34:12.305Z+ export BINDIR=/var/tmp/bins
1202024-02-12T01:34:12.305Z+ BINDIR=/var/tmp/bins
1212024-02-12T01:34:12.306Z+ banner repair
1222024-02-12T01:34:12.306Z
1232024-02-12T01:34:12.306Z ##### ###### ##### ## # #####
1242024-02-12T01:34:12.306Z # # # # # # # # # #
1252024-02-12T01:34:12.306Z # # ##### # # # # # # #
1262024-02-12T01:34:12.306Z ##### # ##### ###### # #####
1272024-02-12T01:34:12.306Z # # # # # # # # #
1282024-02-12T01:34:12.306Z # # ###### # # # # # #
1292024-02-12T01:34:12.306Z
1302024-02-12T01:34:12.306Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-12T01:34:12.416ZTurn off color for downstairs dump
1322024-02-12T01:34:12.427ZCreating region directory at: /var/tmp/test_repair
1332024-02-12T01:34:12.430ZCreating dsc directory at: /tmp/dsc
1342024-02-12T01:34:12.530ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.10284928
1352024-02-12T01:34:12.569ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.03878379
1362024-02-12T01:34:12.604ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.035289355
1372024-02-12T01:34:12.604ZRegion set with 3 regions was created
1382024-02-12T01:34:12.618Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-12T01:34:12.618ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-12T01:34:12.650Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.567620261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
1412024-02-12T01:34:12.650Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.568349637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
1422024-02-12T01:34:12.650Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.568368786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
1432024-02-12T01:34:12.650Z{"msg":"Crucible 2eccbfd9-e667-4acc-89f3-33aeded92364 has session id: 95548fde-c7d9-4170-8e16-a37a1e6b2ca3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.568382565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1442024-02-12T01:34:12.650Z{"msg":"Upstairs opts: Upstairs UUID: 2eccbfd9-e667-4acc-89f3-33aeded92364, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.568401784Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1452024-02-12T01:34:12.650Z{"msg":"Crucible stats registered with UUID: 2eccbfd9-e667-4acc-89f3-33aeded92364","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.568424963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1462024-02-12T01:34:12.654ZCrucible runtime is spawned
1472024-02-12T01:34:12.654Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569153399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
1482024-02-12T01:34:12.654Z{"msg":"2eccbfd9-e667-4acc-89f3-33aeded92364 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569188627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1492024-02-12T01:34:12.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569204176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1502024-02-12T01:34:12.654Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569220275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1512024-02-12T01:34:12.654Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569252674Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1522024-02-12T01:34:12.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569297452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1532024-02-12T01:34:12.655Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.56932557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1542024-02-12T01:34:12.655Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569364608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1552024-02-12T01:34:12.655Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569388437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1562024-02-12T01:34:12.655Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569498302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1572024-02-12T01:34:12.655Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569565449Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1582024-02-12T01:34:12.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569613726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1592024-02-12T01:34:12.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569650645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1602024-02-12T01:34:12.655Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.569734411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1612024-02-12T01:34:12.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.571885348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1622024-02-12T01:34:12.655Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.571929806Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1632024-02-12T01:34:12.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.571959934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1642024-02-12T01:34:12.655Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.572009032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1652024-02-12T01:34:12.655Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.57205234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1662024-02-12T01:34:12.655Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.572088059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1672024-02-12T01:34:12.655Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.573337759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1682024-02-12T01:34:12.655Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.573385866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1692024-02-12T01:34:12.655Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.573467033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1702024-02-12T01:34:12.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574432816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1712024-02-12T01:34:12.655Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574483424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1722024-02-12T01:34:12.655Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574508773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1732024-02-12T01:34:12.655Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574546661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1742024-02-12T01:34:12.656Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.57456252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1752024-02-12T01:34:12.656Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574594989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1762024-02-12T01:34:12.656Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574615828Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1772024-02-12T01:34:12.656Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574644337Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1782024-02-12T01:34:12.656Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574692364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1792024-02-12T01:34:12.656Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574746012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1802024-02-12T01:34:12.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574907364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1812024-02-12T01:34:12.656Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574936683Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1822024-02-12T01:34:12.656Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574951732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1832024-02-12T01:34:12.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.574978441Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1842024-02-12T01:34:12.656Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.57499271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1852024-02-12T01:34:12.656Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575028998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1862024-02-12T01:34:12.660Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575045187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1872024-02-12T01:34:12.660Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575065896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1882024-02-12T01:34:12.660Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575080186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1892024-02-12T01:34:12.660Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575093975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1902024-02-12T01:34:12.660Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575107544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1912024-02-12T01:34:12.660Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575127703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1922024-02-12T01:34:12.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575140743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1932024-02-12T01:34:12.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575160672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1942024-02-12T01:34:12.660Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575174031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1952024-02-12T01:34:12.660ZWait for a query_work_queue command to finish before sending IO
1962024-02-12T01:34:12.660Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575207679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1972024-02-12T01:34:12.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575240838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1982024-02-12T01:34:12.660Z{"msg":"2eccbfd9-e667-4acc-89f3-33aeded92364 is now active with session: 95548fde-c7d9-4170-8e16-a37a1e6b2ca3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575257747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
1992024-02-12T01:34:12.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575278226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141,"session_id":"95548fde-c7d9-4170-8e16-a37a1e6b2ca3"}
2002024-02-12T01:34:12.660Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:11.575290796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
2012024-02-12T01:34:12.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-12T01:34:12.660ZFill test
2032024-02-12T01:34:12.711ZRead and Verify all blocks (0..600 range:false)
2042024-02-12T01:34:12.779ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-12T01:34:12.779ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-12T01:34:12.782Z----------------------------------------------------------------
2072024-02-12T01:34:12.782Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-12T01:34:12.782ZStates: Active Active Active
2092024-02-12T01:34:12.782ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-12T01:34:12.782Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-12T01:34:12.782Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-12T01:34:12.782Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-12T01:34:12.782Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-12T01:34:12.782Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-12T01:34:12.782Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-12T01:34:12.782Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-12T01:34:12.782Z New 0 0 0 0
2182024-02-12T01:34:12.782Z Sent 0 0 0 0
2192024-02-12T01:34:12.782Z Done 6 6 6 18
2202024-02-12T01:34:12.783Z Skipped 0 0 0 0
2212024-02-12T01:34:12.783Z Error 0 0 0 0
2222024-02-12T01:34:12.783ZLast Flush: 1006 1006 1006
2232024-02-12T01:34:12.783ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-12T01:34:12.783ZUpstairs last five completed: 13 12 11 10 9
2252024-02-12T01:34:12.783ZCLIENT: Up:0 ds:6 act:3
2262024-02-12T01:34:16.780Z----------------------------------------------------------------
2272024-02-12T01:34:16.780Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-12T01:34:16.781ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-12T01:34:16.781ZUpstairs last five completed: 14 13 12 11 10
2302024-02-12T01:34:16.781ZCLIENT: Up:0 ds:0 act:3
2312024-02-12T01:34:16.781ZCLIENT: All crucible jobs finished, exiting program
2322024-02-12T01:34:16.783Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:15.702088189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1141}
2332024-02-12T01:34:16.787Z
2342024-02-12T01:34:16.787ZBegin loop 1
2352024-02-12T01:34:16.787ZDownstairs to restart: 1
2362024-02-12T01:34:16.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.725735941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
2372024-02-12T01:34:16.809Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726019628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
2382024-02-12T01:34:16.809Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726070025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
2392024-02-12T01:34:16.809ZCrucible runtime is spawned
2402024-02-12T01:34:16.809Z{"msg":"Crucible 1edcfec9-5a50-489c-b4cf-11b3af2b6d97 has session id: 3e36373a-579f-4198-8a08-cc978c285698","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726110614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2412024-02-12T01:34:16.809Z{"msg":"Upstairs opts: Upstairs UUID: 1edcfec9-5a50-489c-b4cf-11b3af2b6d97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726167991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2422024-02-12T01:34:16.809Z{"msg":"Crucible stats registered with UUID: 1edcfec9-5a50-489c-b4cf-11b3af2b6d97","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726200129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2432024-02-12T01:34:16.809Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726235348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
2442024-02-12T01:34:16.809Z{"msg":"1edcfec9-5a50-489c-b4cf-11b3af2b6d97 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726285675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2452024-02-12T01:34:16.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726312174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2462024-02-12T01:34:16.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726338472Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2472024-02-12T01:34:16.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.72638455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2482024-02-12T01:34:16.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726423779Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2492024-02-12T01:34:16.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726467837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2502024-02-12T01:34:16.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726499805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2512024-02-12T01:34:16.809Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726544583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2522024-02-12T01:34:16.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.72659875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2532024-02-12T01:34:16.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726633198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2542024-02-12T01:34:16.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726663427Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2552024-02-12T01:34:16.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.726880697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2562024-02-12T01:34:16.810Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:15.727128555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2572024-02-12T01:34:16.810Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:15.727197822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2582024-02-12T01:34:16.810Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:15.727247729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2592024-02-12T01:34:16.810Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727270328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2602024-02-12T01:34:16.810Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727303097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2612024-02-12T01:34:16.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727357714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2622024-02-12T01:34:16.810Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.72744466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2632024-02-12T01:34:16.810Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727521626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2642024-02-12T01:34:16.810Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727566224Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2652024-02-12T01:34:16.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727602512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2662024-02-12T01:34:16.810Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.727969415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2672024-02-12T01:34:16.810Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728123358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2682024-02-12T01:34:16.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.72849667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2692024-02-12T01:34:16.810Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728532908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2702024-02-12T01:34:16.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728557687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2712024-02-12T01:34:16.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728614354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2722024-02-12T01:34:16.810Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728642403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2732024-02-12T01:34:16.810Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:15.728657062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2742024-02-12T01:34:17.807Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.728560328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2752024-02-12T01:34:17.807Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.728637144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2762024-02-12T01:34:17.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.729406107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2772024-02-12T01:34:17.812Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.729440546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2782024-02-12T01:34:17.812Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.729921212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2792024-02-12T01:34:17.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730452017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2802024-02-12T01:34:17.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730486156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2812024-02-12T01:34:17.812Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730501905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2822024-02-12T01:34:17.812Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730516644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2832024-02-12T01:34:17.812Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730537583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2842024-02-12T01:34:17.812Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730551642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2852024-02-12T01:34:17.812Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730571541Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2862024-02-12T01:34:17.812Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730585771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2872024-02-12T01:34:17.812ZWait for a query_work_queue command to finish before sending IO
2882024-02-12T01:34:17.812Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.73060943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2892024-02-12T01:34:17.812Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730624269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2902024-02-12T01:34:17.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730644638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2912024-02-12T01:34:17.813Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730658167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2922024-02-12T01:34:17.813Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730684116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2932024-02-12T01:34:17.813Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730697246Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2942024-02-12T01:34:17.813Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730711565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2952024-02-12T01:34:17.813Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730737424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2962024-02-12T01:34:17.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730750383Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2972024-02-12T01:34:17.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730763882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2982024-02-12T01:34:17.813Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730783901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
2992024-02-12T01:34:17.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730797551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
3002024-02-12T01:34:17.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.7308219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
3012024-02-12T01:34:17.813Z{"msg":"1edcfec9-5a50-489c-b4cf-11b3af2b6d97 is now active with session: 3e36373a-579f-4198-8a08-cc978c285698","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730835699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
3022024-02-12T01:34:17.813Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730847848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143,"session_id":"3e36373a-579f-4198-8a08-cc978c285698"}
3032024-02-12T01:34:17.813Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:16.730866687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
3042024-02-12T01:34:17.813ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3052024-02-12T01:34:17.813ZLoading write count information from file "/tmp/test_repair_verify.data"
3062024-02-12T01:34:17.813ZRun Repair workload
3072024-02-12T01:34:17.816Z01/30 Write block 511 len 4096 data: 2
3082024-02-12T01:34:17.816Z02/30 Write block 188 len 20480 data: 2 2 2 2 2
3092024-02-12T01:34:17.816Z03/30 Read block 408 len 12288
3102024-02-12T01:34:17.817Z04/30 Read block 202 len 40960
3112024-02-12T01:34:17.817Z05/30 Write block 211 len 24576 data: 2 2 2 2 2 2
3122024-02-12T01:34:17.819Z06/30 Write block 347 len 36864 data: 2 2 2 2 2 2 2 2 2
3132024-02-12T01:34:17.819Z07/30 Read block 43 len 20480
3142024-02-12T01:34:17.819Z08/30 Write block 234 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3152024-02-12T01:34:17.819Z09/30 Read block 356 len 12288
3162024-02-12T01:34:17.822Z10/30 Read block 259 len 32768
3172024-02-12T01:34:17.822Z11/30 Read block 305 len 28672
3182024-02-12T01:34:17.825Z12/30 Read block 550 len 28672
3192024-02-12T01:34:17.825Z13/30 Write block 537 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3202024-02-12T01:34:17.825Z14/30 Read block 428 len 12288
3212024-02-12T01:34:17.828Z15/30 Write block 76 len 16384 data: 2 2 2 2
3222024-02-12T01:34:17.828Z16/30 Read block 148 len 4096
3232024-02-12T01:34:17.828Z17/30 Read block 172 len 16384
3242024-02-12T01:34:17.828Z18/30 Read block 179 len 20480
3252024-02-12T01:34:17.831Z19/30 Write block 126 len 28672 data: 2 2 2 2 2 2 2
3262024-02-12T01:34:17.831Z20/30 Read block 71 len 8192
3272024-02-12T01:34:17.831Z21/30 Write block 457 len 12288 data: 2 2 2
3282024-02-12T01:34:17.834Z22/30 Write block 390 len 24576 data: 2 2 2 2 2 2
3292024-02-12T01:34:17.834Z23/30 Read block 295 len 40960
3302024-02-12T01:34:17.834Z24/30 Write block 402 len 4096 data: 2
3312024-02-12T01:34:17.834Z25/30 Read block 470 len 16384
3322024-02-12T01:34:17.837Z26/30 Read block 224 len 40960
3332024-02-12T01:34:17.837Z27/30 Write block 126 len 12288 data: 3 3 3
3342024-02-12T01:34:17.837Z28/30 Write block 575 len 24576 data: 2 2 2 2 2 2
3352024-02-12T01:34:17.840Z29/30 Write block 585 len 36864 data: 2 2 2 2 2 2 2 2 2
3362024-02-12T01:34:17.840Z30/30 Write block 480 len 28672 data: 2 2 2 2 2 2 2
3372024-02-12T01:34:17.840Z----------------------------------------------------------------
3382024-02-12T01:34:17.841Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3392024-02-12T01:34:17.841ZStates: Active Active Active
3402024-02-12T01:34:17.841ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3412024-02-12T01:34:17.841Z 1 Acked 1000 Write 1 Done Done Done false
3422024-02-12T01:34:17.841Z 2 Acked 1001 Write 5 Done Done Done false
3432024-02-12T01:34:17.841Z 3 Acked 1002 Read 3 Done Done Done false
3442024-02-12T01:34:17.841Z 4 Acked 1003 Read 10 Done Done Done false
3452024-02-12T01:34:17.841Z 5 Acked 1004 Write 6 Done Done Done false
3462024-02-12T01:34:17.841Z 6 Acked 1005 Write 9 Done Done Done false
3472024-02-12T01:34:17.841Z 7 Acked 1006 Read 5 Done Done Done false
3482024-02-12T01:34:17.841Z 8 Acked 1007 Write 10 Done Done Done false
3492024-02-12T01:34:17.841Z 9 Acked 1008 Read 3 Done Done Done false
3502024-02-12T01:34:17.841Z 10 Acked 1009 Read 8 Done Done Done false
3512024-02-12T01:34:17.841Z 11 Acked 1010 Read 7 Done Done Done false
3522024-02-12T01:34:17.841Z 12 Acked 1011 Read 7 Done Sent Done false
3532024-02-12T01:34:17.841Z 13 Acked 1012 Write 10 Done Sent Done false
3542024-02-12T01:34:17.841Z 14 Acked 1013 Read 3 Done Sent Done false
3552024-02-12T01:34:17.841Z 15 Acked 1014 Write 4 Done Sent Done false
3562024-02-12T01:34:17.841Z 16 Acked 1015 Read 1 Done Sent Done false
3572024-02-12T01:34:17.841Z 17 Acked 1016 Read 4 Done Sent Done false
3582024-02-12T01:34:17.841Z 18 Acked 1017 Read 5 Done Sent Done false
3592024-02-12T01:34:17.841Z 19 Acked 1018 Write 7 Done Sent Done false
3602024-02-12T01:34:17.841Z 20 Acked 1019 Read 2 Done Sent Done false
3612024-02-12T01:34:17.841Z 21 Acked 1020 Write 3 Done Sent Done false
3622024-02-12T01:34:17.841Z 22 Acked 1021 Write 6 Done Sent Done false
3632024-02-12T01:34:17.842Z 23 Acked 1022 Read 10 Done Sent Done false
3642024-02-12T01:34:17.842Z 24 Acked 1023 Write 1 Done Sent Done false
3652024-02-12T01:34:17.842Z 25 Acked 1024 Read 4 Done Sent Done false
3662024-02-12T01:34:17.842Z 26 Acked 1025 Read 10 Done Sent Done false
3672024-02-12T01:34:17.842Z 27 Acked 1026 Write 3 Done Sent Done false
3682024-02-12T01:34:17.842Z 28 Acked 1027 Write 6 Done Sent Done false
3692024-02-12T01:34:17.842Z 29 Acked 1028 Write 9 Done Sent Done false
3702024-02-12T01:34:17.842Z 30 Acked 1029 Write 7 Sent Sent Sent false
3712024-02-12T01:34:17.842Z STATES DS:0 DS:1 DS:2 TOTAL
3722024-02-12T01:34:17.842Z New 0 0 0 0
3732024-02-12T01:34:17.842Z Sent 1 19 1 21
3742024-02-12T01:34:17.842Z Done 29 11 29 69
3752024-02-12T01:34:17.842Z Skipped 0 0 0 0
3762024-02-12T01:34:17.842Z Error 0 0 0 0
3772024-02-12T01:34:17.842ZLast Flush: 0 0 0
3782024-02-12T01:34:17.842ZDownstairs last five completed:
3792024-02-12T01:34:17.842ZUpstairs last five completed: 30 29 28 27 26
3802024-02-12T01:34:17.842Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:16.760041975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1143}
3812024-02-12T01:34:17.842ZWrote out file "/tmp/test_repair_verify.data"
3822024-02-12T01:34:17.845Z
3832024-02-12T01:34:17.848ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3842024-02-12T01:34:17.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3852024-02-12T01:34:17.873Z 0 000-019 1 1 1 1 1 1 F F F
3862024-02-12T01:34:17.874Z 1 020-039 1 1 1 1 1 1 F F F
3872024-02-12T01:34:17.874Z 2 040-059 1 1 1 1 1 1 F F F
3882024-02-12T01:34:17.874Z 3 060-079 1 1 1 1 1 1 T F T <---
3892024-02-12T01:34:17.874Z 4 080-099 1 1 1 1 1 1 F F F
3902024-02-12T01:34:17.874Z 5 100-119 1 1 1 1 1 1 F F F
3912024-02-12T01:34:17.874Z 6 120-139 1 1 1 1 1 1 T F T <---
3922024-02-12T01:34:17.874Z 7 140-159 1 1 1 1 1 1 F F F
3932024-02-12T01:34:17.874Z 8 160-179 1 1 1 1 1 1 F F F
3942024-02-12T01:34:17.874Z 9 180-199 1 1 1 1 1 1 T T T
3952024-02-12T01:34:17.874Z 10 200-219 1 1 1 1 1 1 T T T
3962024-02-12T01:34:17.874Z 11 220-239 1 1 1 1 1 1 T T T
3972024-02-12T01:34:17.874Z 12 240-259 1 1 1 1 1 1 T T T
3982024-02-12T01:34:17.874Z 13 260-279 1 1 1 1 1 1 F F F
3992024-02-12T01:34:17.874Z 14 280-299 1 1 1 1 1 1 F F F
4002024-02-12T01:34:17.874Z 15 300-319 1 1 1 1 1 1 F F F
4012024-02-12T01:34:17.874Z 16 320-339 1 1 1 1 1 1 F F F
4022024-02-12T01:34:17.874Z 17 340-359 1 1 1 1 1 1 T T T
4032024-02-12T01:34:17.874Z 18 360-379 1 1 1 1 1 1 F F F
4042024-02-12T01:34:17.874Z 19 380-399 1 1 1 1 1 1 T F T <---
4052024-02-12T01:34:17.874Z 20 400-419 1 1 1 1 1 1 T F T <---
4062024-02-12T01:34:17.874Z 21 420-439 1 1 1 1 1 1 F F F
4072024-02-12T01:34:17.874Z 22 440-459 1 1 1 1 1 1 T F T <---
4082024-02-12T01:34:17.874Z 23 460-479 1 1 1 1 1 1 F F F
4092024-02-12T01:34:17.874Z 24 480-499 1 1 1 1 1 1 T F T <---
4102024-02-12T01:34:17.875Z 25 500-519 1 1 1 1 1 1 T T T
4112024-02-12T01:34:17.875Z 26 520-539 1 1 1 1 1 1 T F T <---
4122024-02-12T01:34:17.875Z 27 540-559 1 1 1 1 1 1 T F T <---
4132024-02-12T01:34:17.875Z 28 560-579 1 1 1 1 1 1 T F T <---
4142024-02-12T01:34:17.875Z 29 580-599 1 1 1 1 1 1 T F T <---
4152024-02-12T01:34:17.875ZMax gen: 1, Max flush: 1
4162024-02-12T01:34:17.875ZError: Difference in extent metadata found!
4172024-02-12T01:34:17.875ZA Difference in extent metadata is expected here
4182024-02-12T01:34:17.875ZOn loop 1
4192024-02-12T01:34:17.875Z
4202024-02-12T01:34:17.878ZVerifying data now
4212024-02-12T01:34:24.550ZNo change after verify
4222024-02-12T01:34:24.550ZLoop: 1 Downstairs dump after verify (and repair):
4232024-02-12T01:34:24.565ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4242024-02-12T01:34:24.565Z 0 000-019 1 1 1 1 1 1 F F F
4252024-02-12T01:34:24.565Z 1 020-039 1 1 1 1 1 1 F F F
4262024-02-12T01:34:24.565Z 2 040-059 1 1 1 1 1 1 F F F
4272024-02-12T01:34:24.566Z 3 060-079 2 2 2 2 2 2 F F F
4282024-02-12T01:34:24.566Z 4 080-099 1 1 1 1 1 1 F F F
4292024-02-12T01:34:24.566Z 5 100-119 1 1 1 1 1 1 F F F
4302024-02-12T01:34:24.566Z 6 120-139 2 2 2 2 2 2 F F F
4312024-02-12T01:34:24.566Z 7 140-159 1 1 1 1 1 1 F F F
4322024-02-12T01:34:24.569Z 8 160-179 1 1 1 1 1 1 F F F
4332024-02-12T01:34:24.569Z 9 180-199 2 2 2 2 2 2 F F F
4342024-02-12T01:34:24.569Z 10 200-219 2 2 2 2 2 2 F F F
4352024-02-12T01:34:24.569Z 11 220-239 2 2 2 2 2 2 F F F
4362024-02-12T01:34:24.569Z 12 240-259 2 2 2 2 2 2 F F F
4372024-02-12T01:34:24.569Z 13 260-279 1 1 1 1 1 1 F F F
4382024-02-12T01:34:24.569Z 14 280-299 1 1 1 1 1 1 F F F
4392024-02-12T01:34:24.569Z 15 300-319 1 1 1 1 1 1 F F F
4402024-02-12T01:34:24.569Z 16 320-339 1 1 1 1 1 1 F F F
4412024-02-12T01:34:24.569Z 17 340-359 2 2 2 2 2 2 F F F
4422024-02-12T01:34:24.569Z 18 360-379 1 1 1 1 1 1 F F F
4432024-02-12T01:34:24.569Z 19 380-399 2 2 2 2 2 2 F F F
4442024-02-12T01:34:24.569Z 20 400-419 2 2 2 2 2 2 F F F
4452024-02-12T01:34:24.569Z 21 420-439 1 1 1 1 1 1 F F F
4462024-02-12T01:34:24.569Z 22 440-459 2 2 2 2 2 2 F F F
4472024-02-12T01:34:24.569Z 23 460-479 1 1 1 1 1 1 F F F
4482024-02-12T01:34:24.569Z 24 480-499 2 2 2 2 2 2 F F F
4492024-02-12T01:34:24.569Z 25 500-519 2 2 2 2 2 2 F F F
4502024-02-12T01:34:24.569Z 26 520-539 2 2 2 2 2 2 F F F
4512024-02-12T01:34:24.569Z 27 540-559 2 2 2 2 2 2 F F F
4522024-02-12T01:34:24.569Z 28 560-579 2 2 2 2 2 2 F F F
4532024-02-12T01:34:24.570Z 29 580-599 2 2 2 2 2 2 F F F
4542024-02-12T01:34:24.570ZMax gen: 2, Max flush: 2
4552024-02-12T01:34:24.570Z
4562024-02-12T01:34:24.570ZBegin loop 2
4572024-02-12T01:34:24.570ZDownstairs to restart: 1
4582024-02-12T01:34:24.586Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506424121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
4592024-02-12T01:34:24.586Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
4602024-02-12T01:34:24.590Z9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506775625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
4612024-02-12T01:34:24.590Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506820163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
4622024-02-12T01:34:24.591Z{"msg":"Crucible 065cc96e-a1dc-4880-882a-cd1ac84ec08b has session id: c01e1695-a372-42a8-9f12-e0d3a0f206f3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506849121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4632024-02-12T01:34:24.591Z{"msg":"Upstairs opts: Upstairs UUID: 065cc96e-a1dc-4880-882a-cd1ac84ec08b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506904748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4642024-02-12T01:34:24.591Z{"msg":"Crucible stats registered with UUID: 065cc96e-a1dc-4880-882a-cd1ac84ec08b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506950816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4652024-02-12T01:34:24.591Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506983975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
4662024-02-12T01:34:24.591Z{"msg":"065cc96e-a1dc-4880-882a-cd1ac84ec08b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.506999624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4672024-02-12T01:34:24.591Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507034582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4682024-02-12T01:34:24.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507065211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4692024-02-12T01:34:24.591Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507095939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4702024-02-12T01:34:24.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507126348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4712024-02-12T01:34:24.591Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507168796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4722024-02-12T01:34:24.591Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507219893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4732024-02-12T01:34:24.591Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.50728735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4742024-02-12T01:34:24.591Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507378636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4752024-02-12T01:34:24.591Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507401965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4762024-02-12T01:34:24.591Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507452772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4772024-02-12T01:34:24.591Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.50750781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4782024-02-12T01:34:24.591Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.507553448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4792024-02-12T01:34:24.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508047654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4802024-02-12T01:34:24.591Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508081872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4812024-02-12T01:34:24.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508097571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4822024-02-12T01:34:24.591Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.50812089Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4832024-02-12T01:34:24.591Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508654195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4842024-02-12T01:34:24.592Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508691403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4852024-02-12T01:34:24.592Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.508925662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4862024-02-12T01:34:24.592Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.50896176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4872024-02-12T01:34:24.592Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509217308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4882024-02-12T01:34:24.592Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509288515Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4892024-02-12T01:34:24.592Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509328733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4902024-02-12T01:34:24.592Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509347652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4912024-02-12T01:34:24.592Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.50937853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4922024-02-12T01:34:24.592Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509408209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4932024-02-12T01:34:24.592Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509457757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4942024-02-12T01:34:24.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509864657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4952024-02-12T01:34:24.596Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509920715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4962024-02-12T01:34:24.596Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509958493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4972024-02-12T01:34:24.596Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509975202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4982024-02-12T01:34:24.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.509997741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
4992024-02-12T01:34:24.596Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.51001415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5002024-02-12T01:34:24.596Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510028919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5012024-02-12T01:34:24.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
5022024-02-12T01:34:24.596Z2024-02-12T01:34:23.510050488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5032024-02-12T01:34:24.596Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510067308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5042024-02-12T01:34:24.596Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510088217Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5052024-02-12T01:34:24.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510103206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5062024-02-12T01:34:24.596Z{"msg":"Max found gen is 3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510124195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5072024-02-12T01:34:24.596Z{"msg":"Generation requested: 4 >= found:3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510138024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5082024-02-12T01:34:24.596Z{"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510158213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5092024-02-12T01:34:24.597Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510171773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5102024-02-12T01:34:24.597Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510192002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5112024-02-12T01:34:24.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510205131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5122024-02-12T01:34:24.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.51021867Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5132024-02-12T01:34:24.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510240019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5142024-02-12T01:34:24.597Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510254639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5152024-02-12T01:34:24.597Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510274848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5162024-02-12T01:34:24.597Z{"msg":"065cc96e-a1dc-4880-882a-cd1ac84ec08b is now active with session: c01e1695-a372-42a8-9f12-e0d3a0f206f3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510287927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5172024-02-12T01:34:24.597Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510306826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151,"session_id":"c01e1695-a372-42a8-9f12-e0d3a0f206f3"}
5182024-02-12T01:34:24.597Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:23.510319036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
5192024-02-12T01:34:24.597ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5202024-02-12T01:34:24.597ZLoading write count information from file "/tmp/test_repair_verify.data"
5212024-02-12T01:34:24.597ZRun Repair workload
5222024-02-12T01:34:24.597Z01/30 Write block 199 len 28672 data: 2 2 2 2 2 2 2
5232024-02-12T01:34:24.597Z02/30 Write block 61 len 24576 data: 2 2 2 2 2 2
5242024-02-12T01:34:24.597Z03/30 Read block 151 len 16384
5252024-02-12T01:34:24.597Z04/30 Flush
5262024-02-12T01:34:24.600Z05/30 Write block 365 len 12288 data: 2 2 2
5272024-02-12T01:34:24.600Z06/30 Read block 390 len 4096
5282024-02-12T01:34:24.601Z07/30 Flush
5292024-02-12T01:34:24.601Z08/30 Write block 57 len 16384 data: 2 2 2 2
5302024-02-12T01:34:24.601Z09/30 Write block 528 len 16384 data: 2 2 2 2
5312024-02-12T01:34:24.601Z10/30 Read block 459 len 40960
5322024-02-12T01:34:24.604Z11/30 Write block 14 len 32768 data: 2 2 2 2 2 2 2 2
5332024-02-12T01:34:24.604Z12/30 Read block 30 len 12288
5342024-02-12T01:34:24.604Z13/30 Read block 63 len 40960
5352024-02-12T01:34:24.609Z14/30 Write block 44 len 16384 data: 2 2 2 2
5362024-02-12T01:34:24.609Z15/30 Read block 369 len 12288
5372024-02-12T01:34:24.610Z16/30 Read block 486 len 20480
5382024-02-12T01:34:24.612Z17/30 Write block 499 len 12288 data: 2 2 2
5392024-02-12T01:34:24.612Z18/30 Write block 495 len 28672 data: 2 2 2 2 3 3 3
5402024-02-12T01:34:24.612Z19/30 Read block 214 len 20480
5412024-02-12T01:34:24.613Z20/30 Read block 158 len 28672
5422024-02-12T01:34:24.613Z21/30 Write block 298 len 16384 data: 2 2 2 2
5432024-02-12T01:34:24.613Z22/30 Read block 131 len 4096
5442024-02-12T01:34:24.613Z23/30 Read block 429 len 28672
5452024-02-12T01:34:24.613Z24/30 Read block 431 len 32768
5462024-02-12T01:34:24.615Z25/30 Write block 223 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5472024-02-12T01:34:24.615Z26/30 Write block 138 len 12288 data: 2 2 2
5482024-02-12T01:34:24.615Z27/30 Write block 146 len 8192 data: 2 2
5492024-02-12T01:34:24.615Z28/30 Read block 255 len 24576
5502024-02-12T01:34:24.619Z29/30 Read block 10 len 24576
5512024-02-12T01:34:24.619Z30/30 Write block 390 len 12288 data: 3 3 3
5522024-02-12T01:34:24.619Z----------------------------------------------------------------
5532024-02-12T01:34:24.619Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:26
5542024-02-12T01:34:24.619ZStates: Active Active Active
5552024-02-12T01:34:24.619ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5562024-02-12T01:34:24.619Z 5 Acked 1004 Write 3 Done Done Done false
5572024-02-12T01:34:24.619Z 6 Acked 1005 Read 1 Done Done Done false
5582024-02-12T01:34:24.619Z 7 Acked 1006 Flush 0 Done Sent Done false
5592024-02-12T01:34:24.619Z 8 Acked 1007 Write 4 Done Sent Done false
5602024-02-12T01:34:24.619Z 9 Acked 1008 Write 4 Done Sent Done false
5612024-02-12T01:34:24.619Z 10 Acked 1009 Read 10 Done Sent Done false
5622024-02-12T01:34:24.619Z 11 Acked 1010 Write 8 Done Sent Done false
5632024-02-12T01:34:24.619Z 12 Acked 1011 Read 3 Done Sent Done false
5642024-02-12T01:34:24.619Z 13 Acked 1012 Read 10 Done Sent Done false
5652024-02-12T01:34:24.619Z 14 Acked 1013 Write 4 Done Sent Done false
5662024-02-12T01:34:24.619Z 15 Acked 1014 Read 3 Done Sent Done false
5672024-02-12T01:34:24.619Z 16 Acked 1015 Read 5 Done Sent Done false
5682024-02-12T01:34:24.619Z 17 Acked 1016 Write 3 Done Sent Done false
5692024-02-12T01:34:24.619Z 18 Acked 1017 Write 7 Done Sent Done false
5702024-02-12T01:34:24.619Z 19 Acked 1018 Read 5 Done Sent Done false
5712024-02-12T01:34:24.620Z 20 Acked 1019 Read 7 Done Sent Done false
5722024-02-12T01:34:24.620Z 21 Acked 1020 Write 4 Done Sent Done false
5732024-02-12T01:34:24.620Z 22 Acked 1021 Read 1 Done Sent Done false
5742024-02-12T01:34:24.620Z 23 Acked 1022 Read 7 Done Sent Done false
5752024-02-12T01:34:24.620Z 24 Acked 1023 Read 8 Done Sent Done false
5762024-02-12T01:34:24.620Z 25 Acked 1024 Write 10 Done Sent Done false
5772024-02-12T01:34:24.620Z 26 Acked 1025 Write 3 Done Sent Done false
5782024-02-12T01:34:24.620Z 27 Acked 1026 Write 2 Done Sent Done false
5792024-02-12T01:34:24.620Z 28 Acked 1027 Read 6 Done Sent Done false
5802024-02-12T01:34:24.620Z 29 Acked 1028 Read 6 Done Sent Done false
5812024-02-12T01:34:24.620Z 30 Acked 1029 Write 3 Sent Sent Sent false
5822024-02-12T01:34:24.620Z STATES DS:0 DS:1 DS:2 TOTAL
5832024-02-12T01:34:24.620Z New 0 0 0 0
5842024-02-12T01:34:24.620Z Sent 1 24 1 26
5852024-02-12T01:34:24.620Z Done 25 2 25 52
5862024-02-12T01:34:24.620Z Skipped 0 0 0 0
5872024-02-12T01:34:24.620Z Error 0 0 0 0
5882024-02-12T01:34:24.620ZLast Flush: 1006 1003 1006
5892024-02-12T01:34:24.620ZDownstairs last five completed: 1003 1002 1001 1000
5902024-02-12T01:34:24.620ZUpstairs last five completed: 30 29 28 27 26
5912024-02-12T01:34:24.620Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:23.538608626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1151}
5922024-02-12T01:34:24.620ZWrote out file "/tmp/test_repair_verify.data"
5932024-02-12T01:34:24.623Z
5942024-02-12T01:34:24.626ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5952024-02-12T01:34:24.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5962024-02-12T01:34:24.645Z 0 000-019 1 1 1 1 1 1 T F T <---
5972024-02-12T01:34:24.645Z 1 020-039 1 1 1 1 1 1 T F T <---
5982024-02-12T01:34:24.645Z 2 040-059 1 1 1 1 1 1 T F T <---
5992024-02-12T01:34:24.645Z 3 060-079 4 4 4 3 3 3 T F T <---
6002024-02-12T01:34:24.645Z 4 080-099 1 1 1 1 1 1 F F F
6012024-02-12T01:34:24.645Z 5 100-119 1 1 1 1 1 1 F F F
6022024-02-12T01:34:24.648Z 6 120-139 2 2 2 2 2 2 T F T <---
6032024-02-12T01:34:24.648Z 7 140-159 1 1 1 1 1 1 T F T <---
6042024-02-12T01:34:24.648Z 8 160-179 1 1 1 1 1 1 F F F
6052024-02-12T01:34:24.648Z 9 180-199 4 4 4 3 3 3 F F F
6062024-02-12T01:34:24.648Z 10 200-219 4 4 4 3 3 3 F F F
6072024-02-12T01:34:24.648Z 11 220-239 2 2 2 2 2 2 T F T <---
6082024-02-12T01:34:24.648Z 12 240-259 2 2 2 2 2 2 F F F
6092024-02-12T01:34:24.648Z 13 260-279 1 1 1 1 1 1 F F F
6102024-02-12T01:34:24.648Z 14 280-299 1 1 1 1 1 1 T F T <---
6112024-02-12T01:34:24.648Z 15 300-319 1 1 1 1 1 1 T F T <---
6122024-02-12T01:34:24.648Z 16 320-339 1 1 1 1 1 1 F F F
6132024-02-12T01:34:24.648Z 17 340-359 2 2 2 2 2 2 F F F
6142024-02-12T01:34:24.649Z 18 360-379 4 1 4 4 1 4 F T F <---
6152024-02-12T01:34:24.649Z 19 380-399 2 2 2 2 2 2 T F T <---
6162024-02-12T01:34:24.649Z 20 400-419 2 2 2 2 2 2 F F F
6172024-02-12T01:34:24.649Z 21 420-439 1 1 1 1 1 1 F F F
6182024-02-12T01:34:24.649Z 22 440-459 2 2 2 2 2 2 F F F
6192024-02-12T01:34:24.649Z 23 460-479 1 1 1 1 1 1 F F F
6202024-02-12T01:34:24.649Z 24 480-499 2 2 2 2 2 2 T F T <---
6212024-02-12T01:34:24.649Z 25 500-519 2 2 2 2 2 2 T F T <---
6222024-02-12T01:34:24.649Z 26 520-539 2 2 2 2 2 2 T F T <---
6232024-02-12T01:34:24.649Z 27 540-559 2 2 2 2 2 2 F F F
6242024-02-12T01:34:24.649Z 28 560-579 2 2 2 2 2 2 F F F
6252024-02-12T01:34:24.649Z 29 580-599 2 2 2 2 2 2 F F F
6262024-02-12T01:34:24.649ZMax gen: 4, Max flush: 4
6272024-02-12T01:34:24.649ZError: Difference in extent metadata found!
6282024-02-12T01:34:24.649ZA Difference in extent metadata is expected here
6292024-02-12T01:34:24.649ZOn loop 2
6302024-02-12T01:34:24.649Z
6312024-02-12T01:34:24.652ZVerifying data now
6322024-02-12T01:34:31.487ZNo change after verify
6332024-02-12T01:34:31.487ZLoop: 2 Downstairs dump after verify (and repair):
6342024-02-12T01:34:31.499ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6352024-02-12T01:34:31.499Z 0 000-019 5 5 5 5 5 5 F F F
6362024-02-12T01:34:31.499Z 1 020-039 5 5 5 5 5 5 F F F
6372024-02-12T01:34:31.499Z 2 040-059 5 5 5 5 5 5 F F F
6382024-02-12T01:34:31.499Z 3 060-079 5 5 5 5 5 5 F F F
6392024-02-12T01:34:31.499Z 4 080-099 1 1 1 1 1 1 F F F
6402024-02-12T01:34:31.499Z 5 100-119 1 1 1 1 1 1 F F F
6412024-02-12T01:34:31.499Z 6 120-139 5 5 5 5 5 5 F F F
6422024-02-12T01:34:31.499Z 7 140-159 5 5 5 5 5 5 F F F
6432024-02-12T01:34:31.499Z 8 160-179 1 1 1 1 1 1 F F F
6442024-02-12T01:34:31.499Z 9 180-199 4 4 4 3 3 3 F F F
6452024-02-12T01:34:31.499Z 10 200-219 4 4 4 3 3 3 F F F
6462024-02-12T01:34:31.499Z 11 220-239 5 5 5 5 5 5 F F F
6472024-02-12T01:34:31.499Z 12 240-259 2 2 2 2 2 2 F F F
6482024-02-12T01:34:31.499Z 13 260-279 1 1 1 1 1 1 F F F
6492024-02-12T01:34:31.500Z 14 280-299 5 5 5 5 5 5 F F F
6502024-02-12T01:34:31.500Z 15 300-319 5 5 5 5 5 5 F F F
6512024-02-12T01:34:31.500Z 16 320-339 1 1 1 1 1 1 F F F
6522024-02-12T01:34:31.500Z 17 340-359 2 2 2 2 2 2 F F F
6532024-02-12T01:34:31.500Z 18 360-379 4 4 4 4 4 4 F F F
6542024-02-12T01:34:31.500Z 19 380-399 5 5 5 5 5 5 F F F
6552024-02-12T01:34:31.500Z 20 400-419 2 2 2 2 2 2 F F F
6562024-02-12T01:34:31.500Z 21 420-439 1 1 1 1 1 1 F F F
6572024-02-12T01:34:31.500Z 22 440-459 2 2 2 2 2 2 F F F
6582024-02-12T01:34:31.500Z 23 460-479 1 1 1 1 1 1 F F F
6592024-02-12T01:34:31.500Z 24 480-499 5 5 5 5 5 5 F F F
6602024-02-12T01:34:31.500Z 25 500-519 5 5 5 5 5 5 F F F
6612024-02-12T01:34:31.500Z 26 520-539 5 5 5 5 5 5 F F F
6622024-02-12T01:34:31.500Z 27 540-559 2 2 2 2 2 2 F F F
6632024-02-12T01:34:31.500Z 28 560-579 2 2 2 2 2 2 F F F
6642024-02-12T01:34:31.500Z 29 580-599 2 2 2 2 2 2 F F F
6652024-02-12T01:34:31.500ZMax gen: 5, Max flush: 5
6662024-02-12T01:34:31.500Z
6672024-02-12T01:34:31.500ZBegin loop 3
6682024-02-12T01:34:31.500ZDownstairs to restart: 1
6692024-02-12T01:34:31.500Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032268207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
6702024-02-12T01:34:31.501Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032537265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
6712024-02-12T01:34:31.501Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032572063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
6722024-02-12T01:34:31.501Z{"msg":"Crucible a5e0c116-4443-47d1-a680-5e230653d032 has session id: db7fcf89-f571-4f51-8602-e14d2e0d5a8c","v":0,"name":"crucible","level":30Crucible runtime is spawned
6732024-02-12T01:34:31.501Z,"time":"2024-02-12T01:34:30.032600312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6742024-02-12T01:34:31.501Z{"msg":"Upstairs opts: Upstairs UUID: a5e0c116-4443-47d1-a680-5e230653d032, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032651029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6752024-02-12T01:34:31.501Z{"msg":"Crucible stats registered with UUID: a5e0c116-4443-47d1-a680-5e230653d032","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032689158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6762024-02-12T01:34:31.501Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032717526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
6772024-02-12T01:34:31.501Z{"msg":"a5e0c116-4443-47d1-a680-5e230653d032 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032829561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6782024-02-12T01:34:31.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032872159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6792024-02-12T01:34:31.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032923336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6802024-02-12T01:34:31.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032952105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6812024-02-12T01:34:31.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.032985943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6822024-02-12T01:34:31.501Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033009722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6832024-02-12T01:34:31.501Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033032991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6842024-02-12T01:34:31.501Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033082949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6852024-02-12T01:34:31.501Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033116187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6862024-02-12T01:34:31.501Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033151025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6872024-02-12T01:34:31.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033184624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6882024-02-12T01:34:31.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033216562Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6892024-02-12T01:34:31.502Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033241041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6902024-02-12T01:34:31.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033876011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6912024-02-12T01:34:31.502Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033918009Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6922024-02-12T01:34:31.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.033977966Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6932024-02-12T01:34:31.502Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.034016574Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6942024-02-12T01:34:31.502Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.034473992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6952024-02-12T01:34:31.502Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.034626025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6962024-02-12T01:34:31.502Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.034952699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6972024-02-12T01:34:31.502Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035021216Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6982024-02-12T01:34:31.502Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035064194Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
6992024-02-12T01:34:31.502Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035091193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7002024-02-12T01:34:31.503Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035119111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7012024-02-12T01:34:31.503Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035142901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7022024-02-12T01:34:31.503Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035199588Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7032024-02-12T01:34:31.503Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035236796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7042024-02-12T01:34:31.503Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.035638257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7052024-02-12T01:34:31.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036020159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7062024-02-12T01:34:31.503Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036072946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7072024-02-12T01:34:31.503Z{"msg":"[0]R flush_numbers[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 3, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036107244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7082024-02-12T01:34:31.503Z{"msg":"[0]R generation[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036122944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7092024-02-12T01:34:31.503Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036144713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7102024-02-12T01:34:31.503Z{"msg":"[1]R flush_numbers[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 3, 3, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036159672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7112024-02-12T01:34:31.503Z{"msg":"[1]R generation[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036174751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7122024-02-12T01:34:31.503Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.03619695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7132024-02-12T01:34:31.503Z{"msg":"[2]R flush_numbers[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 3, 3, 5]"Wait for a query_work_queue command to finish before sending IO
7142024-02-12T01:34:31.503Z,"v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036213719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7152024-02-12T01:34:31.503Z{"msg":"[2]R generation[0..12]: [5, 5, 5, 5, 1, 1, 5, 5, 1, 4, 4, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036227789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7162024-02-12T01:34:31.503Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036248438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7172024-02-12T01:34:31.504Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036262707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7182024-02-12T01:34:31.504Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036276996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7192024-02-12T01:34:31.504Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036289946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7202024-02-12T01:34:31.504Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036310815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7212024-02-12T01:34:31.504Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036324344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7222024-02-12T01:34:31.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036336673Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7232024-02-12T01:34:31.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036356472Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7242024-02-12T01:34:31.504Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036370082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7252024-02-12T01:34:31.504Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036390181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7262024-02-12T01:34:31.504Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.03640278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7272024-02-12T01:34:31.504Z{"msg":"a5e0c116-4443-47d1-a680-5e230653d032 is now active with session: db7fcf89-f571-4f51-8602-e14d2e0d5a8c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.03641587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7282024-02-12T01:34:31.504Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036427669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159,"session_id":"db7fcf89-f571-4f51-8602-e14d2e0d5a8c"}
7292024-02-12T01:34:31.504Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:30.036440579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
7302024-02-12T01:34:31.505ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7312024-02-12T01:34:31.505ZLoading write count information from file "/tmp/test_repair_verify.data"
7322024-02-12T01:34:31.505ZRun Repair workload
7332024-02-12T01:34:31.505Z01/30 Write block 43 len 12288 data: 2 3 3
7342024-02-12T01:34:31.523Z02/30 Write block 149 len 24576 data: 2 2 2 2 2 2
7352024-02-12T01:34:31.523Z03/30 Read block 564 len 8192
7362024-02-12T01:34:31.523Z04/30 Read block 312 len 12288
7372024-02-12T01:34:31.523Z05/30 Write block 585 len 32768 data: 3 3 3 3 3 3 3 3
7382024-02-12T01:34:31.523Z06/30 Read block 130 len 20480
7392024-02-12T01:34:31.523Z07/30 Read block 102 len 8192
7402024-02-12T01:34:31.523Z08/30 Write block 477 len 16384 data: 2 2 2 3
7412024-02-12T01:34:31.524Z09/30 Write block 599 len 4096 data: 2
7422024-02-12T01:34:31.524Z10/30 Read block 127 len 24576
7432024-02-12T01:34:31.524Z11/30 Write block 144 len 12288 data: 2 2 3
7442024-02-12T01:34:31.524Z12/30 Write block 415 len 4096 data: 2
7452024-02-12T01:34:31.524Z13/30 Write block 95 len 16384 data: 2 2 2 2
7462024-02-12T01:34:31.524Z14/30 Read block 78 len 16384
7472024-02-12T01:34:31.524Z15/30 Write block 595 len 16384 data: 2 2 2 2
7482024-02-12T01:34:31.524Z16/30 Flush
7492024-02-12T01:34:31.524Z17/30 Write block 322 len 12288 data: 2 2 2
7502024-02-12T01:34:31.525Z18/30 Write block 117 len 24576 data: 2 2 2 2 2 2
7512024-02-12T01:34:31.525Z19/30 Write block 80 len 20480 data: 2 2 2 2 2
7522024-02-12T01:34:31.525Z20/30 Write block 203 len 28672 data: 3 3 3 2 2 2 2
7532024-02-12T01:34:31.525Z21/30 Read block 262 len 40960
7542024-02-12T01:34:31.525Z22/30 Read block 130 len 16384
7552024-02-12T01:34:31.525Z23/30 Flush
7562024-02-12T01:34:31.525Z24/30 Write block 225 len 36864 data: 3 3 3 3 3 3 3 3 2
7572024-02-12T01:34:31.525Z25/30 Write block 26 len 36864 data: 2 2 2 2 2 2 2 2 2
7582024-02-12T01:34:31.525Z26/30 Read block 345 len 40960
7592024-02-12T01:34:31.525Z27/30 Read block 246 len 12288
7602024-02-12T01:34:31.526Z28/30 Write block 304 len 4096 data: 2
7612024-02-12T01:34:31.526Z29/30 Read block 408 len 40960
7622024-02-12T01:34:31.526Z30/30 Read block 325 len 24576
7632024-02-12T01:34:31.526Z----------------------------------------------------------------
7642024-02-12T01:34:31.526Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7652024-02-12T01:34:31.526ZStates: Active Active Active
7662024-02-12T01:34:31.526ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7672024-02-12T01:34:31.526Z 1 Acked 1000 Write 3 Done Done Done false
7682024-02-12T01:34:31.526Z 2 Acked 1001 Write 6 Done Done Done false
7692024-02-12T01:34:31.526Z 3 Acked 1002 Read 2 Done Sent Done false
7702024-02-12T01:34:31.526Z 4 Acked 1003 Read 3 Done Sent Done false
7712024-02-12T01:34:31.526Z 5 Acked 1004 Write 8 Done Sent Done false
7722024-02-12T01:34:31.526Z 6 Acked 1005 Read 5 Done Sent Done false
7732024-02-12T01:34:31.527Z 7 Acked 1006 Read 2 Done Sent Done false
7742024-02-12T01:34:31.527Z 8 Acked 1007 Write 4 Done Sent Done false
7752024-02-12T01:34:31.527Z 9 Acked 1008 Write 1 Done Sent Done false
7762024-02-12T01:34:31.527Z 10 Acked 1009 Read 6 Done Sent Done false
7772024-02-12T01:34:31.527Z 11 Acked 1010 Write 3 Done Sent Done false
7782024-02-12T01:34:31.527Z 12 Acked 1011 Write 1 Done Sent Done false
7792024-02-12T01:34:31.527Z 13 Acked 1012 Write 4 Done Sent Done false
7802024-02-12T01:34:31.527Z 14 Acked 1013 Read 4 Done Sent Done false
7812024-02-12T01:34:31.527Z 15 Acked 1014 Write 4 Done Sent Done false
7822024-02-12T01:34:31.527Z 16 Acked 1015 Flush 0 Done Sent Done false
7832024-02-12T01:34:31.528Z 17 Acked 1016 Write 3 Done Sent Done false
7842024-02-12T01:34:31.528Z 18 Acked 1017 Write 6 Done Sent Done false
7852024-02-12T01:34:31.528Z 19 Acked 1018 Write 5 Done Sent Done false
7862024-02-12T01:34:31.528Z 20 Acked 1019 Write 7 Done Sent Done false
7872024-02-12T01:34:31.528Z 21 Acked 1020 Read 10 Done Sent Done false
7882024-02-12T01:34:31.528Z 22 Acked 1021 Read 4 Done Sent Done false
7892024-02-12T01:34:31.528Z 23 Acked 1022 Flush 0 Done Sent Done false
7902024-02-12T01:34:31.528Z 24 Acked 1023 Write 9 Done Sent Done false
7912024-02-12T01:34:31.528Z 25 Acked 1024 Write 9 Done Sent Done false
7922024-02-12T01:34:31.528Z 26 Acked 1025 Read 10 Done Sent Done false
7932024-02-12T01:34:31.529Z 27 Acked 1026 Read 3 Done Sent Done false
7942024-02-12T01:34:31.529Z 28 Acked 1027 Write 1 Done Sent Done false
7952024-02-12T01:34:31.529Z 29 Acked 1028 Read 10 Done Sent Done false
7962024-02-12T01:34:31.529Z 30 Acked 1029 Read 6 Done Sent Done false
7972024-02-12T01:34:31.529Z STATES DS:0 DS:1 DS:2 TOTAL
7982024-02-12T01:34:31.529Z New 0 0 0 0
7992024-02-12T01:34:31.529Z Sent 0 28 0 28
8002024-02-12T01:34:31.529Z Done 30 2 30 62
8012024-02-12T01:34:31.529Z Skipped 0 0 0 0
8022024-02-12T01:34:31.529Z Error 0 0 0 0
8032024-02-12T01:34:31.529ZLast Flush: 1022 0 1022
8042024-02-12T01:34:31.529ZDownstairs last five completed:
8052024-02-12T01:34:31.530ZUpstairs last five completed: 30 29 28 27 26
8062024-02-12T01:34:31.530Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:30.0647068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1159}
8072024-02-12T01:34:31.530ZWrote out file "/tmp/test_repair_verify.data"
8082024-02-12T01:34:31.530Z
8092024-02-12T01:34:31.530ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8102024-02-12T01:34:31.530ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8112024-02-12T01:34:31.530Z 0 000-019 5 5 5 5 5 5 F F F
8122024-02-12T01:34:31.530Z 1 020-039 5 5 5 5 5 5 T F T <---
8132024-02-12T01:34:31.530Z 2 040-059 6 5 6 6 5 6 F T F <---
8142024-02-12T01:34:31.530Z 3 060-079 5 5 5 5 5 5 F F F
8152024-02-12T01:34:31.531Z 4 080-099 6 1 6 7 1 7 F F F <---
8162024-02-12T01:34:31.531Z 5 100-119 6 1 6 7 1 7 F F F <---
8172024-02-12T01:34:31.531Z 6 120-139 6 5 6 7 5 7 F F F <---
8182024-02-12T01:34:31.531Z 7 140-159 6 5 6 6 5 6 F T F <---
8192024-02-12T01:34:31.531Z 8 160-179 1 1 1 1 1 1 F F F
8202024-02-12T01:34:31.531Z 9 180-199 4 4 4 3 3 3 F F F
8212024-02-12T01:34:31.531Z 10 200-219 6 4 6 7 3 7 F F F <---
8222024-02-12T01:34:31.531Z 11 220-239 5 5 5 5 5 5 T F T <---
8232024-02-12T01:34:31.531Z 12 240-259 2 2 2 2 2 2 F F F
8242024-02-12T01:34:31.531Z 13 260-279 1 1 1 1 1 1 F F F
8252024-02-12T01:34:31.532Z 14 280-299 5 5 5 5 5 5 F F F
8262024-02-12T01:34:31.532Z 15 300-319 5 5 5 5 5 5 T F T <---
8272024-02-12T01:34:31.532Z 16 320-339 6 1 6 7 1 7 F F F <---
8282024-02-12T01:34:31.532Z 17 340-359 2 2 2 2 2 2 F F F
8292024-02-12T01:34:31.532Z 18 360-379 4 4 4 4 4 4 F F F
8302024-02-12T01:34:31.532Z 19 380-399 5 5 5 5 5 5 F F F
8312024-02-12T01:34:31.532Z 20 400-419 6 2 6 6 2 6 F F F <---
8322024-02-12T01:34:31.532Z 21 420-439 1 1 1 1 1 1 F F F
8332024-02-12T01:34:31.532Z 22 440-459 2 2 2 2 2 2 F F F
8342024-02-12T01:34:31.547Z 23 460-479 6 1 6 6 1 6 F F F <---
8352024-02-12T01:34:31.547Z 24 480-499 6 5 6 6 5 6 F F F <---
8362024-02-12T01:34:31.547Z 25 500-519 5 5 5 5 5 5 F F F
8372024-02-12T01:34:31.547Z 26 520-539 5 5 5 5 5 5 F F F
8382024-02-12T01:34:31.547Z 27 540-559 2 2 2 2 2 2 F F F
8392024-02-12T01:34:31.547Z 28 560-579 2 2 2 2 2 2 F F F
8402024-02-12T01:34:31.547Z 29 580-599 6 2 6 6 2 6 F F F <---
8412024-02-12T01:34:31.547ZMax gen: 6, Max flush: 7
8422024-02-12T01:34:31.548ZError: Difference in extent metadata found!
8432024-02-12T01:34:31.548ZA Difference in extent metadata is expected here
8442024-02-12T01:34:31.548ZOn loop 3
8452024-02-12T01:34:31.548Z
8462024-02-12T01:34:31.548ZVerifying data now
8472024-02-12T01:34:37.390ZNo change after verify
8482024-02-12T01:34:37.393ZLoop: 3 Downstairs dump after verify (and repair):
8492024-02-12T01:34:37.405ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8502024-02-12T01:34:37.405Z 0 000-019 5 5 5 5 5 5 F F F
8512024-02-12T01:34:37.405Z 1 020-039 7 7 7 8 8 8 F F F
8522024-02-12T01:34:37.405Z 2 040-059 6 6 6 6 6 6 F F F
8532024-02-12T01:34:37.405Z 3 060-079 5 5 5 5 5 5 F F F
8542024-02-12T01:34:37.406Z 4 080-099 6 6 6 7 7 7 F F F
8552024-02-12T01:34:37.406Z 5 100-119 6 6 6 7 7 7 F F F
8562024-02-12T01:34:37.406Z 6 120-139 6 6 6 7 7 7 F F F
8572024-02-12T01:34:37.406Z 7 140-159 6 6 6 6 6 6 F F F
8582024-02-12T01:34:37.406Z 8 160-179 1 1 1 1 1 1 F F F
8592024-02-12T01:34:37.406Z 9 180-199 4 4 4 3 3 3 F F F
8602024-02-12T01:34:37.409Z 10 200-219 6 6 6 7 7 7 F F F
8612024-02-12T01:34:37.409Z 11 220-239 7 7 7 8 8 8 F F F
8622024-02-12T01:34:37.409Z 12 240-259 2 2 2 2 2 2 F F F
8632024-02-12T01:34:37.409Z 13 260-279 1 1 1 1 1 1 F F F
8642024-02-12T01:34:37.409Z 14 280-299 5 5 5 5 5 5 F F F
8652024-02-12T01:34:37.409Z 15 300-319 7 7 7 8 8 8 F F F
8662024-02-12T01:34:37.409Z 16 320-339 6 6 6 7 7 7 F F F
8672024-02-12T01:34:37.409Z 17 340-359 2 2 2 2 2 2 F F F
8682024-02-12T01:34:37.409Z 18 360-379 4 4 4 4 4 4 F F F
8692024-02-12T01:34:37.409Z 19 380-399 5 5 5 5 5 5 F F F
8702024-02-12T01:34:37.409Z 20 400-419 6 6 6 6 6 6 F F F
8712024-02-12T01:34:37.409Z 21 420-439 1 1 1 1 1 1 F F F
8722024-02-12T01:34:37.409Z 22 440-459 2 2 2 2 2 2 F F F
8732024-02-12T01:34:37.409Z 23 460-479 6 6 6 6 6 6 F F F
8742024-02-12T01:34:37.409Z 24 480-499 6 6 6 6 6 6 F F F
8752024-02-12T01:34:37.409Z 25 500-519 5 5 5 5 5 5 F F F
8762024-02-12T01:34:37.409Z 26 520-539 5 5 5 5 5 5 F F F
8772024-02-12T01:34:37.409Z 27 540-559 2 2 2 2 2 2 F F F
8782024-02-12T01:34:37.410Z 28 560-579 2 2 2 2 2 2 F F F
8792024-02-12T01:34:37.410Z 29 580-599 6 6 6 6 6 6 F F F
8802024-02-12T01:34:37.410ZMax gen: 7, Max flush: 8
8812024-02-12T01:34:37.410Z
8822024-02-12T01:34:37.410ZBegin loop 4
8832024-02-12T01:34:37.410ZDownstairs to restart: 2
8842024-02-12T01:34:37.431Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.346589358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
8852024-02-12T01:34:37.431Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.346888354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
8862024-02-12T01:34:37.436Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
8872024-02-12T01:34:37.436Z,"time":"2024-02-12T01:34:36.346930742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
8882024-02-12T01:34:37.436Z{"msg":"Crucible 03fec439-a691-4994-afd3-8636771c7107 has session id: 989d1c56-6e60-43d4-ae12-3788194c35a1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.34697317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8892024-02-12T01:34:37.436Z{"msg":"Upstairs opts: Upstairs UUID: 03fec439-a691-4994-afd3-8636771c7107, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347056476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8902024-02-12T01:34:37.436Z{"msg":"Crucible stats registered with UUID: 03fec439-a691-4994-afd3-8636771c7107","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347095204Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8912024-02-12T01:34:37.436Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347119953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
8922024-02-12T01:34:37.436Z{"msg":"03fec439-a691-4994-afd3-8636771c7107 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347135092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8932024-02-12T01:34:37.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347149131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8942024-02-12T01:34:37.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347166841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8952024-02-12T01:34:37.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347202959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8962024-02-12T01:34:37.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347228158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8972024-02-12T01:34:37.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347267466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8982024-02-12T01:34:37.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347293355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
8992024-02-12T01:34:37.437Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347330763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9002024-02-12T01:34:37.437Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347365711Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9012024-02-12T01:34:37.437Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.3473934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9022024-02-12T01:34:37.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347531663Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9032024-02-12T01:34:37.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.34758719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9042024-02-12T01:34:37.437Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.347616029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9052024-02-12T01:34:37.437Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.348296007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9062024-02-12T01:34:37.437Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.348333265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9072024-02-12T01:34:37.437Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.348349734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9082024-02-12T01:34:37.437Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.348388932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9092024-02-12T01:34:37.437Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.348873519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9102024-02-12T01:34:37.437Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349044451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9112024-02-12T01:34:37.437Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349180234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9122024-02-12T01:34:37.437Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349227452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9132024-02-12T01:34:37.437Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349315318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9142024-02-12T01:34:37.437Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349360826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9152024-02-12T01:34:37.438Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349394654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9162024-02-12T01:34:37.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349415703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9172024-02-12T01:34:37.438Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349449432Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9182024-02-12T01:34:37.438Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.34948564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9192024-02-12T01:34:37.438Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.349746238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9202024-02-12T01:34:37.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350149148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9212024-02-12T01:34:37.438Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350184566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9222024-02-12T01:34:37.438Z{"msg":"[0]R flush_numbers[0..12]: [5, 8, 6, 5, 7, 7, 7, 6, 1, 3, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350200886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9232024-02-12T01:34:37.438Z{"msg":"[0]R generation[0..12]: [5, 7, 6, 5, 6, 6, 6, 6, 1, 4, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350214925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9242024-02-12T01:34:37.438Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350238394Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9252024-02-12T01:34:37.438Z{"msg":"[1]R flush_numbers[0..12]: [5, 8, 6, 5, 7, 7, 7, 6, 1, 3, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350252603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9262024-02-12T01:34:37.438Z{"msg":"[1]R generation[0..12]: [5, 7, 6, 5, 6, 6, 6, 6, 1, 4, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350266383Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9272024-02-12T01:34:37.438Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350286862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9282024-02-12T01:34:37.438Z{"msg":"[2]R flush_numbers[0..12]: [5, 8, 6, 5, 7, 7, 7, 6, 1, 3, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350308281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9292024-02-12T01:34:37.438Z{"msg":"[2]R generation[0..12]: [5, 7, 6, 5, 6, 6, 6, 6, 1, 4, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.35032299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9302024-02-12T01:34:37.438Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350365838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9312024-02-12T01:34:37.438Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350379617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9322024-02-12T01:34:37.438Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
9332024-02-12T01:34:37.438Z,"time":"2024-02-12T01:34:36.350394196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9342024-02-12T01:34:37.438Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350411456Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9352024-02-12T01:34:37.438Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350431705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9362024-02-12T01:34:37.438Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350445604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9372024-02-12T01:34:37.438Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350457993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9382024-02-12T01:34:37.438Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350482952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9392024-02-12T01:34:37.439Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350496412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9402024-02-12T01:34:37.439Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350516381Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9412024-02-12T01:34:37.439Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.3505291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9422024-02-12T01:34:37.439Z{"msg":"03fec439-a691-4994-afd3-8636771c7107 is now active with session: 989d1c56-6e60-43d4-ae12-3788194c35a1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350543279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9432024-02-12T01:34:37.439Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350555799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167,"session_id":"989d1c56-6e60-43d4-ae12-3788194c35a1"}
9442024-02-12T01:34:37.439Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:36.350568158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
9452024-02-12T01:34:37.439ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9462024-02-12T01:34:37.439ZLoading write count information from file "/tmp/test_repair_verify.data"
9472024-02-12T01:34:37.439ZRun Repair workload
9482024-02-12T01:34:37.439Z01/30 Write block 275 len 36864 data: 2 2 2 2 2 2 2 2 2
9492024-02-12T01:34:37.439Z02/30 Write block 575 len 16384 data: 3 3 3 3
9502024-02-12T01:34:37.442Z03/30 Read block 364 len 20480
9512024-02-12T01:34:37.442Z04/30 Flush
9522024-02-12T01:34:37.442Z05/30 Write block 114 len 32768 data: 2 2 2 3 3 3 3 3
9532024-02-12T01:34:37.442Z06/30 Read block 82 len 28672
9542024-02-12T01:34:37.442Z07/30 Write block 28 len 28672 data: 3 3 3 3 3 3 3
9552024-02-12T01:34:37.442Z08/30 Write block 413 len 24576 data: 2 2 3 2 2 2
9562024-02-12T01:34:37.445Z09/30 Write block 184 len 32768 data: 2 2 2 2 3 3 3 3
9572024-02-12T01:34:37.445Z10/30 Flush
9582024-02-12T01:34:37.470Z11/30 Flush
9592024-02-12T01:34:37.482Z12/30 Write block 175 len 20480 data: 2 2 2 2 2
9602024-02-12T01:34:37.482Z13/30 Read block 416 len 36864
9612024-02-12T01:34:37.482Z14/30 Write block 27 len 12288 data: 3 4 4
9622024-02-12T01:34:37.483Z15/30 Write block 550 len 4096 data: 2
9632024-02-12T01:34:37.483Z16/30 Read block 355 len 16384
9642024-02-12T01:34:37.483Z17/30 Flush
9652024-02-12T01:34:37.483Z18/30 Write block 375 len 16384 data: 2 2 2 2
9662024-02-12T01:34:37.483Z19/30 Read block 482 len 12288
9672024-02-12T01:34:37.483Z20/30 Read block 259 len 32768
9682024-02-12T01:34:37.483Z21/30 Write block 449 len 28672 data: 2 2 2 2 2 2 2
9692024-02-12T01:34:37.483Z22/30 Read block 212 len 12288
9702024-02-12T01:34:37.922Z23/30 Write block 399 len 24576 data: 2 2 2 3 2 2
9712024-02-12T01:34:37.922Z24/30 Read block 423 len 8192
9722024-02-12T01:34:37.922Z25/30 Read block 333 len 8192
9732024-02-12T01:34:37.922Z26/30 Write block 178 len 16384 data: 3 3 2 2
9742024-02-12T01:34:37.922Z27/30 Read block 183 len 32768
9752024-02-12T01:34:37.922Z28/30 Write block 532 len 32768 data: 2 2 2 2 2 3 3 3
9762024-02-12T01:34:37.922Z29/30 Read block 389 len 32768
9772024-02-12T01:34:37.922Z30/30 Read block 561 len 20480
9782024-02-12T01:34:37.922Z----------------------------------------------------------------
9792024-02-12T01:34:37.923Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9802024-02-12T01:34:37.923ZStates: Active Active Active
9812024-02-12T01:34:37.923ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9822024-02-12T01:34:37.923Z 1 Acked 1000 Write 9 Done Done Done false
9832024-02-12T01:34:37.923Z 2 Acked 1001 Write 4 Done Done Done false
9842024-02-12T01:34:37.923Z 3 Acked 1002 Read 5 Done Done Sent false
9852024-02-12T01:34:37.923Z 4 Acked 1003 Flush 0 Done Done Sent false
9862024-02-12T01:34:37.923Z 5 Acked 1004 Write 8 Done Done Sent false
9872024-02-12T01:34:37.923Z 6 Acked 1005 Read 7 Done Done Sent false
9882024-02-12T01:34:37.923Z 7 Acked 1006 Write 7 Done Done Sent false
9892024-02-12T01:34:37.923Z 8 Acked 1007 Write 6 Done Done Sent false
9902024-02-12T01:34:37.923Z 9 Acked 1008 Write 8 Done Done Sent false
9912024-02-12T01:34:37.923Z 10 Acked 1009 Flush 0 Done Done Sent false
9922024-02-12T01:34:37.923Z 11 Acked 1010 Flush 0 Done Done Sent false
9932024-02-12T01:34:37.923Z 12 Acked 1011 Write 5 Done Done Sent false
9942024-02-12T01:34:37.924Z 13 Acked 1012 Read 9 Done Done Sent false
9952024-02-12T01:34:37.924Z 14 Acked 1013 Write 3 Done Done Sent false
9962024-02-12T01:34:37.924Z 15 Acked 1014 Write 1 Done Done Sent false
9972024-02-12T01:34:37.924Z 16 Acked 1015 Read 4 Done Done Sent false
9982024-02-12T01:34:37.924Z 17 Acked 1016 Flush 0 Done Done Sent false
9992024-02-12T01:34:37.924Z 18 Acked 1017 Write 4 Done Done Sent false
10002024-02-12T01:34:37.924Z 19 Acked 1018 Read 3 Done Done Sent false
10012024-02-12T01:34:37.924Z 20 Acked 1019 Read 8 Done Done Sent false
10022024-02-12T01:34:37.924Z 21 Acked 1020 Write 7 Done Done Sent false
10032024-02-12T01:34:37.924Z 22 Acked 1021 Read 3 Done Done Sent false
10042024-02-12T01:34:37.924Z 23 Acked 1022 Write 6 Done Done Sent false
10052024-02-12T01:34:37.924Z 24 Acked 1023 Read 2 Done Done Sent false
10062024-02-12T01:34:37.924Z 25 Acked 1024 Read 2 Done Done Sent false
10072024-02-12T01:34:37.924Z 26 Acked 1025 Write 4 Done Done Sent false
10082024-02-12T01:34:37.924Z 27 Acked 1026 Read 8 Done Done Sent false
10092024-02-12T01:34:37.924Z 28 Acked 1027 Write 8 Done Done Sent false
10102024-02-12T01:34:37.924Z 29 Acked 1028 Read 8 Done Done Sent false
10112024-02-12T01:34:37.925Z 30 Acked 1029 Read 5 Done Done Sent false
10122024-02-12T01:34:37.925Z STATES DS:0 DS:1 DS:2 TOTAL
10132024-02-12T01:34:37.925Z New 0 0 0 0
10142024-02-12T01:34:37.925Z Sent 0 0 28 28
10152024-02-12T01:34:37.925Z Done 30 30 2 62
10162024-02-12T01:34:37.925Z Skipped 0 0 0 0
10172024-02-12T01:34:37.925Z Error 0 0 0 0
10182024-02-12T01:34:37.925ZLast Flush: 1016 1016 0
10192024-02-12T01:34:37.925ZDownstairs last five completed:
10202024-02-12T01:34:37.925ZUpstairs last five completed: 30 29 28 27 26
10212024-02-12T01:34:37.925Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:36.403324411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1167}
10222024-02-12T01:34:37.925ZWrote out file "/tmp/test_repair_verify.data"
10232024-02-12T01:34:37.925Z
10242024-02-12T01:34:37.947ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10252024-02-12T01:34:37.947ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10262024-02-12T01:34:37.947Z 0 000-019 5 5 5 5 5 5 F F F
10272024-02-12T01:34:37.947Z 1 020-039 8 8 7 12 12 8 F F F <---
10282024-02-12T01:34:37.948Z 2 040-059 6 6 6 6 6 6 F F F
10292024-02-12T01:34:37.948Z 3 060-079 5 5 5 5 5 5 F F F
10302024-02-12T01:34:37.948Z 4 080-099 6 6 6 7 7 7 F F F
10312024-02-12T01:34:37.948Z 5 100-119 8 8 6 10 10 7 F F F <---
10322024-02-12T01:34:37.948Z 6 120-139 8 8 6 10 10 7 F F F <---
10332024-02-12T01:34:37.948Z 7 140-159 6 6 6 6 6 6 F F F
10342024-02-12T01:34:37.948Z 8 160-179 8 8 1 12 12 1 T T F <---
10352024-02-12T01:34:37.948Z 9 180-199 8 8 4 10 10 3 T T F <---
10362024-02-12T01:34:37.948Z 10 200-219 6 6 6 7 7 7 F F F
10372024-02-12T01:34:37.948Z 11 220-239 7 7 7 8 8 8 F F F
10382024-02-12T01:34:37.948Z 12 240-259 2 2 2 2 2 2 F F F
10392024-02-12T01:34:37.948Z 13 260-279 8 8 1 9 9 1 F F T <---
10402024-02-12T01:34:37.948Z 14 280-299 8 8 5 9 9 5 F F T <---
10412024-02-12T01:34:37.948Z 15 300-319 7 7 7 8 8 8 F F F
10422024-02-12T01:34:37.948Z 16 320-339 6 6 6 7 7 7 F F F
10432024-02-12T01:34:37.949Z 17 340-359 2 2 2 2 2 2 F F F
10442024-02-12T01:34:37.949Z 18 360-379 4 4 4 4 4 4 T T F <---
10452024-02-12T01:34:37.949Z 19 380-399 5 5 5 5 5 5 T T F <---
10462024-02-12T01:34:37.949Z 20 400-419 8 8 6 10 10 6 T T F <---
10472024-02-12T01:34:37.949Z 21 420-439 1 1 1 1 1 1 F F F
10482024-02-12T01:34:37.949Z 22 440-459 2 2 2 2 2 2 T T F <---
10492024-02-12T01:34:37.949Z 23 460-479 6 6 6 6 6 6 F F F
10502024-02-12T01:34:37.949Z 24 480-499 6 6 6 6 6 6 F F F
10512024-02-12T01:34:37.949Z 25 500-519 5 5 5 5 5 5 F F F
10522024-02-12T01:34:37.949Z 26 520-539 5 5 5 5 5 5 T T F <---
10532024-02-12T01:34:37.949Z 27 540-559 8 8 2 12 12 2 F F F <---
10542024-02-12T01:34:37.949Z 28 560-579 8 8 2 9 9 2 F F T <---
10552024-02-12T01:34:37.949Z 29 580-599 6 6 6 6 6 6 F F F
10562024-02-12T01:34:37.949ZMax gen: 8, Max flush: 12
10572024-02-12T01:34:37.950ZError: Difference in extent metadata found!
10582024-02-12T01:34:37.950ZA Difference in extent metadata is expected here
10592024-02-12T01:34:37.950ZOn loop 4
10602024-02-12T01:34:37.950Z
10612024-02-12T01:34:37.950ZVerifying data now
10622024-02-12T01:34:43.736ZNo change after verify
10632024-02-12T01:34:43.736ZLoop: 4 Downstairs dump after verify (and repair):
10642024-02-12T01:34:43.750ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10652024-02-12T01:34:43.751Z 0 000-019 5 5 5 5 5 5 F F F
10662024-02-12T01:34:43.751Z 1 020-039 8 8 8 12 12 12 F F F
10672024-02-12T01:34:43.751Z 2 040-059 6 6 6 6 6 6 F F F
10682024-02-12T01:34:43.751Z 3 060-079 5 5 5 5 5 5 F F F
10692024-02-12T01:34:43.751Z 4 080-099 6 6 6 7 7 7 F F F
10702024-02-12T01:34:43.751Z 5 100-119 8 8 8 10 10 10 F F F
10712024-02-12T01:34:43.751Z 6 120-139 8 8 8 10 10 10 F F F
10722024-02-12T01:34:43.751Z 7 140-159 6 6 6 6 6 6 F F F
10732024-02-12T01:34:43.751Z 8 160-179 9 9 9 13 13 13 F F F
10742024-02-12T01:34:43.751Z 9 180-199 9 9 9 13 13 13 F F F
10752024-02-12T01:34:43.754Z 10 200-219 6 6 6 7 7 7 F F F
10762024-02-12T01:34:43.754Z 11 220-239 7 7 7 8 8 8 F F F
10772024-02-12T01:34:43.754Z 12 240-259 2 2 2 2 2 2 F F F
10782024-02-12T01:34:43.754Z 13 260-279 8 8 8 9 9 9 F F F
10792024-02-12T01:34:43.754Z 14 280-299 8 8 8 9 9 9 F F F
10802024-02-12T01:34:43.754Z 15 300-319 7 7 7 8 8 8 F F F
10812024-02-12T01:34:43.754Z 16 320-339 6 6 6 7 7 7 F F F
10822024-02-12T01:34:43.755Z 17 340-359 2 2 2 2 2 2 F F F
10832024-02-12T01:34:43.755Z 18 360-379 9 9 9 13 13 13 F F F
10842024-02-12T01:34:43.755Z 19 380-399 9 9 9 13 13 13 F F F
10852024-02-12T01:34:43.755Z 20 400-419 9 9 9 13 13 13 F F F
10862024-02-12T01:34:43.755Z 21 420-439 1 1 1 1 1 1 F F F
10872024-02-12T01:34:43.755Z 22 440-459 9 9 9 13 13 13 F F F
10882024-02-12T01:34:43.755Z 23 460-479 6 6 6 6 6 6 F F F
10892024-02-12T01:34:43.755Z 24 480-499 6 6 6 6 6 6 F F F
10902024-02-12T01:34:43.755Z 25 500-519 5 5 5 5 5 5 F F F
10912024-02-12T01:34:43.755Z 26 520-539 9 9 9 13 13 13 F F F
10922024-02-12T01:34:43.755Z 27 540-559 8 8 8 12 12 12 F F F
10932024-02-12T01:34:43.755Z 28 560-579 8 8 8 9 9 9 F F F
10942024-02-12T01:34:43.755Z 29 580-599 6 6 6 6 6 6 F F F
10952024-02-12T01:34:43.755ZMax gen: 9, Max flush: 13
10962024-02-12T01:34:43.758Z
10972024-02-12T01:34:43.758ZBegin loop 5
10982024-02-12T01:34:43.758ZDownstairs to restart: 0
10992024-02-12T01:34:43.773Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69041161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
11002024-02-12T01:34:43.773Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"Crucible runtime is spawned
11012024-02-12T01:34:43.780Zx86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690700997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
11022024-02-12T01:34:43.780Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690758244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
11032024-02-12T01:34:43.780Z{"msg":"Crucible eadebbb6-39e9-4f3a-8a2b-0735bcb0f49d has session id: 56630cde-2cc3-492c-90fc-4347fe0c7fcb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690778213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11042024-02-12T01:34:43.780Z{"msg":"Upstairs opts: Upstairs UUID: eadebbb6-39e9-4f3a-8a2b-0735bcb0f49d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690824931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11052024-02-12T01:34:43.780Z{"msg":"Crucible stats registered with UUID: eadebbb6-39e9-4f3a-8a2b-0735bcb0f49d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69084106Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11062024-02-12T01:34:43.780Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690866978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
11072024-02-12T01:34:43.780Z{"msg":"eadebbb6-39e9-4f3a-8a2b-0735bcb0f49d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690878958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11082024-02-12T01:34:43.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690891697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11092024-02-12T01:34:43.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690906167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11102024-02-12T01:34:43.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690927956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11112024-02-12T01:34:43.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690941975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11122024-02-12T01:34:43.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690955774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11132024-02-12T01:34:43.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.690988893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11142024-02-12T01:34:43.780Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.691008302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11152024-02-12T01:34:43.780Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69104043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11162024-02-12T01:34:43.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69105502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11172024-02-12T01:34:43.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.691214092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11182024-02-12T01:34:43.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69126182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11192024-02-12T01:34:43.781Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:42.691428962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11202024-02-12T01:34:43.781Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:42.691487669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11212024-02-12T01:34:43.781Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:42.691564225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11222024-02-12T01:34:43.781Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.691601503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11232024-02-12T01:34:43.781Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.691627052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11242024-02-12T01:34:43.781Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.691751956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11252024-02-12T01:34:43.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692114889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11262024-02-12T01:34:43.781Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692171896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11272024-02-12T01:34:43.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692243573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11282024-02-12T01:34:43.781Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692289261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11292024-02-12T01:34:43.781Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692713361Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11302024-02-12T01:34:43.781Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.692854094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11312024-02-12T01:34:43.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.693228506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11322024-02-12T01:34:43.781Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.693273064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11332024-02-12T01:34:43.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.693300082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11342024-02-12T01:34:43.781Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.693328071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11352024-02-12T01:34:43.781Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.69335291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11362024-02-12T01:34:43.781Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:42.693380109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11372024-02-12T01:34:44.786Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.693299023Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11382024-02-12T01:34:44.786Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.693427207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11392024-02-12T01:34:44.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.694266857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11402024-02-12T01:34:44.807Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.694302245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11412024-02-12T01:34:44.807Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.694748994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11422024-02-12T01:34:44.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.69526669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11432024-02-12T01:34:44.807Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695310637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11442024-02-12T01:34:44.807Z{"msg":"[0]R flush_numbers[0..12]: [5, 12, 6, 5, 7, 10, 10, 6, 13, 13, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695335976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11452024-02-12T01:34:44.807Z{"msg":"[0]R generation[0..12]: [5, 8, 6, 5, 6, 8, 8, 6, 9, 9, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695374954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11462024-02-12T01:34:44.807Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695416082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11472024-02-12T01:34:44.807Z{"msg":"[1]R flush_numbers[0..12]: [5, 12, 6, 5, 7, 10, 10, 6, 13, 13, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695452261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11482024-02-12T01:34:44.807Z{"msg":"[1]R generation[0..12]: [5, 8, 6, 5, 6, 8, 8, 6, 9, 9, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.69547448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11492024-02-12T01:34:44.807ZWait for a query_work_queue command to finish before sending IO
11502024-02-12T01:34:44.807Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695517607Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11512024-02-12T01:34:44.807Z{"msg":"[2]R flush_numbers[0..12]: [5, 12, 6, 5, 7, 10, 10, 6, 13, 13, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695538257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11522024-02-12T01:34:44.808Z{"msg":"[2]R generation[0..12]: [5, 8, 6, 5, 6, 8, 8, 6, 9, 9, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695555426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11532024-02-12T01:34:44.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695595144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11542024-02-12T01:34:44.808Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695614883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11552024-02-12T01:34:44.808Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695652781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11562024-02-12T01:34:44.808Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.69567373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11572024-02-12T01:34:44.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695710718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11582024-02-12T01:34:44.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695731617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11592024-02-12T01:34:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695766775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11602024-02-12T01:34:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695782465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11612024-02-12T01:34:44.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695812893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11622024-02-12T01:34:44.808Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695831272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11632024-02-12T01:34:44.808Z56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11642024-02-12T01:34:44.808Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695861281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11652024-02-12T01:34:44.809Z{"msg":"eadebbb6-39e9-4f3a-8a2b-0735bcb0f49d is now active with session: 56630cde-2cc3-492c-90fc-4347fe0c7fcb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.69588301Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11662024-02-12T01:34:44.809Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695910789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175,"session_id":"56630cde-2cc3-492c-90fc-4347fe0c7fcb"}
11672024-02-12T01:34:44.809Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:43.695927788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
11682024-02-12T01:34:44.809ZLoading write count information from file "/tmp/test_repair_verify.data"
11692024-02-12T01:34:44.809ZRun Repair workload
11702024-02-12T01:34:44.809Z01/30 Write block 420 len 32768 data: 2 2 2 2 2 2 2 2
11712024-02-12T01:34:44.809Z02/30 Write block 488 len 36864 data: 2 2 2 2 2 2 2 3 3
11722024-02-12T01:34:44.809Z03/30 Read block 371 len 28672
11732024-02-12T01:34:44.809Z04/30 Write block 318 len 16384 data: 2 2 2 2
11742024-02-12T01:34:44.809Z05/30 Write block 461 len 36864 data: 2 2 2 2 2 2 2 2 2
11752024-02-12T01:34:44.809Z06/30 Write block 559 len 32768 data: 2 2 2 2 2 2 2 2
11762024-02-12T01:34:44.809Z07/30 Write block 188 len 12288 data: 4 4 4
11772024-02-12T01:34:44.809Z08/30 Write block 350 len 12288 data: 3 3 3
11782024-02-12T01:34:44.810Z09/30 Write block 595 len 8192 data: 3 3
11792024-02-12T01:34:44.810Z10/30 Write block 32 len 12288 data: 4 4 4
11802024-02-12T01:34:44.810Z11/30 Write block 577 len 32768 data: 4 4 3 3 2 2 2 2
11812024-02-12T01:34:44.810Z12/30 Write block 43 len 8192 data: 3 4
11822024-02-12T01:34:44.810Z13/30 Read block 347 len 28672
11832024-02-12T01:34:44.810Z14/30 Read block 167 len 16384
11842024-02-12T01:34:44.810Z15/30 Read block 465 len 8192
11852024-02-12T01:34:44.810Z16/30 Read block 28 len 4096
11862024-02-12T01:34:44.810Z17/30 Write block 240 len 16384 data: 3 3 3 3
11872024-02-12T01:34:44.810Z18/30 Write block 87 len 24576 data: 2 2 2 2 2 2
11882024-02-12T01:34:44.810Z19/30 Write block 385 len 36864 data: 2 2 2 2 2 4 4 4 3
11892024-02-12T01:34:44.811Z20/30 Read block 260 len 16384
11902024-02-12T01:34:44.811Z21/30 Flush
11912024-02-12T01:34:44.823Z22/30 Write block 110 len 40960 data: 2 2 2 2 3 3 3 4 4 4
11922024-02-12T01:34:44.826Z23/30 Read block 531 len 24576
11932024-02-12T01:34:44.826Z24/30 Read block 96 len 20480
11942024-02-12T01:34:44.826Z25/30 Write block 512 len 20480 data: 2 2 2 2 2
11952024-02-12T01:34:44.826Z26/30 Read block 142 len 4096
11962024-02-12T01:34:44.829Z27/30 Write block 165 len 24576 data: 2 2 2 2 2 2
11972024-02-12T01:34:44.830Z28/30 Write block 243 len 36864 data: 4 2 2 2 2 2 2 2 2
11982024-02-12T01:34:44.830Z29/30 Read block 407 len 4096
11992024-02-12T01:34:44.830Z30/30 Write block 355 len 24576 data: 3 2 2 2 2 2
12002024-02-12T01:34:44.833Z----------------------------------------------------------------
12012024-02-12T01:34:44.833Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
12022024-02-12T01:34:44.833ZStates: Active Active Active
12032024-02-12T01:34:44.834ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
12042024-02-12T01:34:44.834Z 1 Acked 1000 Write 8 Sent Done Done false
12052024-02-12T01:34:44.834Z 2 Acked 1001 Write 9 Sent Done Done false
12062024-02-12T01:34:44.834Z 3 Acked 1002 Read 7 Sent Done Done false
12072024-02-12T01:34:44.834Z 4 Acked 1003 Write 4 Sent Done Done false
12082024-02-12T01:34:44.834Z 5 Acked 1004 Write 9 Sent Done Done false
12092024-02-12T01:34:44.834Z 6 Acked 1005 Write 8 Sent Done Done false
12102024-02-12T01:34:44.834Z 7 Acked 1006 Write 3 Sent Done Done false
12112024-02-12T01:34:44.834Z 8 Acked 1007 Write 3 Sent Done Done false
12122024-02-12T01:34:44.834Z 9 Acked 1008 Write 2 Sent Done Done false
12132024-02-12T01:34:44.834Z 10 Acked 1009 Write 3 Sent Done Done false
12142024-02-12T01:34:44.834Z 11 Acked 1010 Write 8 Sent Done Done false
12152024-02-12T01:34:44.834Z 12 Acked 1011 Write 2 Sent Done Done false
12162024-02-12T01:34:44.835Z 13 Acked 1012 Read 7 Sent Done Done false
12172024-02-12T01:34:44.835Z 14 Acked 1013 Read 4 Sent Done Done false
12182024-02-12T01:34:44.835Z 15 Acked 1014 Read 2 Sent Done Done false
12192024-02-12T01:34:44.835Z 16 Acked 1015 Read 1 Sent Done Done false
12202024-02-12T01:34:44.835Z 17 Acked 1016 Write 4 Sent Done Done false
12212024-02-12T01:34:44.835Z 18 Acked 1017 Write 6 Sent Done Done false
12222024-02-12T01:34:44.835Z 19 Acked 1018 Write 9 Sent Done Done false
12232024-02-12T01:34:44.835Z 20 Acked 1019 Read 4 Sent Done Done false
12242024-02-12T01:34:44.835Z 21 Acked 1020 Flush 0 Sent Done Done false
12252024-02-12T01:34:44.835Z 22 Acked 1021 Write 10 Sent Done Done false
12262024-02-12T01:34:44.835Z 23 Acked 1022 Read 6 Sent Done Done false
12272024-02-12T01:34:44.835Z 24 Acked 1023 Read 5 Sent Done Done false
12282024-02-12T01:34:44.835Z 25 Acked 1024 Write 5 Sent Done Done false
12292024-02-12T01:34:44.835Z 26 Acked 1025 Read 1 Sent Done Done false
12302024-02-12T01:34:44.836Z 27 Acked 1026 Write 6 Sent Done Done false
12312024-02-12T01:34:44.836Z 28 Acked 1027 Write 9 Sent Done Done false
12322024-02-12T01:34:44.836Z 29 Acked 1028 Read 1 Sent Done Done false
12332024-02-12T01:34:44.836Z 30 Acked 1029 Write 6 Sent Sent Sent false
12342024-02-12T01:34:44.836Z STATES DS:0 DS:1 DS:2 TOTAL
12352024-02-12T01:34:44.836Z New 0 0 0 0
12362024-02-12T01:34:44.836Z Sent 30 1 1 32
12372024-02-12T01:34:44.836Z Done 0 29 29 58
12382024-02-12T01:34:44.836Z Skipped 0 0 0 0
12392024-02-12T01:34:44.836Z Error 0 0 0 0
12402024-02-12T01:34:44.836ZLast Flush: 0 1020 1020
12412024-02-12T01:34:44.836ZDownstairs last five completed:
12422024-02-12T01:34:44.836ZUpstairs last five completed: 30 29 28 27 26
12432024-02-12T01:34:44.836Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:43.748375066Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1175}
12442024-02-12T01:34:44.837ZWrote out file "/tmp/test_repair_verify.data"
12452024-02-12T01:34:44.843Z
12462024-02-12T01:34:44.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12472024-02-12T01:34:44.856ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12482024-02-12T01:34:44.856Z 0 000-019 5 5 5 5 5 5 F F F
12492024-02-12T01:34:44.856Z 1 020-039 8 10 10 12 14 14 F F F <---
12502024-02-12T01:34:44.856Z 2 040-059 6 10 10 6 14 14 F F F <---
12512024-02-12T01:34:44.856Z 3 060-079 5 5 5 5 5 5 F F F
12522024-02-12T01:34:44.856Z 4 080-099 6 10 10 7 14 14 F F F <---
12532024-02-12T01:34:44.856Z 5 100-119 8 8 8 10 10 10 F T T <---
12542024-02-12T01:34:44.856Z 6 120-139 8 8 8 10 10 10 F F F
12552024-02-12T01:34:44.860Z 7 140-159 6 6 6 6 6 6 F F F
12562024-02-12T01:34:44.860Z 8 160-179 9 9 9 13 13 13 F T T <---
12572024-02-12T01:34:44.860Z 9 180-199 9 10 10 13 14 14 F F F <---
12582024-02-12T01:34:44.860Z 10 200-219 6 6 6 7 7 7 F F F
12592024-02-12T01:34:44.860Z 11 220-239 7 7 7 8 8 8 F F F
12602024-02-12T01:34:44.860Z 12 240-259 2 10 10 2 14 14 F T T <---
12612024-02-12T01:34:44.860Z 13 260-279 8 8 8 9 9 9 F F F
12622024-02-12T01:34:44.860Z 14 280-299 8 8 8 9 9 9 F F F
12632024-02-12T01:34:44.860Z 15 300-319 7 10 10 8 14 14 F F F <---
12642024-02-12T01:34:44.860Z 16 320-339 6 10 10 7 14 14 F F F <---
12652024-02-12T01:34:44.860Z 17 340-359 2 10 10 2 14 14 F T T <---
12662024-02-12T01:34:44.860Z 18 360-379 9 9 9 13 13 13 F T T <---
12672024-02-12T01:34:44.860Z 19 380-399 9 10 10 13 14 14 F F F <---
12682024-02-12T01:34:44.861Z 20 400-419 9 9 9 13 13 13 F F F
12692024-02-12T01:34:44.861Z 21 420-439 1 10 10 1 14 14 F F F <---
12702024-02-12T01:34:44.861Z 22 440-459 9 9 9 13 13 13 F F F
12712024-02-12T01:34:44.861Z 23 460-479 6 10 10 6 14 14 F F F <---
12722024-02-12T01:34:44.861Z 24 480-499 6 10 10 6 14 14 F F F <---
12732024-02-12T01:34:44.861Z 25 500-519 5 5 5 5 5 5 F T T <---
12742024-02-12T01:34:44.861Z 26 520-539 9 9 9 13 13 13 F F F
12752024-02-12T01:34:44.861Z 27 540-559 8 10 10 12 14 14 F F F <---
12762024-02-12T01:34:44.861Z 28 560-579 8 10 10 9 14 14 F F F <---
12772024-02-12T01:34:44.861Z 29 580-599 6 10 10 6 14 14 F F F <---
12782024-02-12T01:34:44.861ZMax gen: 10, Max flush: 14
12792024-02-12T01:34:44.861ZError: Difference in extent metadata found!
12802024-02-12T01:34:44.861ZA Difference in extent metadata is expected here
12812024-02-12T01:34:44.862ZOn loop 5
12822024-02-12T01:34:44.862Z
12832024-02-12T01:34:44.865ZVerifying data now
12842024-02-12T01:34:50.529ZNo change after verify
12852024-02-12T01:34:50.529ZLoop: 5 Downstairs dump after verify (and repair):
12862024-02-12T01:34:50.555ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12872024-02-12T01:34:50.555Z 0 000-019 5 5 5 5 5 5 F F F
12882024-02-12T01:34:50.555Z 1 020-039 10 10 10 14 14 14 F F F
12892024-02-12T01:34:50.555Z 2 040-059 10 10 10 14 14 14 F F F
12902024-02-12T01:34:50.555Z 3 060-079 5 5 5 5 5 5 F F F
12912024-02-12T01:34:50.555Z 4 080-099 10 10 10 14 14 14 F F F
12922024-02-12T01:34:50.973Z 5 100-119 11 11 11 15 15 15 F F F
12932024-02-12T01:34:50.973Z 6 120-139 8 8 8 10 10 10 F F F
12942024-02-12T01:34:50.973Z 7 140-159 6 6 6 6 6 6 F F F
12952024-02-12T01:34:50.973Z 8 160-179 11 11 11 15 15 15 F F F
12962024-02-12T01:34:50.973Z 9 180-199 10 10 10 14 14 14 F F F
12972024-02-12T01:34:50.974Z 10 200-219 6 6 6 7 7 7 F F F
12982024-02-12T01:34:50.974Z 11 220-239 7 7 7 8 8 8 F F F
12992024-02-12T01:34:50.974Z 12 240-259 11 11 11 15 15 15 F F F
13002024-02-12T01:34:50.974Z 13 260-279 8 8 8 9 9 9 F F F
13012024-02-12T01:34:50.974Z 14 280-299 8 8 8 9 9 9 F F F
13022024-02-12T01:34:50.974Z 15 300-319 10 10 10 14 14 14 F F F
13032024-02-12T01:34:50.975Z 16 320-339 10 10 10 14 14 14 F F F
13042024-02-12T01:34:50.975Z 17 340-359 11 11 11 15 15 15 F F F
13052024-02-12T01:34:50.975Z 18 360-379 11 11 11 15 15 15 F F F
13062024-02-12T01:34:50.975Z 19 380-399 10 10 10 14 14 14 F F F
13072024-02-12T01:34:50.975Z 20 400-419 9 9 9 13 13 13 F F F
13082024-02-12T01:34:50.975Z 21 420-439 10 10 10 14 14 14 F F F
13092024-02-12T01:34:50.976Z 22 440-459 9 9 9 13 13 13 F F F
13102024-02-12T01:34:50.976Z 23 460-479 10 10 10 14 14 14 F F F
13112024-02-12T01:34:50.976Z 24 480-499 10 10 10 14 14 14 F F F
13122024-02-12T01:34:50.976Z 25 500-519 11 11 11 15 15 15 F F F
13132024-02-12T01:34:50.976Z 26 520-539 9 9 9 13 13 13 F F F
13142024-02-12T01:34:50.976Z 27 540-559 10 10 10 14 14 14 F F F
13152024-02-12T01:34:50.976Z 28 560-579 10 10 10 14 14 14 F F F
13162024-02-12T01:34:50.977Z 29 580-599 10 10 10 14 14 14 F F F
13172024-02-12T01:34:50.977ZMax gen: 11, Max flush: 15
13182024-02-12T01:34:50.977Z
13192024-02-12T01:34:50.977ZBegin loop 6
13202024-02-12T01:34:50.977ZDownstairs to restart: 0
13212024-02-12T01:34:50.992Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480395068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
13222024-02-12T01:34:50.993Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
13232024-02-12T01:34:50.993Z,"time":"2024-02-12T01:34:49.480693133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
13242024-02-12T01:34:50.993Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480740881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
13252024-02-12T01:34:50.993Z{"msg":"Crucible a1cccb6b-dd01-4a14-b485-82c4bc3b4100 has session id: 4ebcaa7a-59a4-4b44-9e87-a85defdde390","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48076838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13262024-02-12T01:34:50.993Z{"msg":"Upstairs opts: Upstairs UUID: a1cccb6b-dd01-4a14-b485-82c4bc3b4100, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480824477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13272024-02-12T01:34:50.993Z{"msg":"Crucible stats registered with UUID: a1cccb6b-dd01-4a14-b485-82c4bc3b4100","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480868435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13282024-02-12T01:34:50.993Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480905793Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
13292024-02-12T01:34:50.993Z{"msg":"a1cccb6b-dd01-4a14-b485-82c4bc3b4100 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480931612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13302024-02-12T01:34:50.993Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48096834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13312024-02-12T01:34:50.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.480999689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13322024-02-12T01:34:50.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481042547Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13332024-02-12T01:34:50.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481088824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13342024-02-12T01:34:50.994Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481121803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13352024-02-12T01:34:50.994Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48118764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13362024-02-12T01:34:50.994Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481243297Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13372024-02-12T01:34:50.995Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481287275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13382024-02-12T01:34:50.995Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481330693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13392024-02-12T01:34:50.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481465186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13402024-02-12T01:34:50.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481491595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13412024-02-12T01:34:50.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.481508704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13422024-02-12T01:34:50.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482065248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13432024-02-12T01:34:50.996Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482101366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13442024-02-12T01:34:50.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482117085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13452024-02-12T01:34:50.996Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482130975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13462024-02-12T01:34:50.996Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482709467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13472024-02-12T01:34:50.997Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.482876749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13482024-02-12T01:34:50.997Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483214113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13492024-02-12T01:34:50.997Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48327327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13502024-02-12T01:34:50.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483402284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13512024-02-12T01:34:50.997Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483435832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13522024-02-12T01:34:50.997Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483461101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13532024-02-12T01:34:50.997Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48349047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13542024-02-12T01:34:50.997Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483514619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13552024-02-12T01:34:50.997Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483550317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13562024-02-12T01:34:50.998Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.483812694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13572024-02-12T01:34:50.998Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484209406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13582024-02-12T01:34:50.998Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484247924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13592024-02-12T01:34:50.998Z{"msg":"[0]R flush_numbers[0..12]: [5, 14, 14, 5, 14, 15, 10, 6, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484264293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13602024-02-12T01:34:50.998Z{"msg":"[0]R generation[0..12]: [5, 10, 10, 5, 10, 11, 8, 6, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484286592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13612024-02-12T01:34:50.998Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484301971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13622024-02-12T01:34:50.999Z{"msg":"[1]R flush_numbers[0..12]: [5, 14, 14, 5, 14, 15, 10, 6, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48432232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13632024-02-12T01:34:50.999Z{"msg":"[1]R generation[0..12]: [5, 10, 10, 5, 10, 11, 8, 6, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.4843365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13642024-02-12T01:34:50.999Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484362758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13652024-02-12T01:34:50.999Z{"msg":"[2]R flush_numbers[0..12]: [5, 14, 14, 5, 14, 15, 10, 6, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484376667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13662024-02-12T01:34:51.000Z{"msg":"[2]R generation[0..12]: [5, 10, 10, 5, 10, 11, 8, 6, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484390377Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13672024-02-12T01:34:51.000ZWait for a query_work_queue command to finish before sending IO
13682024-02-12T01:34:51.000Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484415386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13692024-02-12T01:34:51.000Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484436775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13702024-02-12T01:34:51.001Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484463663Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13712024-02-12T01:34:51.001Z{"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484477873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13722024-02-12T01:34:51.001Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484491152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13732024-02-12T01:34:51.001Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484511761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13742024-02-12T01:34:51.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484524151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13752024-02-12T01:34:51.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.48453747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13762024-02-12T01:34:51.002Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484557099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13772024-02-12T01:34:51.002Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484570878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13782024-02-12T01:34:51.002Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484583478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13792024-02-12T01:34:51.003Z{"msg":"a1cccb6b-dd01-4a14-b485-82c4bc3b4100 is now active with session: 4ebcaa7a-59a4-4b44-9e87-a85defdde390","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484613236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13802024-02-12T01:34:51.003Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484627246Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183,"session_id":"4ebcaa7a-59a4-4b44-9e87-a85defdde390"}
13812024-02-12T01:34:51.003Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:49.484650594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
13822024-02-12T01:34:51.003ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13832024-02-12T01:34:51.004ZLoading write count information from file "/tmp/test_repair_verify.data"
13842024-02-12T01:34:51.004ZRun Repair workload
13852024-02-12T01:34:51.004Z01/30 Write block 563 len 32768 data: 3 3 3 3 2 2 2 2
13862024-02-12T01:34:51.004Z02/30 Read block 139 len 12288
13872024-02-12T01:34:51.004Z03/30 Read block 488 len 16384
13882024-02-12T01:34:51.004Z04/30 Write block 402 len 40960 data: 4 3 3 2 2 2 2 2 2 2
13892024-02-12T01:34:51.004Z05/30 Write block 263 len 24576 data: 2 2 2 2 2 2
13902024-02-12T01:34:51.004Z06/30 Write block 87 len 28672 data: 3 3 3 3 3 3 2
13912024-02-12T01:34:51.005Z07/30 Write block 77 len 24576 data: 3 3 3 3 3 3
13922024-02-12T01:34:51.005Z08/30 Write block 458 len 16384 data: 3 3 2 3
13932024-02-12T01:34:51.005Z09/30 Flush
13942024-02-12T01:34:51.005Z10/30 Write block 271 len 24576 data: 2 2 2 2 3 3
13952024-02-12T01:34:51.005Z11/30 Write block 141 len 12288 data: 2 2 2
13962024-02-12T01:34:51.006Z12/30 Read block 433 len 12288
13972024-02-12T01:34:51.006Z13/30 Write block 377 len 28672 data: 3 3 2 2 2 2 2
13982024-02-12T01:34:51.006Z14/30 Write block 274 len 16384 data: 3 4 4 3
13992024-02-12T01:34:51.006Z15/30 Read block 80 len 36864
14002024-02-12T01:34:51.006Z16/30 Write block 306 len 32768 data: 2 2 2 2 2 2 2 2
14012024-02-12T01:34:51.007Z17/30 Flush
14022024-02-12T01:34:51.007Z18/30 Write block 303 len 4096 data: 2
14032024-02-12T01:34:51.007Z19/30 Read block 30 len 24576
14042024-02-12T01:34:51.007Z20/30 Write block 337 len 40960 data: 2 2 2 2 2 2 2 2 2 2
14052024-02-12T01:34:51.007Z21/30 Read block 205 len 36864
14062024-02-12T01:34:51.007Z22/30 Read block 507 len 4096
14072024-02-12T01:34:51.007Z23/30 Read block 523 len 28672
14082024-02-12T01:34:51.008Z24/30 Read block 500 len 24576
14092024-02-12T01:34:51.008Z25/30 Read block 537 len 40960
14102024-02-12T01:34:51.008Z26/30 Read block 355 len 24576
14112024-02-12T01:34:51.008Z27/30 Read block 301 len 32768
14122024-02-12T01:34:51.008Z28/30 Write block 438 len 32768 data: 2 2 2 2 2 2 2 2
14132024-02-12T01:34:51.008Z29/30 Read block 68 len 40960
14142024-02-12T01:34:51.009Z30/30 Read block 68 len 12288
14152024-02-12T01:34:51.009Z----------------------------------------------------------------
14162024-02-12T01:34:51.009Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14172024-02-12T01:34:51.010ZStates: Active Active Active
14182024-02-12T01:34:51.010ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14192024-02-12T01:34:51.010Z 1 Acked 1000 Write 8 Done Done Done false
14202024-02-12T01:34:51.010Z 2 Acked 1001 Read 3 Done Done Done false
14212024-02-12T01:34:51.019Z 3 Acked 1002 Read 4 Done Done Done false
14222024-02-12T01:34:51.019Z 4 Acked 1003 Write 10 Done Done Done false
14232024-02-12T01:34:51.020Z 5 Acked 1004 Write 6 Done Done Done false
14242024-02-12T01:34:51.020Z 6 Acked 1005 Write 7 Done Done Done false
14252024-02-12T01:34:51.020Z 7 Acked 1006 Write 6 Sent Done Done false
14262024-02-12T01:34:51.020Z 8 Acked 1007 Write 4 Sent Done Done false
14272024-02-12T01:34:51.020Z 9 Acked 1008 Flush 0 Sent Done Done false
14282024-02-12T01:34:51.021Z 10 Acked 1009 Write 6 Sent Done Done false
14292024-02-12T01:34:51.021Z 11 Acked 1010 Write 3 Sent Done Done false
14302024-02-12T01:34:51.021Z 12 Acked 1011 Read 3 Sent Done Done false
14312024-02-12T01:34:51.021Z 13 Acked 1012 Write 7 Sent Done Done false
14322024-02-12T01:34:51.021Z 14 Acked 1013 Write 4 Sent Done Done false
14332024-02-12T01:34:51.022Z 15 Acked 1014 Read 9 Sent Done Done false
14342024-02-12T01:34:51.022Z 16 Acked 1015 Write 8 Sent Done Done false
14352024-02-12T01:34:51.022Z 17 Acked 1016 Flush 0 Sent Done Done false
14362024-02-12T01:34:51.022Z 18 Acked 1017 Write 1 Sent Done Done false
14372024-02-12T01:34:51.022Z 19 Acked 1018 Read 6 Sent Done Done false
14382024-02-12T01:34:51.022Z 20 Acked 1019 Write 10 Sent Done Done false
14392024-02-12T01:34:51.022Z 21 Acked 1020 Read 9 Sent Done Done false
14402024-02-12T01:34:51.023Z 22 Acked 1021 Read 1 Sent Done Done false
14412024-02-12T01:34:51.023Z 23 Acked 1022 Read 7 Sent Done Done false
14422024-02-12T01:34:51.023Z 24 Acked 1023 Read 6 Sent Done Done false
14432024-02-12T01:34:51.023Z 25 Acked 1024 Read 10 Sent Done Done false
14442024-02-12T01:34:51.023Z 26 Acked 1025 Read 6 Sent Done Done false
14452024-02-12T01:34:51.024Z 27 Acked 1026 Read 8 Sent Done Done false
14462024-02-12T01:34:51.024Z 28 Acked 1027 Write 8 Sent Done Done false
14472024-02-12T01:34:51.024Z 29 Acked 1028 Read 10 Sent Done Done false
14482024-02-12T01:34:51.024Z 30 Acked 1029 Read 3 Sent Done Done false
14492024-02-12T01:34:51.025Z STATES DS:0 DS:1 DS:2 TOTAL
14502024-02-12T01:34:51.025Z New 0 0 0 0
14512024-02-12T01:34:51.025Z Sent 24 0 0 24
14522024-02-12T01:34:51.025Z Done 6 30 30 66
14532024-02-12T01:34:51.025Z Skipped 0 0 0 0
14542024-02-12T01:34:51.025Z Error 0 0 0 0
14552024-02-12T01:34:51.025ZLast Flush: 0 1016 1016
14562024-02-12T01:34:51.026ZDownstairs last five completed:
14572024-02-12T01:34:51.026ZUpstairs last five completed: 30 29 28 27 26
14582024-02-12T01:34:51.026Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:49.558017954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1183}
14592024-02-12T01:34:51.026ZWrote out file "/tmp/test_repair_verify.data"
14602024-02-12T01:34:51.026Z
14612024-02-12T01:34:51.026ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14622024-02-12T01:34:51.026ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14632024-02-12T01:34:51.026Z 0 000-019 5 5 5 5 5 5 F F F
14642024-02-12T01:34:51.026Z 1 020-039 10 10 10 14 14 14 F F F
14652024-02-12T01:34:51.026Z 2 040-059 10 10 10 14 14 14 F F F
14662024-02-12T01:34:51.026Z 3 060-079 5 12 12 5 16 16 F F F <---
14672024-02-12T01:34:51.026Z 4 080-099 10 12 12 14 16 16 T F F <---
14682024-02-12T01:34:51.026Z 5 100-119 11 11 11 15 15 15 F F F
14692024-02-12T01:34:51.026Z 6 120-139 8 8 8 10 10 10 F F F
14702024-02-12T01:34:51.026Z 7 140-159 6 12 12 6 17 17 F F F <---
14712024-02-12T01:34:51.026Z 8 160-179 11 11 11 15 15 15 F F F
14722024-02-12T01:34:51.026Z 9 180-199 10 10 10 14 14 14 F F F
14732024-02-12T01:34:51.027Z 10 200-219 6 6 6 7 7 7 F F F
14742024-02-12T01:34:51.027Z 11 220-239 7 7 7 8 8 8 F F F
14752024-02-12T01:34:51.027Z 12 240-259 11 11 11 15 15 15 F F F
14762024-02-12T01:34:51.027Z 13 260-279 8 12 12 9 17 17 T F F <---
14772024-02-12T01:34:51.027Z 14 280-299 8 8 8 9 9 9 F F F
14782024-02-12T01:34:51.027Z 15 300-319 10 12 12 14 17 17 F T T <---
14792024-02-12T01:34:51.027Z 16 320-339 10 10 10 14 14 14 F T T <---
14802024-02-12T01:34:51.027Z 17 340-359 11 11 11 15 15 15 F T T <---
14812024-02-12T01:34:51.027Z 18 360-379 11 12 12 15 17 17 F F F <---
14822024-02-12T01:34:51.027Z 19 380-399 10 12 12 14 17 17 F F F <---
14832024-02-12T01:34:51.027Z 20 400-419 9 12 12 13 16 16 T F F <---
14842024-02-12T01:34:51.028Z 21 420-439 10 10 10 14 14 14 F T T <---
14852024-02-12T01:34:51.028Z 22 440-459 9 12 12 13 16 16 F T T <---
14862024-02-12T01:34:51.028Z 23 460-479 10 12 12 14 16 16 F F F <---
14872024-02-12T01:34:51.028Z 24 480-499 10 10 10 14 14 14 F F F
14882024-02-12T01:34:51.028Z 25 500-519 11 11 11 15 15 15 F F F
14892024-02-12T01:34:51.028Z 26 520-539 9 9 9 13 13 13 F F F
14902024-02-12T01:34:51.028Z 27 540-559 10 10 10 14 14 14 F F F
14912024-02-12T01:34:51.028Z 28 560-579 10 12 12 14 16 16 T F F <---
14922024-02-12T01:34:51.028Z 29 580-599 10 10 10 14 14 14 F F F
14932024-02-12T01:34:51.028ZMax gen: 12, Max flush: 17
14942024-02-12T01:34:51.028ZError: Difference in extent metadata found!
14952024-02-12T01:34:51.028ZA Difference in extent metadata is expected here
14962024-02-12T01:34:51.028ZOn loop 6
14972024-02-12T01:34:51.029Z
14982024-02-12T01:34:51.029ZVerifying data now
14992024-02-12T01:34:57.058ZNo change after verify
15002024-02-12T01:34:57.058ZLoop: 6 Downstairs dump after verify (and repair):
15012024-02-12T01:34:57.075ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
15022024-02-12T01:34:57.075Z 0 000-019 5 5 5 5 5 5 F F F
15032024-02-12T01:34:57.075Z 1 020-039 10 10 10 14 14 14 F F F
15042024-02-12T01:34:57.075Z 2 040-059 10 10 10 14 14 14 F F F
15052024-02-12T01:34:57.075Z 3 060-079 12 12 12 16 16 16 F F F
15062024-02-12T01:34:57.075Z 4 080-099 12 12 12 16 16 16 F F F
15072024-02-12T01:34:57.075Z 5 100-119 11 11 11 15 15 15 F F F
15082024-02-12T01:34:57.075Z 6 120-139 8 8 8 10 10 10 F F F
15092024-02-12T01:34:57.075Z 7 140-159 12 12 12 17 17 17 F F F
15102024-02-12T01:34:57.075Z 8 160-179 11 11 11 15 15 15 F F F
15112024-02-12T01:34:57.075Z 9 180-199 10 10 10 14 14 14 F F F
15122024-02-12T01:34:57.075Z 10 200-219 6 6 6 7 7 7 F F F
15132024-02-12T01:34:57.075Z 11 220-239 7 7 7 8 8 8 F F F
15142024-02-12T01:34:57.075Z 12 240-259 11 11 11 15 15 15 F F F
15152024-02-12T01:34:57.075Z 13 260-279 12 12 12 17 17 17 F F F
15162024-02-12T01:34:57.076Z 14 280-299 8 8 8 9 9 9 F F F
15172024-02-12T01:34:57.076Z 15 300-319 13 13 13 18 18 18 F F F
15182024-02-12T01:34:57.076Z 16 320-339 13 13 13 18 18 18 F F F
15192024-02-12T01:34:57.076Z 17 340-359 13 13 13 18 18 18 F F F
15202024-02-12T01:34:57.076Z 18 360-379 12 12 12 17 17 17 F F F
15212024-02-12T01:34:57.076Z 19 380-399 12 12 12 17 17 17 F F F
15222024-02-12T01:34:57.076Z 20 400-419 12 12 12 16 16 16 F F F
15232024-02-12T01:34:57.076Z 21 420-439 13 13 13 18 18 18 F F F
15242024-02-12T01:34:57.076Z 22 440-459 13 13 13 18 18 18 F F F
15252024-02-12T01:34:57.076Z 23 460-479 12 12 12 16 16 16 F F F
15262024-02-12T01:34:57.076Z 24 480-499 10 10 10 14 14 14 F F F
15272024-02-12T01:34:57.076Z 25 500-519 11 11 11 15 15 15 F F F
15282024-02-12T01:34:57.076Z 26 520-539 9 9 9 13 13 13 F F F
15292024-02-12T01:34:57.076Z 27 540-559 10 10 10 14 14 14 F F F
15302024-02-12T01:34:57.076Z 28 560-579 12 12 12 16 16 16 F F F
15312024-02-12T01:34:57.076Z 29 580-599 10 10 10 14 14 14 F F F
15322024-02-12T01:34:57.076ZMax gen: 13, Max flush: 18
15332024-02-12T01:34:57.079Z
15342024-02-12T01:34:57.079ZBegin loop 7
15352024-02-12T01:34:57.079ZDownstairs to restart: 0
15362024-02-12T01:34:57.094Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011162899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
15372024-02-12T01:34:57.094Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011403558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
15382024-02-12T01:34:57.094Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011421047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
15392024-02-12T01:34:57.094Z{"msg":"Crucible runtime is spawned
15402024-02-12T01:34:57.099ZCrucible fb92f6cd-d655-468c-a11f-a939106982af has session id: 9ea0d3fa-260d-49df-88e6-8cab214559ec","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011501933Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15412024-02-12T01:34:57.099Z{"msg":"Upstairs opts: Upstairs UUID: fb92f6cd-d655-468c-a11f-a939106982af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.01155593Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15422024-02-12T01:34:57.099Z{"msg":"Crucible stats registered with UUID: fb92f6cd-d655-468c-a11f-a939106982af","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011595078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15432024-02-12T01:34:57.099Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011619227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
15442024-02-12T01:34:57.100Z{"msg":"fb92f6cd-d655-468c-a11f-a939106982af active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011639076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15452024-02-12T01:34:57.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011659355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15462024-02-12T01:34:57.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011674095Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15472024-02-12T01:34:57.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011703993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15482024-02-12T01:34:57.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011729992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15492024-02-12T01:34:57.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.01176682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15502024-02-12T01:34:57.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011791859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15512024-02-12T01:34:57.100Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011820718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15522024-02-12T01:34:57.100Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011841297Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15532024-02-12T01:34:57.100Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.011865776Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15542024-02-12T01:34:57.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012082065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15552024-02-12T01:34:57.100Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:56.012247927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15562024-02-12T01:34:57.100Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:56.012293965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15572024-02-12T01:34:57.100Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012315004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15582024-02-12T01:34:57.100Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:56.012361232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15592024-02-12T01:34:57.100Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012385601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15602024-02-12T01:34:57.100Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.01240238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15612024-02-12T01:34:57.100Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012515554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15622024-02-12T01:34:57.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012927595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15632024-02-12T01:34:57.100Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.012977023Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15642024-02-12T01:34:57.100Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.0130302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15652024-02-12T01:34:57.100Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.013061618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15662024-02-12T01:34:57.100Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.013536596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15672024-02-12T01:34:57.100Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.013586313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15682024-02-12T01:34:57.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014130817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15692024-02-12T01:34:57.101Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014167126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15702024-02-12T01:34:57.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014182785Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15712024-02-12T01:34:57.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014203454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15722024-02-12T01:34:57.101Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014217863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15732024-02-12T01:34:57.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:56.014237752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15742024-02-12T01:34:58.097Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.013964826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15752024-02-12T01:34:58.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.014037623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15762024-02-12T01:34:58.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.014971588Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15772024-02-12T01:34:58.102Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.015020236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15782024-02-12T01:34:58.102Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.015486083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15792024-02-12T01:34:58.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.01598323Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15802024-02-12T01:34:58.102Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016011968Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15812024-02-12T01:34:58.102Z{"msg":"[0]R flush_numbers[0..12]: [5, 14, 14, 16, 16, 15, 10, 17, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016112293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15822024-02-12T01:34:58.102Z{"msg":"[0]R generation[0..12]: [5, 10, 10, 12, 12, 11, 8, 12, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016143382Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15832024-02-12T01:34:58.102Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016158861Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15842024-02-12T01:34:58.102Z{"msg":"[1]R flush_numbers[0..12]: [5, 14, 14, 16, 16, 15, 10, 17, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016173641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15852024-02-12T01:34:58.102Z{"msg":"[1]R generation[0..12]: [5, 10, 10, 12, 12, 11, 8, 12, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.01619391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15862024-02-12T01:34:58.102Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016208529Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15872024-02-12T01:34:58.102Z{"msg":"[2]R flush_numbers[0..12]: [5, 14, 14, 16, 16, 15, 10, 17, 15, 14, 7, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016229888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15882024-02-12T01:34:58.102Z{"msg":"[2]R generation[0..12]: [5, 10, 10, 12, 12, 11, 8, 12, 11, 10, 6, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016243997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15892024-02-12T01:34:58.102Z{"msg":"Wait for a query_work_queue command to finish before sending IO
15902024-02-12T01:34:58.102Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016267676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15912024-02-12T01:34:58.102Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016282025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15922024-02-12T01:34:58.102Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016303255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15932024-02-12T01:34:58.102Z{"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016316344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15942024-02-12T01:34:58.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016340453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15952024-02-12T01:34:58.103Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016359212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15962024-02-12T01:34:58.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016372511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15972024-02-12T01:34:58.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.01639371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15982024-02-12T01:34:58.103Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016416599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
15992024-02-12T01:34:58.103Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016433418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
16002024-02-12T01:34:58.103Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016446128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
16012024-02-12T01:34:58.103Z{"msg":"fb92f6cd-d655-468c-a11f-a939106982af is now active with session: 9ea0d3fa-260d-49df-88e6-8cab214559ec","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016459357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
16022024-02-12T01:34:58.103Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016479126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191,"session_id":"9ea0d3fa-260d-49df-88e6-8cab214559ec"}
16032024-02-12T01:34:58.103Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:34:57.016491815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
16042024-02-12T01:34:58.103ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
16052024-02-12T01:34:58.103ZLoading write count information from file "/tmp/test_repair_verify.data"
16062024-02-12T01:34:58.103ZRun Repair workload
16072024-02-12T01:34:58.109Z01/30 Write block 206 len 16384 data: 3 3 3 3
16082024-02-12T01:34:58.109Z02/30 Write block 284 len 24576 data: 2 2 2 2 2 2
16092024-02-12T01:34:58.109Z03/30 Read block 407 len 32768
16102024-02-12T01:34:58.109Z04/30 Read block 353 len 32768
16112024-02-12T01:34:58.109Z05/30 Read block 106 len 36864
16122024-02-12T01:34:58.112Z06/30 Write block 352 len 20480 data: 4 3 3 4 3
16132024-02-12T01:34:58.112Z07/30 Flush
16142024-02-12T01:34:58.112Z08/30 Write block 556 len 40960 data: 2 2 2 3 3 3 3 4 4 4
16152024-02-12T01:34:58.112Z09/30 Read block 233 len 32768
16162024-02-12T01:34:58.115Z10/30 Flush
16172024-02-12T01:34:58.115Z11/30 Write block 58 len 8192 data: 3 3
16182024-02-12T01:34:58.115Z12/30 Read block 58 len 20480
16192024-02-12T01:34:58.118Z13/30 Read block 234 len 20480
16202024-02-12T01:34:58.118Z14/30 Write block 545 len 40960 data: 3 3 2 2 2 3 2 2 2 2
16212024-02-12T01:34:58.118Z15/30 Write block 84 len 16384 data: 3 2 2 4
16222024-02-12T01:34:58.118Z16/30 Write block 247 len 12288 data: 3 3 3
16232024-02-12T01:34:58.121Z17/30 Write block 291 len 4096 data: 2
16242024-02-12T01:34:58.121Z18/30 Read block 31 len 36864
16252024-02-12T01:34:58.121Z19/30 Write block 466 len 40960 data: 3 3 3 3 2 2 2 2 2 2
16262024-02-12T01:34:58.121Z20/30 Read block 387 len 16384
16272024-02-12T01:34:58.124Z21/30 Read block 348 len 24576
16282024-02-12T01:34:58.124Z22/30 Read block 24 len 36864
16292024-02-12T01:34:58.124Z23/30 Read block 456 len 20480
16302024-02-12T01:34:58.127Z24/30 Write block 257 len 8192 data: 2 2
16312024-02-12T01:34:58.127Z25/30 Write block 547 len 24576 data: 3 3 3 4 3 3
16322024-02-12T01:34:58.127Z26/30 Read block 387 len 4096
16332024-02-12T01:34:58.127Z27/30 Read block 376 len 24576
16342024-02-12T01:34:58.130Z28/30 Read block 56 len 16384
16352024-02-12T01:34:58.131Z29/30 Write block 487 len 20480 data: 2 3 3 3 3
16362024-02-12T01:34:58.131Z30/30 Write block 180 len 40960 data: 3 3 2 2 3 3 3 3 5 5
16372024-02-12T01:34:58.131Z----------------------------------------------------------------
16382024-02-12T01:34:58.131Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16392024-02-12T01:34:58.131ZStates: Active Active Active
16402024-02-12T01:34:58.131ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16412024-02-12T01:34:58.131Z 1 Acked 1000 Write 4 Done Done Done false
16422024-02-12T01:34:58.131Z 2 Acked 1001 Write 6 Done Done Done false
16432024-02-12T01:34:58.131Z 3 Acked 1002 Read 8 Done Done Done false
16442024-02-12T01:34:58.131Z 4 Acked 1003 Read 8 Done Done Done false
16452024-02-12T01:34:58.131Z 5 Acked 1004 Read 9 Done Done Done false
16462024-02-12T01:34:58.131Z 6 Acked 1005 Write 5 Done Done Done false
16472024-02-12T01:34:58.131Z 7 Acked 1006 Flush 0 Sent Done Done false
16482024-02-12T01:34:58.131Z 8 Acked 1007 Write 10 Sent Done Done false
16492024-02-12T01:34:58.131Z 9 Acked 1008 Read 8 Sent Done Done false
16502024-02-12T01:34:58.131Z 10 Acked 1009 Flush 0 Sent Done Done false
16512024-02-12T01:34:58.131Z 11 Acked 1010 Write 2 Sent Done Done false
16522024-02-12T01:34:58.131Z 12 Acked 1011 Read 5 Sent Done Done false
16532024-02-12T01:34:58.131Z 13 Acked 1012 Read 5 Sent Done Done false
16542024-02-12T01:34:58.131Z 14 Acked 1013 Write 10 Sent Done Done false
16552024-02-12T01:34:58.131Z 15 Acked 1014 Write 4 Sent Done Done false
16562024-02-12T01:34:58.131Z 16 Acked 1015 Write 3 Sent Done Done false
16572024-02-12T01:34:58.132Z 17 Acked 1016 Write 1 Sent Done Done false
16582024-02-12T01:34:58.132Z 18 Acked 1017 Read 9 Sent Done Done false
16592024-02-12T01:34:58.132Z 19 Acked 1018 Write 10 Sent Done Done false
16602024-02-12T01:34:58.132Z 20 Acked 1019 Read 4 Sent Done Done false
16612024-02-12T01:34:58.132Z 21 Acked 1020 Read 6 Sent Done Done false
16622024-02-12T01:34:58.132Z 22 Acked 1021 Read 9 Sent Done Done false
16632024-02-12T01:34:58.132Z 23 Acked 1022 Read 5 Sent Done Done false
16642024-02-12T01:34:58.132Z 24 Acked 1023 Write 2 Sent Done Done false
16652024-02-12T01:34:58.132Z 25 Acked 1024 Write 6 Sent Done Done false
16662024-02-12T01:34:58.132Z 26 Acked 1025 Read 1 Sent Done Done false
16672024-02-12T01:34:58.132Z 27 Acked 1026 Read 6 Sent Done Done false
16682024-02-12T01:34:58.132Z 28 Acked 1027 Read 4 Sent Done Done false
16692024-02-12T01:34:58.132Z 29 Acked 1028 Write 5 Sent Done Done false
16702024-02-12T01:34:58.132Z 30 Acked 1029 Write 10 Sent Sent Sent false
16712024-02-12T01:34:58.132Z STATES DS:0 DS:1 DS:2 TOTAL
16722024-02-12T01:34:58.132Z New 0 0 0 0
16732024-02-12T01:34:58.132Z Sent 24 1 1 26
16742024-02-12T01:34:58.132Z Done 6 29 29 64
16752024-02-12T01:34:58.132Z Skipped 0 0 0 0
16762024-02-12T01:34:58.132Z Error 0 0 0 0
16772024-02-12T01:34:58.132ZLast Flush: 0 1009 1009
16782024-02-12T01:34:58.132ZDownstairs last five completed:
16792024-02-12T01:34:58.132ZUpstairs last five completed: 30 29 28 27 26
16802024-02-12T01:34:58.136Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:34:57.047277287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1191}
16812024-02-12T01:34:58.136ZWrote out file "/tmp/test_repair_verify.data"
16822024-02-12T01:34:58.136Z
16832024-02-12T01:34:58.139ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16842024-02-12T01:34:58.157ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16852024-02-12T01:34:58.157Z 0 000-019 5 5 5 5 5 5 F F F
16862024-02-12T01:34:58.157Z 1 020-039 10 10 10 14 14 14 F F F
16872024-02-12T01:34:58.157Z 2 040-059 10 10 10 14 14 14 F T T <---
16882024-02-12T01:34:58.157Z 3 060-079 12 12 12 16 16 16 F F F
16892024-02-12T01:34:58.157Z 4 080-099 12 12 12 16 16 16 F T T <---
16902024-02-12T01:34:58.157Z 5 100-119 11 11 11 15 15 15 F F F
16912024-02-12T01:34:58.157Z 6 120-139 8 8 8 10 10 10 F F F
16922024-02-12T01:34:58.160Z 7 140-159 12 12 12 17 17 17 F F F
16932024-02-12T01:34:58.160Z 8 160-179 11 11 11 15 15 15 F F F
16942024-02-12T01:34:58.160Z 9 180-199 10 10 10 14 14 14 F T T <---
16952024-02-12T01:34:58.160Z 10 200-219 6 14 14 7 19 19 T F F <---
16962024-02-12T01:34:58.160Z 11 220-239 7 7 7 8 8 8 F F F
16972024-02-12T01:34:58.160Z 12 240-259 11 11 11 15 15 15 F T T <---
16982024-02-12T01:34:58.161Z 13 260-279 12 12 12 17 17 17 F F F
16992024-02-12T01:34:58.161Z 14 280-299 8 14 14 9 19 19 T T T <---
17002024-02-12T01:34:58.161Z 15 300-319 13 13 13 18 18 18 F F F
17012024-02-12T01:34:58.161Z 16 320-339 13 13 13 18 18 18 F F F
17022024-02-12T01:34:58.161Z 17 340-359 13 14 14 18 19 19 T F F <---
17032024-02-12T01:34:58.161Z 18 360-379 12 12 12 17 17 17 F F F
17042024-02-12T01:34:58.161Z 19 380-399 12 12 12 17 17 17 F F F
17052024-02-12T01:34:58.161Z 20 400-419 12 12 12 16 16 16 F F F
17062024-02-12T01:34:58.161Z 21 420-439 13 13 13 18 18 18 F F F
17072024-02-12T01:34:58.161Z 22 440-459 13 13 13 18 18 18 F F F
17082024-02-12T01:34:58.161Z 23 460-479 12 12 12 16 16 16 F T T <---
17092024-02-12T01:34:58.161Z 24 480-499 10 10 10 14 14 14 F T T <---
17102024-02-12T01:34:58.161Z 25 500-519 11 11 11 15 15 15 F F F
17112024-02-12T01:34:58.161Z 26 520-539 9 9 9 13 13 13 F F F
17122024-02-12T01:34:58.161Z 27 540-559 10 14 14 14 20 20 F T T <---
17132024-02-12T01:34:58.161Z 28 560-579 12 14 14 16 20 20 F F F <---
17142024-02-12T01:34:58.161Z 29 580-599 10 10 10 14 14 14 F F F
17152024-02-12T01:34:58.161ZMax gen: 14, Max flush: 20
17162024-02-12T01:34:58.161ZError: Difference in extent metadata found!
17172024-02-12T01:34:58.161ZA Difference in extent metadata is expected here
17182024-02-12T01:34:58.161ZOn loop 7
17192024-02-12T01:34:58.161Z
17202024-02-12T01:34:58.164ZVerifying data now
17212024-02-12T01:35:04.210ZNo change after verify
17222024-02-12T01:35:04.210ZLoop: 7 Downstairs dump after verify (and repair):
17232024-02-12T01:35:04.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17242024-02-12T01:35:04.225Z 0 000-019 5 5 5 5 5 5 F F F
17252024-02-12T01:35:04.225Z 1 020-039 10 10 10 14 14 14 F F F
17262024-02-12T01:35:04.225Z 2 040-059 15 15 15 21 21 21 F F F
17272024-02-12T01:35:04.225Z 3 060-079 12 12 12 16 16 16 F F F
17282024-02-12T01:35:04.225Z 4 080-099 15 15 15 21 21 21 F F F
17292024-02-12T01:35:04.225Z 5 100-119 11 11 11 15 15 15 F F F
17302024-02-12T01:35:04.225Z 6 120-139 8 8 8 10 10 10 F F F
17312024-02-12T01:35:04.225Z 7 140-159 12 12 12 17 17 17 F F F
17322024-02-12T01:35:04.225Z 8 160-179 11 11 11 15 15 15 F F F
17332024-02-12T01:35:04.225Z 9 180-199 15 15 15 21 21 21 F F F
17342024-02-12T01:35:04.225Z 10 200-219 14 14 14 19 19 19 F F F
17352024-02-12T01:35:04.229Z 11 220-239 7 7 7 8 8 8 F F F
17362024-02-12T01:35:04.229Z 12 240-259 15 15 15 21 21 21 F F F
17372024-02-12T01:35:04.230Z 13 260-279 12 12 12 17 17 17 F F F
17382024-02-12T01:35:04.230Z 14 280-299 15 15 15 21 21 21 F F F
17392024-02-12T01:35:04.230Z 15 300-319 13 13 13 18 18 18 F F F
17402024-02-12T01:35:04.230Z 16 320-339 13 13 13 18 18 18 F F F
17412024-02-12T01:35:04.230Z 17 340-359 14 14 14 19 19 19 F F F
17422024-02-12T01:35:04.230Z 18 360-379 12 12 12 17 17 17 F F F
17432024-02-12T01:35:04.230Z 19 380-399 12 12 12 17 17 17 F F F
17442024-02-12T01:35:04.230Z 20 400-419 12 12 12 16 16 16 F F F
17452024-02-12T01:35:04.230Z 21 420-439 13 13 13 18 18 18 F F F
17462024-02-12T01:35:04.230Z 22 440-459 13 13 13 18 18 18 F F F
17472024-02-12T01:35:04.230Z 23 460-479 15 15 15 21 21 21 F F F
17482024-02-12T01:35:04.230Z 24 480-499 15 15 15 21 21 21 F F F
17492024-02-12T01:35:04.230Z 25 500-519 11 11 11 15 15 15 F F F
17502024-02-12T01:35:04.230Z 26 520-539 9 9 9 13 13 13 F F F
17512024-02-12T01:35:04.230Z 27 540-559 15 15 15 21 21 21 F F F
17522024-02-12T01:35:04.230Z 28 560-579 14 14 14 20 20 20 F F F
17532024-02-12T01:35:04.230Z 29 580-599 10 10 10 14 14 14 F F F
17542024-02-12T01:35:04.230ZMax gen: 15, Max flush: 21
17552024-02-12T01:35:04.230Z
17562024-02-12T01:35:04.230ZBegin loop 8
17572024-02-12T01:35:04.230ZDownstairs to restart: 2
17582024-02-12T01:35:04.247Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.16194284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
17592024-02-12T01:35:04.247Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162175989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
17602024-02-12T01:35:04.256Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162211027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
17612024-02-12T01:35:04.256ZCrucible runtime is spawned
17622024-02-12T01:35:04.256Z{"msg":"Crucible c57f6d0c-4a34-4bfb-a297-b41e5f59e909 has session id: 2e959875-84b0-4ae7-a526-8cdad54b3778","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162252445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17632024-02-12T01:35:04.256Z{"msg":"Upstairs opts: Upstairs UUID: c57f6d0c-4a34-4bfb-a297-b41e5f59e909, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162307723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17642024-02-12T01:35:04.256Z{"msg":"Crucible stats registered with UUID: c57f6d0c-4a34-4bfb-a297-b41e5f59e909","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162347411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17652024-02-12T01:35:04.256Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.16236353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
17662024-02-12T01:35:04.256Z{"msg":"c57f6d0c-4a34-4bfb-a297-b41e5f59e909 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162471285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17672024-02-12T01:35:04.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162500424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17682024-02-12T01:35:04.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162526372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17692024-02-12T01:35:04.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162565561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17702024-02-12T01:35:04.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162604799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17712024-02-12T01:35:04.256Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162630687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17722024-02-12T01:35:04.256Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162661936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17732024-02-12T01:35:04.256Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162692944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17742024-02-12T01:35:04.257Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162725093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17752024-02-12T01:35:04.257Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162773841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17762024-02-12T01:35:04.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162814529Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17772024-02-12T01:35:04.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162849817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17782024-02-12T01:35:04.257Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.162940863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17792024-02-12T01:35:04.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.163604371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17802024-02-12T01:35:04.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.163661558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17812024-02-12T01:35:04.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.163718856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17822024-02-12T01:35:04.257Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.163761074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17832024-02-12T01:35:04.257Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164168664Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17842024-02-12T01:35:04.257Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164224331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17852024-02-12T01:35:04.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164593454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17862024-02-12T01:35:04.257Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164648611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17872024-02-12T01:35:04.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.16467891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17882024-02-12T01:35:04.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164781135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17892024-02-12T01:35:04.257Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164817253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17902024-02-12T01:35:04.257Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164832802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17912024-02-12T01:35:04.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164852572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17922024-02-12T01:35:04.257Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.164873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17932024-02-12T01:35:04.257Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165325919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17942024-02-12T01:35:04.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165713671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17952024-02-12T01:35:04.257Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165744519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17962024-02-12T01:35:04.257Z{"msg":"[0]R flush_numbers[0..12]: [5, 14, 21, 16, 21, 15, 10, 17, 15, 21, 19, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165759148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17972024-02-12T01:35:04.258Z{"msg":"[0]R generation[0..12]: [5, 10, 15, 12, 15, 11, 8, 12, 11, 15, 14, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165780037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17982024-02-12T01:35:04.258Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165794476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
17992024-02-12T01:35:04.258Z{"msg":"[1]R flush_numbers[0..12]: [5, 14, 21, 16, 21, 15, 10, 17, 15, 21, 19, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165814646Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18002024-02-12T01:35:04.258Z{"msg":"[1]R generation[0..12]: [5, 10, 15, 12, 15, 11, 8, 12, 11, 15, 14, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165828665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18012024-02-12T01:35:04.258Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165849244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18022024-02-12T01:35:04.258Z{"msg":"[2]R flush_numbers[0..12]: [5, 14, 21, 16, 21, 15, 10, 17, 15, 21, 19, 8]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165863453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18032024-02-12T01:35:04.258Z{"msg":"[2]R generation[0..12]: [5, 10, 15, 12, 15, 11, 8, 12, 11, 15, 14, 7]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165883182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18042024-02-12T01:35:04.258Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165897492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18052024-02-12T01:35:04.258Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165917091Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
18062024-02-12T01:35:04.258Zdownstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18072024-02-12T01:35:04.258Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.16593357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18082024-02-12T01:35:04.258Z{"msg":"Next flush: 22","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165947139Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18092024-02-12T01:35:04.258Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165966958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18102024-02-12T01:35:04.258Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165980588Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18112024-02-12T01:35:04.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.165998947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18122024-02-12T01:35:04.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166012206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18132024-02-12T01:35:04.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166026445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18142024-02-12T01:35:04.258Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166039785Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18152024-02-12T01:35:04.258Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166052874Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18162024-02-12T01:35:04.258Z{"msg":"c57f6d0c-4a34-4bfb-a297-b41e5f59e909 is now active with session: 2e959875-84b0-4ae7-a526-8cdad54b3778","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166072553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18172024-02-12T01:35:04.258Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166084783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199,"session_id":"2e959875-84b0-4ae7-a526-8cdad54b3778"}
18182024-02-12T01:35:04.258Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:03.166103892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
18192024-02-12T01:35:04.258ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18202024-02-12T01:35:04.262ZLoading write count information from file "/tmp/test_repair_verify.data"
18212024-02-12T01:35:04.262ZRun Repair workload
18222024-02-12T01:35:04.262Z01/30 Write block 284 len 12288 data: 3 3 3
18232024-02-12T01:35:04.262Z02/30 Read block 45 len 16384
18242024-02-12T01:35:04.262Z03/30 Write block 367 len 40960 data: 3 2 2 2 2 2 2 2 3 3
18252024-02-12T01:35:04.263Z04/30 Flush
18262024-02-12T01:35:04.305Z05/30 Flush
18272024-02-12T01:35:04.308Z06/30 Write block 237 len 12288 data: 3 3 3
18282024-02-12T01:35:04.308Z07/30 Read block 183 len 8192
18292024-02-12T01:35:04.308Z08/30 Read block 275 len 32768
18302024-02-12T01:35:04.308Z09/30 Flush
18312024-02-12T01:35:04.312Z10/30 Write block 260 len 32768 data: 2 2 2 3 3 3 3 3
18322024-02-12T01:35:04.317Z11/30 Read block 335 len 4096
18332024-02-12T01:35:04.317Z12/30 Read block 270 len 12288
18342024-02-12T01:35:04.317Z13/30 Read block 577 len 36864
18352024-02-12T01:35:04.318Z14/30 Read block 0 len 8192
18362024-02-12T01:35:04.318Z15/30 Write block 589 len 12288 data: 4 4 4
18372024-02-12T01:35:04.318Z16/30 Write block 408 len 4096 data: 3
18382024-02-12T01:35:04.318Z17/30 Write block 101 len 32768 data: 2 2 2 2 2 2 2 2
18392024-02-12T01:35:04.318Z18/30 Write block 501 len 20480 data: 4 2 2 2 2
18402024-02-12T01:35:04.323Z19/30 Read block 179 len 28672
18412024-02-12T01:35:04.323Z20/30 Read block 62 len 32768
18422024-02-12T01:35:04.323Z21/30 Read block 320 len 32768
18432024-02-12T01:35:04.323Z22/30 Write block 572 len 16384 data: 2 2 2 4
18442024-02-12T01:35:04.327Z23/30 Write block 502 len 4096 data: 3
18452024-02-12T01:35:04.327Z24/30 Write block 571 len 24576 data: 2 3 3 3 5 4
18462024-02-12T01:35:04.327Z25/30 Read block 340 len 4096
18472024-02-12T01:35:04.327Z26/30 Write block 47 len 20480 data: 3 2 2 2 2
18482024-02-12T01:35:04.327Z27/30 Write block 343 len 28672 data: 3 3 3 3 3 3 3
18492024-02-12T01:35:04.327Z28/30 Write block 252 len 12288 data: 2 2 2
18502024-02-12T01:35:04.327Z29/30 Read block 218 len 8192
18512024-02-12T01:35:04.327Z30/30 Read block 42 len 36864
18522024-02-12T01:35:04.331Z----------------------------------------------------------------
18532024-02-12T01:35:04.331Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18542024-02-12T01:35:04.331ZStates: Active Active Active
18552024-02-12T01:35:04.331ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18562024-02-12T01:35:04.331Z 1 Acked 1000 Write 3 Done Done Done false
18572024-02-12T01:35:04.331Z 2 Acked 1001 Read 4 Done Done Done false
18582024-02-12T01:35:04.331Z 3 Acked 1002 Write 10 Done Done Done false
18592024-02-12T01:35:04.331Z 4 Acked 1003 Flush 0 Done Done Sent false
18602024-02-12T01:35:04.331Z 5 Acked 1004 Flush 0 Done Done Sent false
18612024-02-12T01:35:04.332Z 6 Acked 1005 Write 3 Done Done Sent false
18622024-02-12T01:35:04.332Z 7 Acked 1006 Read 2 Done Done Sent false
18632024-02-12T01:35:04.332Z 8 Acked 1007 Read 8 Done Done Sent false
18642024-02-12T01:35:04.332Z 9 Acked 1008 Flush 0 Done Done Sent false
18652024-02-12T01:35:04.332Z 10 Acked 1009 Write 8 Done Done Sent false
18662024-02-12T01:35:04.332Z 11 Acked 1010 Read 1 Done Done Sent false
18672024-02-12T01:35:04.332Z 12 Acked 1011 Read 3 Done Done Sent false
18682024-02-12T01:35:04.332Z 13 Acked 1012 Read 9 Done Done Sent false
18692024-02-12T01:35:04.332Z 14 Acked 1013 Read 2 Done Done Sent false
18702024-02-12T01:35:04.332Z 15 Acked 1014 Write 3 Done Done Sent false
18712024-02-12T01:35:04.332Z 16 Acked 1015 Write 1 Done Done Sent false
18722024-02-12T01:35:04.332Z 17 Acked 1016 Write 8 Done Done Sent false
18732024-02-12T01:35:04.332Z 18 Acked 1017 Write 5 Done Done Sent false
18742024-02-12T01:35:04.332Z 19 Acked 1018 Read 7 Done Done Sent false
18752024-02-12T01:35:04.332Z 20 Acked 1019 Read 8 Done Done Sent false
18762024-02-12T01:35:04.332Z 21 Acked 1020 Read 8 Done Done Sent false
18772024-02-12T01:35:04.332Z 22 Acked 1021 Write 4 Done Done Sent false
18782024-02-12T01:35:04.332Z 23 Acked 1022 Write 1 Done Done Sent false
18792024-02-12T01:35:04.332Z 24 Acked 1023 Write 6 Done Done Sent false
18802024-02-12T01:35:04.332Z 25 Acked 1024 Read 1 Done Done Sent false
18812024-02-12T01:35:04.332Z 26 Acked 1025 Write 5 Done Done Sent false
18822024-02-12T01:35:04.332Z 27 Acked 1026 Write 7 Done Done Sent false
18832024-02-12T01:35:04.332Z 28 Acked 1027 Write 3 Done Done Sent false
18842024-02-12T01:35:04.332Z 29 Acked 1028 Read 2 Done Done Sent false
18852024-02-12T01:35:04.333Z 30 Acked 1029 Read 9 Done Sent Sent false
18862024-02-12T01:35:04.333Z STATES DS:0 DS:1 DS:2 TOTAL
18872024-02-12T01:35:04.333Z New 0 0 0 0
18882024-02-12T01:35:04.333Z Sent 0 1 27 28
18892024-02-12T01:35:04.333Z Done 30 29 3 62
18902024-02-12T01:35:04.333Z Skipped 0 0 0 0
18912024-02-12T01:35:04.333Z Error 0 0 0 0
18922024-02-12T01:35:04.333ZLast Flush: 1008 1008 0
18932024-02-12T01:35:04.333ZDownstairs last five completed:
18942024-02-12T01:35:04.333ZUpstairs last five completed: 30 29 28 27 26
18952024-02-12T01:35:04.333Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:03.243717259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1199}
18962024-02-12T01:35:04.333ZWrote out file "/tmp/test_repair_verify.data"
18972024-02-12T01:35:04.336Z
18982024-02-12T01:35:04.336ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18992024-02-12T01:35:04.354ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19002024-02-12T01:35:04.354Z 0 000-019 5 5 5 5 5 5 F F F
19012024-02-12T01:35:04.354Z 1 020-039 10 10 10 14 14 14 F F F
19022024-02-12T01:35:04.354Z 2 040-059 15 15 15 21 21 21 T T F <---
19032024-02-12T01:35:04.354Z 3 060-079 12 12 12 16 16 16 F F F
19042024-02-12T01:35:04.354Z 4 080-099 15 15 15 21 21 21 F F F
19052024-02-12T01:35:04.354Z 5 100-119 11 11 11 15 15 15 T T F <---
19062024-02-12T01:35:04.355Z 6 120-139 8 8 8 10 10 10 F F F
19072024-02-12T01:35:04.355Z 7 140-159 12 12 12 17 17 17 F F F
19082024-02-12T01:35:04.355Z 8 160-179 11 11 11 15 15 15 F F F
19092024-02-12T01:35:04.355Z 9 180-199 15 15 15 21 21 21 F F F
19102024-02-12T01:35:04.355Z 10 200-219 14 14 14 19 19 19 F F F
19112024-02-12T01:35:04.355Z 11 220-239 16 16 7 24 24 8 F F F <---
19122024-02-12T01:35:04.355Z 12 240-259 15 15 15 21 21 21 T T F <---
19132024-02-12T01:35:04.355Z 13 260-279 12 12 12 17 17 17 T T F <---
19142024-02-12T01:35:04.355Z 14 280-299 16 16 15 22 22 21 F F T <---
19152024-02-12T01:35:04.358Z 15 300-319 13 13 13 18 18 18 F F F
19162024-02-12T01:35:04.358Z 16 320-339 13 13 13 18 18 18 F F F
19172024-02-12T01:35:04.358Z 17 340-359 14 14 14 19 19 19 T T F <---
19182024-02-12T01:35:04.358Z 18 360-379 16 16 12 22 22 17 F F T <---
19192024-02-12T01:35:04.359Z 19 380-399 12 12 12 17 17 17 F F F
19202024-02-12T01:35:04.359Z 20 400-419 12 12 12 16 16 16 T T F <---
19212024-02-12T01:35:04.359Z 21 420-439 13 13 13 18 18 18 F F F
19222024-02-12T01:35:04.359Z 22 440-459 13 13 13 18 18 18 F F F
19232024-02-12T01:35:04.359Z 23 460-479 15 15 15 21 21 21 F F F
19242024-02-12T01:35:04.359Z 24 480-499 15 15 15 21 21 21 F F F
19252024-02-12T01:35:04.359Z 25 500-519 11 11 11 15 15 15 T T F <---
19262024-02-12T01:35:04.359Z 26 520-539 9 9 9 13 13 13 F F F
19272024-02-12T01:35:04.359Z 27 540-559 15 15 15 21 21 21 F F F
19282024-02-12T01:35:04.359Z 28 560-579 14 14 14 20 20 20 T T F <---
19292024-02-12T01:35:04.359Z 29 580-599 10 10 10 14 14 14 T T F <---
19302024-02-12T01:35:04.359ZMax gen: 16, Max flush: 24
19312024-02-12T01:35:04.359ZError: Difference in extent metadata found!
19322024-02-12T01:35:04.359ZA Difference in extent metadata is expected here
19332024-02-12T01:35:04.359ZOn loop 8
19342024-02-12T01:35:04.359Z
19352024-02-12T01:35:04.363ZVerifying data now
19362024-02-12T01:35:10.672ZNo change after verify
19372024-02-12T01:35:10.673ZLoop: 8 Downstairs dump after verify (and repair):
19382024-02-12T01:35:10.687ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19392024-02-12T01:35:10.687Z 0 000-019 5 5 5 5 5 5 F F F
19402024-02-12T01:35:10.687Z 1 020-039 10 10 10 14 14 14 F F F
19412024-02-12T01:35:10.688Z 2 040-059 17 17 17 25 25 25 F F F
19422024-02-12T01:35:10.688Z 3 060-079 12 12 12 16 16 16 F F F
19432024-02-12T01:35:10.688Z 4 080-099 15 15 15 21 21 21 F F F
19442024-02-12T01:35:10.691Z 5 100-119 17 17 17 25 25 25 F F F
19452024-02-12T01:35:10.691Z 6 120-139 8 8 8 10 10 10 F F F
19462024-02-12T01:35:10.691Z 7 140-159 12 12 12 17 17 17 F F F
19472024-02-12T01:35:10.691Z 8 160-179 11 11 11 15 15 15 F F F
19482024-02-12T01:35:10.691Z 9 180-199 15 15 15 21 21 21 F F F
19492024-02-12T01:35:10.691Z 10 200-219 14 14 14 19 19 19 F F F
19502024-02-12T01:35:10.691Z 11 220-239 16 16 16 24 24 24 F F F
19512024-02-12T01:35:10.691Z 12 240-259 17 17 17 25 25 25 F F F
19522024-02-12T01:35:10.691Z 13 260-279 17 17 17 25 25 25 F F F
19532024-02-12T01:35:10.691Z 14 280-299 16 16 16 22 22 22 F F F
19542024-02-12T01:35:10.691Z 15 300-319 13 13 13 18 18 18 F F F
19552024-02-12T01:35:10.691Z 16 320-339 13 13 13 18 18 18 F F F
19562024-02-12T01:35:10.691Z 17 340-359 17 17 17 25 25 25 F F F
19572024-02-12T01:35:10.691Z 18 360-379 16 16 16 22 22 22 F F F
19582024-02-12T01:35:10.691Z 19 380-399 12 12 12 17 17 17 F F F
19592024-02-12T01:35:10.692Z 20 400-419 17 17 17 25 25 25 F F F
19602024-02-12T01:35:10.692Z 21 420-439 13 13 13 18 18 18 F F F
19612024-02-12T01:35:10.692Z 22 440-459 13 13 13 18 18 18 F F F
19622024-02-12T01:35:10.692Z 23 460-479 15 15 15 21 21 21 F F F
19632024-02-12T01:35:10.692Z 24 480-499 15 15 15 21 21 21 F F F
19642024-02-12T01:35:10.692Z 25 500-519 17 17 17 25 25 25 F F F
19652024-02-12T01:35:10.692Z 26 520-539 9 9 9 13 13 13 F F F
19662024-02-12T01:35:10.692Z 27 540-559 15 15 15 21 21 21 F F F
19672024-02-12T01:35:10.692Z 28 560-579 17 17 17 25 25 25 F F F
19682024-02-12T01:35:10.692Z 29 580-599 17 17 17 25 25 25 F F F
19692024-02-12T01:35:10.692ZMax gen: 17, Max flush: 25
19702024-02-12T01:35:10.695Z
19712024-02-12T01:35:10.695ZBegin loop 9
19722024-02-12T01:35:10.695ZDownstairs to restart: 2
19732024-02-12T01:35:10.710Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.62506123Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
19742024-02-12T01:35:10.710Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"Crucible runtime is spawned
19752024-02-12T01:35:10.714Zname":"crucible","level":30,"time":"2024-02-12T01:35:09.625393014Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
19762024-02-12T01:35:10.714Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625433652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
19772024-02-12T01:35:10.714Z{"msg":"Crucible c00cba16-da6f-49fc-a19f-a942fbf1daa6 has session id: cbe6438e-9600-40fc-9ae8-c528b979235e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.62548393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19782024-02-12T01:35:10.714Z{"msg":"Upstairs opts: Upstairs UUID: c00cba16-da6f-49fc-a19f-a942fbf1daa6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625516608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19792024-02-12T01:35:10.714Z{"msg":"Crucible stats registered with UUID: c00cba16-da6f-49fc-a19f-a942fbf1daa6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625546707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19802024-02-12T01:35:10.714Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625564236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
19812024-02-12T01:35:10.715Z{"msg":"c00cba16-da6f-49fc-a19f-a942fbf1daa6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625644442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19822024-02-12T01:35:10.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625676071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19832024-02-12T01:35:10.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625703579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19842024-02-12T01:35:10.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625761477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19852024-02-12T01:35:10.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625794695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19862024-02-12T01:35:10.715Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625820264Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19872024-02-12T01:35:10.715Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625857312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19882024-02-12T01:35:10.715Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625884041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19892024-02-12T01:35:10.715Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625918129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19902024-02-12T01:35:10.715Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625944648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19912024-02-12T01:35:10.715Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.625968457Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19922024-02-12T01:35:10.715Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:09.625995345Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19932024-02-12T01:35:10.715Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:09.626012375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19942024-02-12T01:35:10.715Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:09.626037723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19952024-02-12T01:35:10.715Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626054363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19962024-02-12T01:35:10.715Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626073242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19972024-02-12T01:35:10.715Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.62611201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19982024-02-12T01:35:10.715Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626141029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
19992024-02-12T01:35:10.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626756409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20002024-02-12T01:35:10.715Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626798127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20012024-02-12T01:35:10.715Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626896602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20022024-02-12T01:35:10.715Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.626930081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20032024-02-12T01:35:10.716Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.62737152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20042024-02-12T01:35:10.716Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.627419528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20052024-02-12T01:35:10.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.627944583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20062024-02-12T01:35:10.716Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.62800147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20072024-02-12T01:35:10.716Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.628038878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20082024-02-12T01:35:10.716Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.628062177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20092024-02-12T01:35:10.716Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.628091236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20102024-02-12T01:35:10.716Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:09.628115324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20112024-02-12T01:35:11.713Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.62758905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20122024-02-12T01:35:11.717Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.627804949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20132024-02-12T01:35:11.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.628566713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20142024-02-12T01:35:11.718Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.628611731Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20152024-02-12T01:35:11.718Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629111147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20162024-02-12T01:35:11.718Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629645422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20172024-02-12T01:35:11.718Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.62967795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20182024-02-12T01:35:11.718Z{"msg":"[0]R flush_numbers[0..12]: [5, 14, 25, 16, 21, 25, 10, 17, 15, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629694419Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20192024-02-12T01:35:11.718Z{"msg":"[0]R generation[0..12]: [5, 10, 17, 12, 15, 17, 8, 12, 11, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629709259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20202024-02-12T01:35:11.718Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629728288Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20212024-02-12T01:35:11.718Z{"msg":"[1]R flush_numbers[0..12]: [5, 14, 25, 16, 21, 25, 10, 17, 15, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629756906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20222024-02-12T01:35:11.718Z{"msg":"[1]R generation[0..12]: [5, 10, 17, 12, 15, 17, 8, 12, 11, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629771656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20232024-02-12T01:35:11.718Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629794394Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20242024-02-12T01:35:11.718Z{"msg":"[2]R flush_numbers[0..12]: [5, 14, 25, 16, 21, 25, 10, 17, 15, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629808864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20252024-02-12T01:35:11.718Z{"msg":"[2]R generation[0..12]: [5, 10, 17, 12, 15, 17, 8, 12, 11, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629829553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20262024-02-12T01:35:11.718Z{Wait for a query_work_queue command to finish before sending IO
20272024-02-12T01:35:11.718Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629847002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20282024-02-12T01:35:11.718Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629860322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20292024-02-12T01:35:11.718Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629874581Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20302024-02-12T01:35:11.718Z{"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.62988841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20312024-02-12T01:35:11.718Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.6299028Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20322024-02-12T01:35:11.718Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629916969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20332024-02-12T01:35:11.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629929458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20342024-02-12T01:35:11.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629943038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20352024-02-12T01:35:11.718Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.629974566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20362024-02-12T01:35:11.719Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.630000795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20372024-02-12T01:35:11.719Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.630036193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20382024-02-12T01:35:11.719Z{"msg":"c00cba16-da6f-49fc-a19f-a942fbf1daa6 is now active with session: cbe6438e-9600-40fc-9ae8-c528b979235e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.630073011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20392024-02-12T01:35:11.719Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.63010768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207,"session_id":"cbe6438e-9600-40fc-9ae8-c528b979235e"}
20402024-02-12T01:35:11.719Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:10.630142308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
20412024-02-12T01:35:11.719ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20422024-02-12T01:35:11.719ZLoading write count information from file "/tmp/test_repair_verify.data"
20432024-02-12T01:35:11.719ZRun Repair workload
20442024-02-12T01:35:11.722Z01/30 Write block 111 len 20480 data: 3 3 3 4 4
20452024-02-12T01:35:11.722Z02/30 Write block 255 len 36864 data: 2 2 3 3 2 3 3 3 4
20462024-02-12T01:35:11.722Z03/30 Read block 301 len 24576
20472024-02-12T01:35:11.722Z04/30 Flush
20482024-02-12T01:35:11.722Z05/30 Write block 356 len 4096 data: 4
20492024-02-12T01:35:11.725Z06/30 Write block 383 len 36864 data: 3 2 3 3 3 3 3 5 5
20502024-02-12T01:35:11.725Z07/30 Read block 583 len 4096
20512024-02-12T01:35:11.725Z08/30 Write block 80 len 40960 data: 4 4 4 3 4 3 3 5 4 4
20522024-02-12T01:35:11.726Z09/30 Write block 59 len 28672 data: 4 3 3 3 3 3 3
20532024-02-12T01:35:11.728Z10/30 Write block 142 len 24576 data: 3 3 3 3 4 3
20542024-02-12T01:35:11.729Z11/30 Read block 136 len 20480
20552024-02-12T01:35:11.729Z12/30 Write block 145 len 28672 data: 4 5 4 2 3 3 3
20562024-02-12T01:35:11.729Z13/30 Write block 315 len 36864 data: 2 2 2 3 3 3 3 3 3
20572024-02-12T01:35:11.731Z14/30 Read block 345 len 20480
20582024-02-12T01:35:11.731Z15/30 Read block 85 len 12288
20592024-02-12T01:35:11.731Z16/30 Flush
20602024-02-12T01:35:11.734Z17/30 Write block 467 len 40960 data: 4 4 4 3 3 3 3 3 3 2
20612024-02-12T01:35:11.737Z18/30 Read block 485 len 40960
20622024-02-12T01:35:11.737Z19/30 Read block 455 len 36864
20632024-02-12T01:35:11.740Z20/30 Write block 145 len 32768 data: 5 6 5 3 4 4 4 3
20642024-02-12T01:35:11.740Z21/30 Read block 417 len 36864
20652024-02-12T01:35:11.740Z22/30 Write block 411 len 24576 data: 3 2 3 3 4 3
20662024-02-12T01:35:11.743Z23/30 Read block 271 len 36864
20672024-02-12T01:35:11.743Z24/30 Read block 189 len 12288
20682024-02-12T01:35:11.744Z25/30 Write block 385 len 40960 data: 4 4 4 4 4 6 6 5 4 3
20692024-02-12T01:35:11.746Z26/30 Read block 70 len 32768
20702024-02-12T01:35:11.746Z27/30 Write block 386 len 24576 data: 5 5 5 5 7 7
20712024-02-12T01:35:11.747Z28/30 Write block 434 len 24576 data: 2 2 2 2 3 3
20722024-02-12T01:35:11.753Z29/30 Write block 167 len 36864 data: 3 3 3 3 2 2 2 2 3
20732024-02-12T01:35:11.753Z30/30 Write block 404 len 40960 data: 4 3 3 3 4 3 3 4 3 4
20742024-02-12T01:35:11.753Z----------------------------------------------------------------
20752024-02-12T01:35:11.753Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20762024-02-12T01:35:11.753ZStates: Active Active Active
20772024-02-12T01:35:11.753ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20782024-02-12T01:35:11.753Z 1 Acked 1000 Write 5 Done Done Done false
20792024-02-12T01:35:11.753Z 2 Acked 1001 Write 9 Done Done Sent false
20802024-02-12T01:35:11.753Z 3 Acked 1002 Read 6 Done Done Sent false
20812024-02-12T01:35:11.753Z 4 Acked 1003 Flush 0 Done Done Sent false
20822024-02-12T01:35:11.753Z 5 Acked 1004 Write 1 Done Done Sent false
20832024-02-12T01:35:11.753Z 6 Acked 1005 Write 9 Done Done Sent false
20842024-02-12T01:35:11.753Z 7 Acked 1006 Read 1 Done Done Sent false
20852024-02-12T01:35:11.753Z 8 Acked 1007 Write 10 Done Done Sent false
20862024-02-12T01:35:11.753Z 9 Acked 1008 Write 7 Done Done Sent false
20872024-02-12T01:35:11.754Z 10 Acked 1009 Write 6 Done Done Sent false
20882024-02-12T01:35:11.754Z 11 Acked 1010 Read 5 Done Done Sent false
20892024-02-12T01:35:11.754Z 12 Acked 1011 Write 7 Done Done Sent false
20902024-02-12T01:35:11.754Z 13 Acked 1012 Write 9 Done Done Sent false
20912024-02-12T01:35:11.754Z 14 Acked 1013 Read 5 Done Done Sent false
20922024-02-12T01:35:11.754Z 15 Acked 1014 Read 3 Done Done Sent false
20932024-02-12T01:35:11.754Z 16 Acked 1015 Flush 0 Done Done Sent false
20942024-02-12T01:35:11.754Z 17 Acked 1016 Write 10 Done Done Sent false
20952024-02-12T01:35:11.754Z 18 Acked 1017 Read 10 Done Done Sent false
20962024-02-12T01:35:11.754Z 19 Acked 1018 Read 9 Done Done Sent false
20972024-02-12T01:35:11.754Z 20 Acked 1019 Write 8 Done Done Sent false
20982024-02-12T01:35:11.754Z 21 Acked 1020 Read 9 Done Done Sent false
20992024-02-12T01:35:11.754Z 22 Acked 1021 Write 6 Done Done Sent false
21002024-02-12T01:35:11.754Z 23 Acked 1022 Read 9 Done Done Sent false
21012024-02-12T01:35:11.754Z 24 Acked 1023 Read 3 Done Done Sent false
21022024-02-12T01:35:11.754Z 25 Acked 1024 Write 10 Done Done Sent false
21032024-02-12T01:35:11.754Z 26 Acked 1025 Read 8 Done Done Sent false
21042024-02-12T01:35:11.754Z 27 Acked 1026 Write 6 Done Done Sent false
21052024-02-12T01:35:11.754Z 28 Acked 1027 Write 6 Done Done Sent false
21062024-02-12T01:35:11.754Z 29 Acked 1028 Write 9 Done Done Sent false
21072024-02-12T01:35:11.754Z 30 Acked 1029 Write 10 Sent Sent Sent false
21082024-02-12T01:35:11.754Z STATES DS:0 DS:1 DS:2 TOTAL
21092024-02-12T01:35:11.755Z New 0 0 0 0
21102024-02-12T01:35:11.755Z Sent 1 1 29 31
21112024-02-12T01:35:11.755Z Done 29 29 1 59
21122024-02-12T01:35:11.755Z Skipped 0 0 0 0
21132024-02-12T01:35:11.755Z Error 0 0 0 0
21142024-02-12T01:35:11.755ZLast Flush: 1015 1015 0
21152024-02-12T01:35:11.755ZDownstairs last five completed:
21162024-02-12T01:35:11.755ZUpstairs last five completed: 30 29 28 27 26
21172024-02-12T01:35:11.755Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:10.664246041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1207}
21182024-02-12T01:35:11.755ZWrote out file "/tmp/test_repair_verify.data"
21192024-02-12T01:35:11.758Z
21202024-02-12T01:35:11.758ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21212024-02-12T01:35:11.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21222024-02-12T01:35:11.774Z 0 000-019 5 5 5 5 5 5 F F F
21232024-02-12T01:35:11.774Z 1 020-039 10 10 10 14 14 14 F F F
21242024-02-12T01:35:11.774Z 2 040-059 18 18 17 27 27 25 F F F <---
21252024-02-12T01:35:11.774Z 3 060-079 18 18 12 27 27 16 F F F <---
21262024-02-12T01:35:11.774Z 4 080-099 18 18 15 27 27 21 F F F <---
21272024-02-12T01:35:11.775Z 5 100-119 18 18 17 26 26 25 F F T <---
21282024-02-12T01:35:11.775Z 6 120-139 8 8 8 10 10 10 F F F
21292024-02-12T01:35:11.775Z 7 140-159 18 18 12 27 27 17 T T F <---
21302024-02-12T01:35:11.775Z 8 160-179 11 11 11 15 15 15 T T F <---
21312024-02-12T01:35:11.775Z 9 180-199 15 15 15 21 21 21 F F F
21322024-02-12T01:35:11.775Z 10 200-219 14 14 14 19 19 19 F F F
21332024-02-12T01:35:11.778Z 11 220-239 16 16 16 24 24 24 F F F
21342024-02-12T01:35:11.778Z 12 240-259 18 18 17 26 26 25 F F F <---
21352024-02-12T01:35:11.778Z 13 260-279 18 18 17 26 26 25 F F F <---
21362024-02-12T01:35:11.778Z 14 280-299 16 16 16 22 22 22 F F F
21372024-02-12T01:35:11.778Z 15 300-319 18 18 13 27 27 18 F F F <---
21382024-02-12T01:35:11.778Z 16 320-339 18 18 13 27 27 18 F F F <---
21392024-02-12T01:35:11.778Z 17 340-359 18 18 17 27 27 25 F F F <---
21402024-02-12T01:35:11.778Z 18 360-379 16 16 16 22 22 22 F F F
21412024-02-12T01:35:11.778Z 19 380-399 18 18 12 27 27 17 T T F <---
21422024-02-12T01:35:11.778Z 20 400-419 17 17 17 25 25 25 T T F <---
21432024-02-12T01:35:11.778Z 21 420-439 13 13 13 18 18 18 T T F <---
21442024-02-12T01:35:11.778Z 22 440-459 13 13 13 18 18 18 F F F
21452024-02-12T01:35:11.778Z 23 460-479 15 15 15 21 21 21 T T F <---
21462024-02-12T01:35:11.778Z 24 480-499 15 15 15 21 21 21 F F F
21472024-02-12T01:35:11.779Z 25 500-519 17 17 17 25 25 25 F F F
21482024-02-12T01:35:11.779Z 26 520-539 9 9 9 13 13 13 F F F
21492024-02-12T01:35:11.779Z 27 540-559 15 15 15 21 21 21 F F F
21502024-02-12T01:35:11.779Z 28 560-579 17 17 17 25 25 25 F F F
21512024-02-12T01:35:11.779Z 29 580-599 17 17 17 25 25 25 F F F
21522024-02-12T01:35:11.779ZMax gen: 18, Max flush: 27
21532024-02-12T01:35:11.779ZError: Difference in extent metadata found!
21542024-02-12T01:35:11.779ZA Difference in extent metadata is expected here
21552024-02-12T01:35:11.779ZOn loop 9
21562024-02-12T01:35:11.779Z
21572024-02-12T01:35:11.782ZVerifying data now
21582024-02-12T01:35:18.346ZNo change after verify
21592024-02-12T01:35:18.349ZLoop: 9 Downstairs dump after verify (and repair):
21602024-02-12T01:35:18.361ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21612024-02-12T01:35:18.361Z 0 000-019 5 5 5 5 5 5 F F F
21622024-02-12T01:35:18.361Z 1 020-039 10 10 10 14 14 14 F F F
21632024-02-12T01:35:18.361Z 2 040-059 18 18 18 27 27 27 F F F
21642024-02-12T01:35:18.361Z 3 060-079 18 18 18 27 27 27 F F F
21652024-02-12T01:35:18.361Z 4 080-099 18 18 18 27 27 27 F F F
21662024-02-12T01:35:18.361Z 5 100-119 18 18 18 26 26 26 F F F
21672024-02-12T01:35:18.361Z 6 120-139 8 8 8 10 10 10 F F F
21682024-02-12T01:35:18.361Z 7 140-159 19 19 19 28 28 28 F F F
21692024-02-12T01:35:18.361Z 8 160-179 19 19 19 28 28 28 F F F
21702024-02-12T01:35:18.361Z 9 180-199 15 15 15 21 21 21 F F F
21712024-02-12T01:35:18.361Z 10 200-219 14 14 14 19 19 19 F F F
21722024-02-12T01:35:18.361Z 11 220-239 16 16 16 24 24 24 F F F
21732024-02-12T01:35:18.365Z 12 240-259 18 18 18 26 26 26 F F F
21742024-02-12T01:35:18.365Z 13 260-279 18 18 18 26 26 26 F F F
21752024-02-12T01:35:18.365Z 14 280-299 16 16 16 22 22 22 F F F
21762024-02-12T01:35:18.365Z 15 300-319 18 18 18 27 27 27 F F F
21772024-02-12T01:35:18.365Z 16 320-339 18 18 18 27 27 27 F F F
21782024-02-12T01:35:18.365Z 17 340-359 18 18 18 27 27 27 F F F
21792024-02-12T01:35:18.365Z 18 360-379 16 16 16 22 22 22 F F F
21802024-02-12T01:35:18.365Z 19 380-399 19 19 19 28 28 28 F F F
21812024-02-12T01:35:18.365Z 20 400-419 19 19 19 28 28 28 F F F
21822024-02-12T01:35:18.365Z 21 420-439 19 19 19 28 28 28 F F F
21832024-02-12T01:35:18.365Z 22 440-459 13 13 13 18 18 18 F F F
21842024-02-12T01:35:18.365Z 23 460-479 19 19 19 28 28 28 F F F
21852024-02-12T01:35:18.365Z 24 480-499 15 15 15 21 21 21 F F F
21862024-02-12T01:35:18.365Z 25 500-519 17 17 17 25 25 25 F F F
21872024-02-12T01:35:18.365Z 26 520-539 9 9 9 13 13 13 F F F
21882024-02-12T01:35:18.365Z 27 540-559 15 15 15 21 21 21 F F F
21892024-02-12T01:35:18.365Z 28 560-579 17 17 17 25 25 25 F F F
21902024-02-12T01:35:18.365Z 29 580-599 17 17 17 25 25 25 F F F
21912024-02-12T01:35:18.365ZMax gen: 19, Max flush: 28
21922024-02-12T01:35:18.365Z
21932024-02-12T01:35:18.365ZBegin loop 10
21942024-02-12T01:35:18.365ZDownstairs to restart: 2
21952024-02-12T01:35:18.383Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.297705674Z"Crucible runtime is spawned
21962024-02-12T01:35:18.383Z,"hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
21972024-02-12T01:35:18.388Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298120575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
21982024-02-12T01:35:18.388Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298172462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
21992024-02-12T01:35:18.388Z{"msg":"Crucible 5d650e17-535d-422c-a511-9ad90c32d534 has session id: 80b11a55-03ae-4dbf-8f94-a3ced875daf3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298202241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22002024-02-12T01:35:18.388Z{"msg":"Upstairs opts: Upstairs UUID: 5d650e17-535d-422c-a511-9ad90c32d534, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298257518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22012024-02-12T01:35:18.389Z{"msg":"Crucible stats registered with UUID: 5d650e17-535d-422c-a511-9ad90c32d534","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298300366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22022024-02-12T01:35:18.389Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298341034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
22032024-02-12T01:35:18.389Z{"msg":"5d650e17-535d-422c-a511-9ad90c32d534 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298381112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22042024-02-12T01:35:18.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.29842148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22052024-02-12T01:35:18.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298478037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22062024-02-12T01:35:18.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298519546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22072024-02-12T01:35:18.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298541945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22082024-02-12T01:35:18.389Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298566243Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22092024-02-12T01:35:18.389Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298589552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22102024-02-12T01:35:18.389Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298614901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22112024-02-12T01:35:18.389Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298657149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22122024-02-12T01:35:18.389Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298754794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22132024-02-12T01:35:18.389Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298780543Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22142024-02-12T01:35:18.389Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.298812932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22152024-02-12T01:35:18.389Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:17.299017532Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22162024-02-12T01:35:18.389Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:17.29906718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22172024-02-12T01:35:18.389Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:17.299105898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22182024-02-12T01:35:18.389Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299127577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22192024-02-12T01:35:18.389Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299160955Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22202024-02-12T01:35:18.389Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299184904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22212024-02-12T01:35:18.389Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299451231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22222024-02-12T01:35:18.390Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299493489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22232024-02-12T01:35:18.390Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299517278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22242024-02-12T01:35:18.390Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.299548926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22252024-02-12T01:35:18.390Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300055372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22262024-02-12T01:35:18.390Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.30009234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22272024-02-12T01:35:18.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300581247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22282024-02-12T01:35:18.390Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300621875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22292024-02-12T01:35:18.390Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300665843Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22302024-02-12T01:35:18.390Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300706031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22312024-02-12T01:35:18.390Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300755569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22322024-02-12T01:35:18.390Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:17.300794147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22332024-02-12T01:35:19.389Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.300424946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22342024-02-12T01:35:19.389Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.300496302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22352024-02-12T01:35:19.396Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.301315643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22362024-02-12T01:35:19.396Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.301357921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22372024-02-12T01:35:19.396Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.30180896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22382024-02-12T01:35:19.396Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302375483Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22392024-02-12T01:35:19.396Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302415001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22402024-02-12T01:35:19.396Z{"msg":"[0]R flush_numbers[0..12]: [5, 14, 27, 27, 27, 26, 10, 28, 28, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302433Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22412024-02-12T01:35:19.396Z{"msg":"[0]R generation[0..12]: [5, 10, 18, 18, 18, 18, 8, 19, 19, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302448619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22422024-02-12T01:35:19.396Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302463618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22432024-02-12T01:35:19.396Z{"msg":"[1]R flush_numbers[0..12]: [5, 14, 27, 27, 27, 26, 10, 28, 28, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302487537Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22442024-02-12T01:35:19.397Z{"msg":"[1]R generation[0..12]: [5, 10, 18, 18, 18, 18, 8, 19, 19, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302502887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22452024-02-12T01:35:19.397Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302517526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22462024-02-12T01:35:19.397Z{"msg":"[2]R flush_numbers[0..12]: [5, 14, 27, 27, 27, 26, 10, 28, 28, 21, 19, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302539865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22472024-02-12T01:35:19.397Z{"msg":"[2]R generation[0..12]: [5, 10, 18, 18, 18, 18, 8, 19, 19, 15, 14, 16]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22482024-02-12T01:35:19.397Z,"time":"2024-02-12T01:35:18.302553944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22492024-02-12T01:35:19.397Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302570773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22502024-02-12T01:35:19.397Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302593492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22512024-02-12T01:35:19.397Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302608291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22522024-02-12T01:35:19.397Z{"msg":"Next flush: 29","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302621941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22532024-02-12T01:35:19.397Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.3026355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22542024-02-12T01:35:19.397Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302657099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22552024-02-12T01:35:19.397Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302671768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22562024-02-12T01:35:19.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302686488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22572024-02-12T01:35:19.398Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302699877Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22582024-02-12T01:35:19.398Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302721346Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22592024-02-12T01:35:19.398Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302734405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22602024-02-12T01:35:19.398Z{"msg":"5d650e17-535d-422c-a511-9ad90c32d534 is now active with session: 80b11a55-03ae-4dbf-8f94-a3ced875daf3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302748385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22612024-02-12T01:35:19.398Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302767734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215,"session_id":"80b11a55-03ae-4dbf-8f94-a3ced875daf3"}
22622024-02-12T01:35:19.398Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:18.302781153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
22632024-02-12T01:35:19.398ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22642024-02-12T01:35:19.398ZLoading write count information from file "/tmp/test_repair_verify.data"
22652024-02-12T01:35:19.398ZRun Repair workload
22662024-02-12T01:35:19.398Z01/30 Write block 465 len 32768 data: 3 4 5 5 5 4 4 4
22672024-02-12T01:35:19.398Z02/30 Read block 555 len 36864
22682024-02-12T01:35:19.398Z03/30 Read block 91 len 24576
22692024-02-12T01:35:19.402Z04/30 Write block 62 len 32768 data: 4 4 4 4 3 2 2 2
22702024-02-12T01:35:19.402Z05/30 Read block 244 len 40960
22712024-02-12T01:35:19.402Z06/30 Read block 194 len 24576
22722024-02-12T01:35:19.402Z07/30 Write block 71 len 28672 data: 2 2 2 2 2 3 4
22732024-02-12T01:35:19.402Z08/30 Read block 399 len 20480
22742024-02-12T01:35:19.402Z09/30 Write block 32 len 32768 data: 5 5 5 2 2 2 2 2
22752024-02-12T01:35:19.403Z10/30 Read block 189 len 20480
22762024-02-12T01:35:19.403Z11/30 Read block 205 len 12288
22772024-02-12T01:35:19.406Z12/30 Write block 76 len 24576 data: 4 5 4 4 5 5
22782024-02-12T01:35:19.406Z13/30 Read block 6 len 16384
22792024-02-12T01:35:19.406Z14/30 Read block 86 len 40960
22802024-02-12T01:35:19.406Z15/30 Write block 352 len 24576 data: 5 4 4 5 5 3
22812024-02-12T01:35:19.406Z16/30 Write block 303 len 36864 data: 3 3 2 3 3 3 3 3 3
22822024-02-12T01:35:19.409Z17/30 Read block 4 len 36864
22832024-02-12T01:35:19.409Z18/30 Write block 207 len 12288 data: 4 4 4
22842024-02-12T01:35:19.410Z19/30 Write block 578 len 4096 data: 5
22852024-02-12T01:35:19.410Z20/30 Read block 300 len 20480
22862024-02-12T01:35:19.413Z21/30 Write block 364 len 40960 data: 2 3 3 4 3 3 3 3 3 3
22872024-02-12T01:35:19.413Z22/30 Read block 25 len 40960
22882024-02-12T01:35:19.413Z23/30 Flush
22892024-02-12T01:35:19.416Z24/30 Write block 361 len 40960 data: 2 2 2 3 4 4 5 4 4 4
22902024-02-12T01:35:19.416Z25/30 Read block 418 len 40960
22912024-02-12T01:35:19.419Z26/30 Flush
22922024-02-12T01:35:19.445Z27/30 Write block 439 len 12288 data: 4 3 3
22932024-02-12T01:35:19.448Z28/30 Read block 544 len 32768
22942024-02-12T01:35:19.448Z29/30 Write block 321 len 8192 data: 4 4
22952024-02-12T01:35:19.448Z30/30 Write block 472 len 12288 data: 5 4 4
22962024-02-12T01:35:19.449Z----------------------------------------------------------------
22972024-02-12T01:35:19.449Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22982024-02-12T01:35:19.449ZStates: Active Active Active
22992024-02-12T01:35:19.449ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
23002024-02-12T01:35:19.449Z 1 Acked 1000 Write 8 Done Done Done false
23012024-02-12T01:35:19.449Z 2 Acked 1001 Read 9 Done Done Done false
23022024-02-12T01:35:19.449Z 3 Acked 1002 Read 6 Done Done Done false
23032024-02-12T01:35:19.449Z 4 Acked 1003 Write 8 Done Done Done false
23042024-02-12T01:35:19.449Z 5 Acked 1004 Read 10 Done Done Done false
23052024-02-12T01:35:19.449Z 6 Acked 1005 Read 6 Done Done Sent false
23062024-02-12T01:35:19.449Z 7 Acked 1006 Write 7 Done Done Sent false
23072024-02-12T01:35:19.449Z 8 Acked 1007 Read 5 Done Done Sent false
23082024-02-12T01:35:19.450Z 9 Acked 1008 Write 8 Done Done Sent false
23092024-02-12T01:35:19.450Z 10 Acked 1009 Read 5 Done Done Sent false
23102024-02-12T01:35:19.450Z 11 Acked 1010 Read 3 Done Done Sent false
23112024-02-12T01:35:19.450Z 12 Acked 1011 Write 6 Done Done Sent false
23122024-02-12T01:35:19.450Z 13 Acked 1012 Read 4 Done Done Sent false
23132024-02-12T01:35:19.450Z 14 Acked 1013 Read 10 Done Done Sent false
23142024-02-12T01:35:19.450Z 15 Acked 1014 Write 6 Done Done Sent false
23152024-02-12T01:35:19.450Z 16 Acked 1015 Write 9 Done Done Sent false
23162024-02-12T01:35:19.450Z 17 Acked 1016 Read 9 Done Done Sent false
23172024-02-12T01:35:19.450Z 18 Acked 1017 Write 3 Done Done Sent false
23182024-02-12T01:35:19.450Z 19 Acked 1018 Write 1 Done Done Sent false
23192024-02-12T01:35:19.450Z 20 Acked 1019 Read 5 Done Done Sent false
23202024-02-12T01:35:19.450Z 21 Acked 1020 Write 10 Done Done Sent false
23212024-02-12T01:35:19.450Z 22 Acked 1021 Read 10 Done Done Sent false
23222024-02-12T01:35:19.451Z 23 Acked 1022 Flush 0 Done Done Sent false
23232024-02-12T01:35:19.451Z 24 Acked 1023 Write 10 Done Done Sent false
23242024-02-12T01:35:19.451Z 25 Acked 1024 Read 10 Done Done Sent false
23252024-02-12T01:35:19.451Z 26 Acked 1025 Flush 0 Done Done Sent false
23262024-02-12T01:35:19.451Z 27 Acked 1026 Write 3 Done Done Sent false
23272024-02-12T01:35:19.451Z 28 Acked 1027 Read 8 Done Done Sent false
23282024-02-12T01:35:19.451Z 29 Acked 1028 Write 2 Done Done Sent false
23292024-02-12T01:35:19.451Z 30 Acked 1029 Write 3 Sent Sent Sent false
23302024-02-12T01:35:19.451Z STATES DS:0 DS:1 DS:2 TOTAL
23312024-02-12T01:35:19.451Z New 0 0 0 0
23322024-02-12T01:35:19.451Z Sent 1 1 25 27
23332024-02-12T01:35:19.451Z Done 29 29 5 63
23342024-02-12T01:35:19.451Z Skipped 0 0 0 0
23352024-02-12T01:35:19.452Z Error 0 0 0 0
23362024-02-12T01:35:19.452ZLast Flush: 1025 1025 0
23372024-02-12T01:35:19.452ZDownstairs last five completed:
23382024-02-12T01:35:19.452ZUpstairs last five completed: 30 29 28 27 26
23392024-02-12T01:35:19.452Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:18.362075774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1215}
23402024-02-12T01:35:19.452ZWrote out file "/tmp/test_repair_verify.data"
23412024-02-12T01:35:19.456Z
23422024-02-12T01:35:19.471ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23432024-02-12T01:35:19.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23442024-02-12T01:35:19.477Z 0 000-019 5 5 5 5 5 5 F F F
23452024-02-12T01:35:19.477Z 1 020-039 20 20 10 29 29 14 F F F <---
23462024-02-12T01:35:19.477Z 2 040-059 18 18 18 27 27 27 F F F
23472024-02-12T01:35:19.477Z 3 060-079 20 20 18 29 29 27 F F T <---
23482024-02-12T01:35:19.477Z 4 080-099 20 20 18 29 29 27 F F F <---
23492024-02-12T01:35:19.477Z 5 100-119 18 18 18 26 26 26 F F F
23502024-02-12T01:35:19.477Z 6 120-139 8 8 8 10 10 10 F F F
23512024-02-12T01:35:19.477Z 7 140-159 19 19 19 28 28 28 F F F
23522024-02-12T01:35:19.477Z 8 160-179 19 19 19 28 28 28 F F F
23532024-02-12T01:35:19.477Z 9 180-199 15 15 15 21 21 21 F F F
23542024-02-12T01:35:19.477Z 10 200-219 20 20 14 29 29 19 F F F <---
23552024-02-12T01:35:19.477Z 11 220-239 16 16 16 24 24 24 F F F
23562024-02-12T01:35:19.478Z 12 240-259 18 18 18 26 26 26 F F F
23572024-02-12T01:35:19.478Z 13 260-279 18 18 18 26 26 26 F F F
23582024-02-12T01:35:19.478Z 14 280-299 16 16 16 22 22 22 F F F
23592024-02-12T01:35:19.478Z 15 300-319 20 20 18 29 29 27 F F F <---
23602024-02-12T01:35:19.478Z 16 320-339 18 18 18 27 27 27 T T F <---
23612024-02-12T01:35:19.478Z 17 340-359 20 20 18 29 29 27 F F F <---
23622024-02-12T01:35:19.478Z 18 360-379 20 20 16 30 30 22 F F F <---
23632024-02-12T01:35:19.478Z 19 380-399 19 19 19 28 28 28 F F F
23642024-02-12T01:35:19.478Z 20 400-419 19 19 19 28 28 28 F F F
23652024-02-12T01:35:19.478Z 21 420-439 19 19 19 28 28 28 T T F <---
23662024-02-12T01:35:19.478Z 22 440-459 13 13 13 18 18 18 T T F <---
23672024-02-12T01:35:19.478Z 23 460-479 20 20 19 29 29 28 T T T <---
23682024-02-12T01:35:19.478Z 24 480-499 15 15 15 21 21 21 F F F
23692024-02-12T01:35:19.478Z 25 500-519 17 17 17 25 25 25 F F F
23702024-02-12T01:35:19.478Z 26 520-539 9 9 9 13 13 13 F F F
23712024-02-12T01:35:19.478Z 27 540-559 15 15 15 21 21 21 F F F
23722024-02-12T01:35:19.479Z 28 560-579 20 20 17 29 29 25 F F F <---
23732024-02-12T01:35:19.479Z 29 580-599 17 17 17 25 25 25 F F F
23742024-02-12T01:35:19.479ZMax gen: 20, Max flush: 30
23752024-02-12T01:35:19.479ZError: Difference in extent metadata found!
23762024-02-12T01:35:19.482ZA Difference in extent metadata is expected here
23772024-02-12T01:35:19.482ZOn loop 10
23782024-02-12T01:35:19.482Z
23792024-02-12T01:35:19.482ZVerifying data now
23802024-02-12T01:35:25.660ZNo change after verify
23812024-02-12T01:35:25.660ZLoop: 10 Downstairs dump after verify (and repair):
23822024-02-12T01:35:25.675ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23832024-02-12T01:35:25.675Z 0 000-019 5 5 5 5 5 5 F F F
23842024-02-12T01:35:25.675Z 1 020-039 20 20 20 29 29 29 F F F
23852024-02-12T01:35:25.675Z 2 040-059 18 18 18 27 27 27 F F F
23862024-02-12T01:35:25.675Z 3 060-079 20 20 20 29 29 29 F F F
23872024-02-12T01:35:25.676Z 4 080-099 20 20 20 29 29 29 F F F
23882024-02-12T01:35:25.676Z 5 100-119 18 18 18 26 26 26 F F F
23892024-02-12T01:35:25.676Z 6 120-139 8 8 8 10 10 10 F F F
23902024-02-12T01:35:25.676Z 7 140-159 19 19 19 28 28 28 F F F
23912024-02-12T01:35:25.676Z 8 160-179 19 19 19 28 28 28 F F F
23922024-02-12T01:35:25.676Z 9 180-199 15 15 15 21 21 21 F F F
23932024-02-12T01:35:25.676Z 10 200-219 20 20 20 29 29 29 F F F
23942024-02-12T01:35:25.679Z 11 220-239 16 16 16 24 24 24 F F F
23952024-02-12T01:35:25.679Z 12 240-259 18 18 18 26 26 26 F F F
23962024-02-12T01:35:25.679Z 13 260-279 18 18 18 26 26 26 F F F
23972024-02-12T01:35:25.679Z 14 280-299 16 16 16 22 22 22 F F F
23982024-02-12T01:35:25.679Z 15 300-319 20 20 20 29 29 29 F F F
23992024-02-12T01:35:25.679Z 16 320-339 21 21 21 31 31 31 F F F
24002024-02-12T01:35:25.679Z 17 340-359 20 20 20 29 29 29 F F F
24012024-02-12T01:35:25.679Z 18 360-379 20 20 20 30 30 30 F F F
24022024-02-12T01:35:25.679Z 19 380-399 19 19 19 28 28 28 F F F
24032024-02-12T01:35:25.679Z 20 400-419 19 19 19 28 28 28 F F F
24042024-02-12T01:35:25.679Z 21 420-439 21 21 21 31 31 31 F F F
24052024-02-12T01:35:25.679Z 22 440-459 21 21 21 31 31 31 F F F
24062024-02-12T01:35:25.679Z 23 460-479 21 21 21 31 31 31 F F F
24072024-02-12T01:35:25.679Z 24 480-499 15 15 15 21 21 21 F F F
24082024-02-12T01:35:25.679Z 25 500-519 17 17 17 25 25 25 F F F
24092024-02-12T01:35:25.679Z 26 520-539 9 9 9 13 13 13 F F F
24102024-02-12T01:35:25.679Z 27 540-559 15 15 15 21 21 21 F F F
24112024-02-12T01:35:25.680Z 28 560-579 20 20 20 29 29 29 F F F
24122024-02-12T01:35:25.680Z 29 580-599 17 17 17 25 25 25 F F F
24132024-02-12T01:35:25.680ZMax gen: 21, Max flush: 31
24142024-02-12T01:35:25.683Z
24152024-02-12T01:35:25.683ZBegin loop 11
24162024-02-12T01:35:25.683ZDownstairs to restart: 1
24172024-02-12T01:35:25.700Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.612612666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
24182024-02-12T01:35:25.700ZCrucible runtime is spawned
24192024-02-12T01:35:25.705Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.61294958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
24202024-02-12T01:35:25.705Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.612992378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
24212024-02-12T01:35:25.705Z{"msg":"Crucible d4c42d16-c5eb-4ba8-a9f3-7216148d69b0 has session id: 06e0fbdd-744e-42b3-9b41-ecc0d341f851","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613061185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24222024-02-12T01:35:25.705Z{"msg":"Upstairs opts: Upstairs UUID: d4c42d16-c5eb-4ba8-a9f3-7216148d69b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613112862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24232024-02-12T01:35:25.705Z{"msg":"Crucible stats registered with UUID: d4c42d16-c5eb-4ba8-a9f3-7216148d69b0","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.61315187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24242024-02-12T01:35:25.705Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613204708Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
24252024-02-12T01:35:25.705Z{"msg":"d4c42d16-c5eb-4ba8-a9f3-7216148d69b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613247746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24262024-02-12T01:35:25.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613273734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24272024-02-12T01:35:25.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613311973Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24282024-02-12T01:35:25.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613346201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24292024-02-12T01:35:25.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613385729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24302024-02-12T01:35:25.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613413568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24312024-02-12T01:35:25.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613455646Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24322024-02-12T01:35:25.705Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613491454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24332024-02-12T01:35:25.705Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613534892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24342024-02-12T01:35:25.705Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613564241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24352024-02-12T01:35:25.705Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:24.613605359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24362024-02-12T01:35:25.705Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:24.613636927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24372024-02-12T01:35:25.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613674755Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24382024-02-12T01:35:25.705Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:24.613705984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24392024-02-12T01:35:25.705Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613728763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24402024-02-12T01:35:25.706Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613772991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24412024-02-12T01:35:25.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613817948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24422024-02-12T01:35:25.706Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.613911604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24432024-02-12T01:35:25.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.614386061Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24442024-02-12T01:35:25.706Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.61442385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24452024-02-12T01:35:25.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.614442689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24462024-02-12T01:35:25.706Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.614477937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24472024-02-12T01:35:25.706Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.614948035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24482024-02-12T01:35:25.706Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615119837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24492024-02-12T01:35:25.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615417522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24502024-02-12T01:35:25.706Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.61546218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24512024-02-12T01:35:25.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615490229Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24522024-02-12T01:35:25.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615570745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24532024-02-12T01:35:25.706Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615600183Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24542024-02-12T01:35:25.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:24.615615243Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24552024-02-12T01:35:26.702Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.615149626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24562024-02-12T01:35:26.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.615243352Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24572024-02-12T01:35:26.708Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.616377118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24582024-02-12T01:35:26.708Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.616415596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24592024-02-12T01:35:26.708Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.616936841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24602024-02-12T01:35:26.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617498284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24612024-02-12T01:35:26.708Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617548502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24622024-02-12T01:35:26.708Z{"msg":"[0]R flush_numbers[0..12]: [5, 29, 27, 29, 29, 26, 10, 28, 28, 21, 29, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.61758764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24632024-02-12T01:35:26.708Z{"msg":"[0]R generation[0..12]: [5, 20, 18, 20, 20, 18, 8, 19, 19, 15, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617619889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24642024-02-12T01:35:26.708Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617643807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24652024-02-12T01:35:26.708Z{"msg":"[1]R flush_numbers[0..12]: [5, 29, 27, 29, 29, 26, 10, 28, 28, 21, 29, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617659107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24662024-02-12T01:35:26.708Z{"msg":"[1]R generation[0..12]: [5, 20, 18, 20, 20, 18, 8, 19, 19, 15, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617679926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24672024-02-12T01:35:26.708Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
24682024-02-12T01:35:26.708Z,"time":"2024-02-12T01:35:25.617694435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24692024-02-12T01:35:26.708Z{"msg":"[2]R flush_numbers[0..12]: [5, 29, 27, 29, 29, 26, 10, 28, 28, 21, 29, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617724384Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24702024-02-12T01:35:26.708Z{"msg":"[2]R generation[0..12]: [5, 20, 18, 20, 20, 18, 8, 19, 19, 15, 20, 16]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617744752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24712024-02-12T01:35:26.708Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617759742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24722024-02-12T01:35:26.708Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617781531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24732024-02-12T01:35:26.708Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.61779589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24742024-02-12T01:35:26.708Z{"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617820669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24752024-02-12T01:35:26.708Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617834448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24762024-02-12T01:35:26.708Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617856997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24772024-02-12T01:35:26.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617870086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24782024-02-12T01:35:26.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617883686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24792024-02-12T01:35:26.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617903915Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24802024-02-12T01:35:26.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617918634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24812024-02-12T01:35:26.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617931963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24822024-02-12T01:35:26.709Z{"msg":"d4c42d16-c5eb-4ba8-a9f3-7216148d69b0 is now active with session: 06e0fbdd-744e-42b3-9b41-ecc0d341f851","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617952162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24832024-02-12T01:35:26.709Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617964252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223,"session_id":"06e0fbdd-744e-42b3-9b41-ecc0d341f851"}
24842024-02-12T01:35:26.709Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:25.617977051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
24852024-02-12T01:35:26.709ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24862024-02-12T01:35:26.709ZLoading write count information from file "/tmp/test_repair_verify.data"
24872024-02-12T01:35:26.709ZRun Repair workload
24882024-02-12T01:35:26.709Z01/30 Write block 118 len 40960 data: 5 5 4 4 3 2 2 2 4 4
24892024-02-12T01:35:26.713Z02/30 Flush
24902024-02-12T01:35:26.713Z03/30 Write block 5 len 40960 data: 2 2 2 2 2 2 2 2 2 3
24912024-02-12T01:35:26.713Z04/30 Read block 61 len 24576
24922024-02-12T01:35:26.713Z05/30 Write block 308 len 4096 data: 4
24932024-02-12T01:35:26.716Z06/30 Write block 131 len 16384 data: 3 3 2 2
24942024-02-12T01:35:26.716Z07/30 Read block 103 len 4096
24952024-02-12T01:35:26.716Z08/30 Write block 58 len 4096 data: 4
24962024-02-12T01:35:26.716Z09/30 Read block 28 len 24576
24972024-02-12T01:35:26.716Z10/30 Read block 537 len 28672
24982024-02-12T01:35:26.716Z11/30 Read block 433 len 4096
24992024-02-12T01:35:26.719Z12/30 Flush
25002024-02-12T01:35:26.719Z13/30 Write block 75 len 24576 data: 3 5 6 5 5 6
25012024-02-12T01:35:26.719Z14/30 Read block 9 len 24576
25022024-02-12T01:35:26.722Z15/30 Read block 428 len 12288
25032024-02-12T01:35:26.722Z16/30 Write block 114 len 4096 data: 5
25042024-02-12T01:35:26.722Z17/30 Flush
25052024-02-12T01:35:26.757Z18/30 Write block 164 len 32768 data: 2 3 3 4 4 4 4 3
25062024-02-12T01:35:26.761Z19/30 Read block 75 len 8192
25072024-02-12T01:35:26.761Z20/30 Read block 462 len 28672
25082024-02-12T01:35:26.761Z21/30 Write block 365 len 12288 data: 5 5 6
25092024-02-12T01:35:26.761Z22/30 Read block 138 len 8192
25102024-02-12T01:35:26.761Z23/30 Read block 595 len 16384
25112024-02-12T01:35:26.765Z24/30 Write block 301 len 32768 data: 3 2 4 4 3 4 4 5
25122024-02-12T01:35:26.765Z25/30 Read block 454 len 16384
25132024-02-12T01:35:26.765Z26/30 Flush
25142024-02-12T01:35:26.769Z27/30 Write block 15 len 4096 data: 3
25152024-02-12T01:35:26.769Z28/30 Write block 278 len 12288 data: 3 3 3
25162024-02-12T01:35:26.769Z29/30 Read block 39 len 8192
25172024-02-12T01:35:26.774Z30/30 Write block 219 len 8192 data: 2 2
25182024-02-12T01:35:26.774Z----------------------------------------------------------------
25192024-02-12T01:35:26.774Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:28
25202024-02-12T01:35:26.774ZStates: Active Active Active
25212024-02-12T01:35:26.774ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25222024-02-12T01:35:26.774Z 3 Acked 1002 Write 10 Done Sent Done false
25232024-02-12T01:35:26.774Z 4 Acked 1003 Read 6 Done Sent Done false
25242024-02-12T01:35:26.774Z 5 Acked 1004 Write 1 Done Sent Done false
25252024-02-12T01:35:26.774Z 6 Acked 1005 Write 4 Done Sent Done false
25262024-02-12T01:35:26.774Z 7 Acked 1006 Read 1 Done Sent Done false
25272024-02-12T01:35:26.775Z 8 Acked 1007 Write 1 Done Sent Done false
25282024-02-12T01:35:26.775Z 9 Acked 1008 Read 6 Done Sent Done false
25292024-02-12T01:35:26.775Z 10 Acked 1009 Read 7 Done Sent Done false
25302024-02-12T01:35:26.775Z 11 Acked 1010 Read 1 Done Sent Done false
25312024-02-12T01:35:26.775Z 12 Acked 1011 Flush 0 Done Sent Done false
25322024-02-12T01:35:26.775Z 13 Acked 1012 Write 6 Done Sent Done false
25332024-02-12T01:35:26.775Z 14 Acked 1013 Read 6 Done Sent Done false
25342024-02-12T01:35:26.775Z 15 Acked 1014 Read 3 Done Sent Done false
25352024-02-12T01:35:26.775Z 16 Acked 1015 Write 1 Done Sent Done false
25362024-02-12T01:35:26.775Z 17 Acked 1016 Flush 0 Done Sent Done false
25372024-02-12T01:35:26.775Z 18 Acked 1017 Write 8 Done Sent Done false
25382024-02-12T01:35:26.775Z 19 Acked 1018 Read 2 Done Sent Done false
25392024-02-12T01:35:26.775Z 20 Acked 1019 Read 7 Done Sent Done false
25402024-02-12T01:35:26.775Z 21 Acked 1020 Write 3 Done Sent Done false
25412024-02-12T01:35:26.775Z 22 Acked 1021 Read 2 Done Sent Done false
25422024-02-12T01:35:26.775Z 23 Acked 1022 Read 4 Done Sent Done false
25432024-02-12T01:35:26.775Z 24 Acked 1023 Write 8 Done Sent Done false
25442024-02-12T01:35:26.775Z 25 Acked 1024 Read 4 Done Sent Done false
25452024-02-12T01:35:26.775Z 26 Acked 1025 Flush 0 Done Sent Done false
25462024-02-12T01:35:26.775Z 27 Acked 1026 Write 1 Done Sent Done false
25472024-02-12T01:35:26.775Z 28 Acked 1027 Write 3 Done Sent Done false
25482024-02-12T01:35:26.775Z 29 Acked 1028 Read 2 Done Sent Done false
25492024-02-12T01:35:26.775Z 30 Acked 1029 Write 2 Sent Sent Sent false
25502024-02-12T01:35:26.776Z STATES DS:0 DS:1 DS:2 TOTAL
25512024-02-12T01:35:26.776Z New 0 0 0 0
25522024-02-12T01:35:26.776Z Sent 1 28 1 30
25532024-02-12T01:35:26.776Z Done 27 0 27 54
25542024-02-12T01:35:26.776Z Skipped 0 0 0 0
25552024-02-12T01:35:26.776Z Error 0 0 0 0
25562024-02-12T01:35:26.776ZLast Flush: 1025 1001 1025
25572024-02-12T01:35:26.776ZDownstairs last five completed: 1001 1000
25582024-02-12T01:35:26.776ZUpstairs last five completed: 30 29 28 27 26
25592024-02-12T01:35:26.776Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:25.682593119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1223}
25602024-02-12T01:35:26.776ZWrote out file "/tmp/test_repair_verify.data"
25612024-02-12T01:35:26.780Z
25622024-02-12T01:35:26.780ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25632024-02-12T01:35:26.794ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25642024-02-12T01:35:26.794Z 0 000-019 22 5 22 33 5 33 T F T <---
25652024-02-12T01:35:26.794Z 1 020-039 20 20 20 29 29 29 F F F
25662024-02-12T01:35:26.795Z 2 040-059 22 18 22 33 27 33 F F F <---
25672024-02-12T01:35:26.795Z 3 060-079 22 20 22 34 29 34 F F F <---
25682024-02-12T01:35:26.795Z 4 080-099 22 20 22 34 29 34 F F F <---
25692024-02-12T01:35:26.795Z 5 100-119 22 22 22 34 32 34 F F F <---
25702024-02-12T01:35:26.795Z 6 120-139 22 22 22 33 32 33 F F F <---
25712024-02-12T01:35:26.795Z 7 140-159 19 19 19 28 28 28 F F F
25722024-02-12T01:35:26.795Z 8 160-179 22 19 22 35 28 35 F F F <---
25732024-02-12T01:35:26.795Z 9 180-199 15 15 15 21 21 21 F F F
25742024-02-12T01:35:26.795Z 10 200-219 20 20 20 29 29 29 T F T <---
25752024-02-12T01:35:26.800Z 11 220-239 16 16 16 24 24 24 T F T <---
25762024-02-12T01:35:26.800Z 12 240-259 18 18 18 26 26 26 F F F
25772024-02-12T01:35:26.800Z 13 260-279 18 18 18 26 26 26 T F T <---
25782024-02-12T01:35:26.800Z 14 280-299 16 16 16 22 22 22 T F T <---
25792024-02-12T01:35:26.800Z 15 300-319 22 20 22 35 29 35 F F F <---
25802024-02-12T01:35:26.800Z 16 320-339 21 21 21 31 31 31 F F F
25812024-02-12T01:35:26.800Z 17 340-359 20 20 20 29 29 29 F F F
25822024-02-12T01:35:26.800Z 18 360-379 22 20 22 35 30 35 F F F <---
25832024-02-12T01:35:26.800Z 19 380-399 19 19 19 28 28 28 F F F
25842024-02-12T01:35:26.800Z 20 400-419 19 19 19 28 28 28 F F F
25852024-02-12T01:35:26.800Z 21 420-439 21 21 21 31 31 31 F F F
25862024-02-12T01:35:26.801Z 22 440-459 21 21 21 31 31 31 F F F
25872024-02-12T01:35:26.801Z 23 460-479 21 21 21 31 31 31 F F F
25882024-02-12T01:35:26.801Z 24 480-499 15 15 15 21 21 21 F F F
25892024-02-12T01:35:26.801Z 25 500-519 17 17 17 25 25 25 F F F
25902024-02-12T01:35:26.801Z 26 520-539 9 9 9 13 13 13 F F F
25912024-02-12T01:35:26.801Z 27 540-559 15 15 15 21 21 21 F F F
25922024-02-12T01:35:26.801Z 28 560-579 20 20 20 29 29 29 F F F
25932024-02-12T01:35:26.801Z 29 580-599 17 17 17 25 25 25 F F F
25942024-02-12T01:35:26.801ZMax gen: 22, Max flush: 35
25952024-02-12T01:35:26.801ZError: Difference in extent metadata found!
25962024-02-12T01:35:26.801ZA Difference in extent metadata is expected here
25972024-02-12T01:35:26.801ZOn loop 11
25982024-02-12T01:35:26.801Z
25992024-02-12T01:35:26.806ZVerifying data now
26002024-02-12T01:35:32.198ZNo change after verify
26012024-02-12T01:35:32.198ZLoop: 11 Downstairs dump after verify (and repair):
26022024-02-12T01:35:32.212ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
26032024-02-12T01:35:32.212Z 0 000-019 23 23 23 36 36 36 F F F
26042024-02-12T01:35:32.212Z 1 020-039 20 20 20 29 29 29 F F F
26052024-02-12T01:35:32.212Z 2 040-059 22 22 22 33 33 33 F F F
26062024-02-12T01:35:32.212Z 3 060-079 22 22 22 34 34 34 F F F
26072024-02-12T01:35:32.212Z 4 080-099 22 22 22 34 34 34 F F F
26082024-02-12T01:35:32.215Z 5 100-119 22 22 22 34 34 34 F F F
26092024-02-12T01:35:32.215Z 6 120-139 22 22 22 33 33 33 F F F
26102024-02-12T01:35:32.215Z 7 140-159 19 19 19 28 28 28 F F F
26112024-02-12T01:35:32.215Z 8 160-179 22 22 22 35 35 35 F F F
26122024-02-12T01:35:32.215Z 9 180-199 15 15 15 21 21 21 F F F
26132024-02-12T01:35:32.215Z 10 200-219 23 23 23 36 36 36 F F F
26142024-02-12T01:35:32.215Z 11 220-239 23 23 23 36 36 36 F F F
26152024-02-12T01:35:32.215Z 12 240-259 18 18 18 26 26 26 F F F
26162024-02-12T01:35:32.215Z 13 260-279 23 23 23 36 36 36 F F F
26172024-02-12T01:35:32.215Z 14 280-299 23 23 23 36 36 36 F F F
26182024-02-12T01:35:32.215Z 15 300-319 22 22 22 35 35 35 F F F
26192024-02-12T01:35:32.215Z 16 320-339 21 21 21 31 31 31 F F F
26202024-02-12T01:35:32.215Z 17 340-359 20 20 20 29 29 29 F F F
26212024-02-12T01:35:32.215Z 18 360-379 22 22 22 35 35 35 F F F
26222024-02-12T01:35:32.215Z 19 380-399 19 19 19 28 28 28 F F F
26232024-02-12T01:35:32.215Z 20 400-419 19 19 19 28 28 28 F F F
26242024-02-12T01:35:32.215Z 21 420-439 21 21 21 31 31 31 F F F
26252024-02-12T01:35:32.215Z 22 440-459 21 21 21 31 31 31 F F F
26262024-02-12T01:35:32.216Z 23 460-479 21 21 21 31 31 31 F F F
26272024-02-12T01:35:32.216Z 24 480-499 15 15 15 21 21 21 F F F
26282024-02-12T01:35:32.216Z 25 500-519 17 17 17 25 25 25 F F F
26292024-02-12T01:35:32.216Z 26 520-539 9 9 9 13 13 13 F F F
26302024-02-12T01:35:32.216Z 27 540-559 15 15 15 21 21 21 F F F
26312024-02-12T01:35:32.216Z 28 560-579 20 20 20 29 29 29 F F F
26322024-02-12T01:35:32.216Z 29 580-599 17 17 17 25 25 25 F F F
26332024-02-12T01:35:32.216ZMax gen: 23, Max flush: 36
26342024-02-12T01:35:32.219Z
26352024-02-12T01:35:32.219ZBegin loop 12
26362024-02-12T01:35:32.219ZDownstairs to restart: 2
26372024-02-12T01:35:32.236Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148051087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
26382024-02-12T01:35:32.242Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
26392024-02-12T01:35:32.242Z2024-02-12T01:35:31.148381971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
26402024-02-12T01:35:32.242Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148433109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
26412024-02-12T01:35:32.242Z{"msg":"Crucible d1f01911-b4c7-4129-aa5c-2d7a8f3b202b has session id: ccf230cb-c5a7-4dff-8565-5785a643fc46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148463007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26422024-02-12T01:35:32.242Z{"msg":"Upstairs opts: Upstairs UUID: d1f01911-b4c7-4129-aa5c-2d7a8f3b202b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148539134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26432024-02-12T01:35:32.242Z{"msg":"Crucible stats registered with UUID: d1f01911-b4c7-4129-aa5c-2d7a8f3b202b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148579312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26442024-02-12T01:35:32.242Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.1486048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
26452024-02-12T01:35:32.242Z{"msg":"d1f01911-b4c7-4129-aa5c-2d7a8f3b202b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148635769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26462024-02-12T01:35:32.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148662748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26472024-02-12T01:35:32.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148682047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26482024-02-12T01:35:32.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148709286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26492024-02-12T01:35:32.242Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148731014Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26502024-02-12T01:35:32.242Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148759763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26512024-02-12T01:35:32.243Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148791682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26522024-02-12T01:35:32.243Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.14882019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26532024-02-12T01:35:32.243Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148875678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26542024-02-12T01:35:32.243Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148908846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26552024-02-12T01:35:32.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148961123Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26562024-02-12T01:35:32.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.148994902Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26572024-02-12T01:35:32.243Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.149022951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26582024-02-12T01:35:32.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.149684079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26592024-02-12T01:35:32.243Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.149733857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26602024-02-12T01:35:32.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.149765405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26612024-02-12T01:35:32.243Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.149793404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26622024-02-12T01:35:32.243Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150327268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26632024-02-12T01:35:32.243Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.15049823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26642024-02-12T01:35:32.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150808075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26652024-02-12T01:35:32.243Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150847424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26662024-02-12T01:35:32.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150874712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26672024-02-12T01:35:32.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150907031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26682024-02-12T01:35:32.243Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.15093178Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26692024-02-12T01:35:32.243Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.150962568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26702024-02-12T01:35:32.243Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151031145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26712024-02-12T01:35:32.243Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151073783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26722024-02-12T01:35:32.243Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151489033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26732024-02-12T01:35:32.243Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151883934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26742024-02-12T01:35:32.244Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151928632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26752024-02-12T01:35:32.244Z{"msg":"[0]R flush_numbers[0..12]: [36, 29, 33, 34, 34, 34, 33, 28, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151947531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26762024-02-12T01:35:32.244Z{"msg":"[0]R generation[0..12]: [23, 20, 22, 22, 22, 22, 22, 19, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.15196327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26772024-02-12T01:35:32.244Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.151978279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26782024-02-12T01:35:32.244Z{"msg":"[1]R flush_numbers[0..12]: [36, 29, 33, 34, 34, 34, 33, 28, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152001209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26792024-02-12T01:35:32.244Z{"msg":"[1]R generation[0..12]: [23, 20, 22, 22, 22, 22, 22, 19, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152015518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26802024-02-12T01:35:32.244Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152030317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26812024-02-12T01:35:32.244Z{"msg":"[2]R flush_numbers[0..12]: [36, 29, 33, 34, 34, 34, 33, 28, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152051076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26822024-02-12T01:35:32.244Z{"msg":"[2]R generation[0..12]: [23, 20, 22, 22, 22, 22, 22, 19, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152065015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26832024-02-12T01:35:32.244Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152080215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26842024-02-12T01:35:32.244Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152093374Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26852024-02-12T01:35:32.244Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
26862024-02-12T01:35:32.244Z,"time":"2024-02-12T01:35:31.152113483Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26872024-02-12T01:35:32.244Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152128482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26882024-02-12T01:35:32.244Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152141832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26892024-02-12T01:35:32.244Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152155331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26902024-02-12T01:35:32.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.15216809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26912024-02-12T01:35:32.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152186209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26922024-02-12T01:35:32.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152210448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26932024-02-12T01:35:32.244Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152224558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26942024-02-12T01:35:32.244Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152237467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26952024-02-12T01:35:32.244Z{"msg":"d1f01911-b4c7-4129-aa5c-2d7a8f3b202b is now active with session: ccf230cb-c5a7-4dff-8565-5785a643fc46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152267826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26962024-02-12T01:35:32.244Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152280775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231,"session_id":"ccf230cb-c5a7-4dff-8565-5785a643fc46"}
26972024-02-12T01:35:32.245Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:31.152294524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
26982024-02-12T01:35:32.248ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26992024-02-12T01:35:32.248ZLoading write count information from file "/tmp/test_repair_verify.data"
27002024-02-12T01:35:32.248ZRun Repair workload
27012024-02-12T01:35:32.248Z01/30 Write block 487 len 8192 data: 3 4
27022024-02-12T01:35:32.248Z02/30 Flush
27032024-02-12T01:35:32.287Z03/30 Write block 454 len 12288 data: 3 3 2
27042024-02-12T01:35:32.289Z04/30 Read block 539 len 16384
27052024-02-12T01:35:32.290Z05/30 Read block 544 len 4096
27062024-02-12T01:35:32.290Z06/30 Write block 141 len 20480 data: 3 4 4 4 6
27072024-02-12T01:35:32.290Z07/30 Read block 444 len 16384
27082024-02-12T01:35:32.293Z08/30 Read block 365 len 36864
27092024-02-12T01:35:32.293Z09/30 Read block 245 len 40960
27102024-02-12T01:35:32.293Z10/30 Read block 199 len 4096
27112024-02-12T01:35:32.296Z11/30 Read block 29 len 36864
27122024-02-12T01:35:32.296Z12/30 Read block 82 len 24576
27132024-02-12T01:35:32.296Z13/30 Read block 505 len 32768
27142024-02-12T01:35:32.299Z14/30 Write block 307 len 32768 data: 5 6 4 4 4 3 3 2
27152024-02-12T01:35:32.299Z15/30 Read block 299 len 20480
27162024-02-12T01:35:32.299Z16/30 Flush
27172024-02-12T01:35:32.305Z17/30 Write block 123 len 36864 data: 3 3 3 5 5 4 3 3 4
27182024-02-12T01:35:32.308Z18/30 Write block 41 len 12288 data: 2 2 4
27192024-02-12T01:35:32.308Z19/30 Write block 22 len 16384 data: 2 2 2 2
27202024-02-12T01:35:32.308Z20/30 Write block 417 len 4096 data: 3
27212024-02-12T01:35:32.308Z21/30 Write block 472 len 36864 data: 6 5 5 4 3 3 3 3 4
27222024-02-12T01:35:32.308Z22/30 Write block 138 len 32768 data: 3 3 3 4 5 5 5 7
27232024-02-12T01:35:32.308Z23/30 Read block 304 len 32768
27242024-02-12T01:35:32.309Z24/30 Read block 46 len 4096
27252024-02-12T01:35:32.309Z25/30 Write block 60 len 12288 data: 4 4 5
27262024-02-12T01:35:32.311Z26/30 Read block 50 len 28672
27272024-02-12T01:35:32.311Z27/30 Read block 161 len 28672
27282024-02-12T01:35:32.312Z28/30 Write block 535 len 24576 data: 3 3 4 4 4 3
27292024-02-12T01:35:32.315Z29/30 Write block 374 len 16384 data: 3 4 4 4
27302024-02-12T01:35:32.315Z30/30 Read block 23 len 32768
27312024-02-12T01:35:32.315Z----------------------------------------------------------------
27322024-02-12T01:35:32.315Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27332024-02-12T01:35:32.315ZStates: Active Active Active
27342024-02-12T01:35:32.315ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27352024-02-12T01:35:32.315Z 1 Acked 1000 Write 2 Done Done Done false
27362024-02-12T01:35:32.315Z 2 Acked 1001 Flush 0 Done Done Sent false
27372024-02-12T01:35:32.315Z 3 Acked 1002 Write 3 Done Done Sent false
27382024-02-12T01:35:32.315Z 4 Acked 1003 Read 4 Done Done Sent false
27392024-02-12T01:35:32.315Z 5 Acked 1004 Read 1 Done Done Sent false
27402024-02-12T01:35:32.315Z 6 Acked 1005 Write 5 Done Done Sent false
27412024-02-12T01:35:32.316Z 7 Acked 1006 Read 4 Done Done Sent false
27422024-02-12T01:35:32.316Z 8 Acked 1007 Read 9 Done Done Sent false
27432024-02-12T01:35:32.316Z 9 Acked 1008 Read 10 Done Done Sent false
27442024-02-12T01:35:32.316Z 10 Acked 1009 Read 1 Done Done Sent false
27452024-02-12T01:35:32.316Z 11 Acked 1010 Read 9 Done Done Sent false
27462024-02-12T01:35:32.316Z 12 Acked 1011 Read 6 Done Done Sent false
27472024-02-12T01:35:32.316Z 13 Acked 1012 Read 8 Done Done Sent false
27482024-02-12T01:35:32.316Z 14 Acked 1013 Write 8 Done Done Sent false
27492024-02-12T01:35:32.316Z 15 Acked 1014 Read 5 Done Done Sent false
27502024-02-12T01:35:32.316Z 16 Acked 1015 Flush 0 Done Done Sent false
27512024-02-12T01:35:32.316Z 17 Acked 1016 Write 9 Done Done Sent false
27522024-02-12T01:35:32.316Z 18 Acked 1017 Write 3 Done Done Sent false
27532024-02-12T01:35:32.316Z 19 Acked 1018 Write 4 Done Done Sent false
27542024-02-12T01:35:32.316Z 20 Acked 1019 Write 1 Done Done Sent false
27552024-02-12T01:35:32.316Z 21 Acked 1020 Write 9 Done Done Sent false
27562024-02-12T01:35:32.316Z 22 Acked 1021 Write 8 Done Done Sent false
27572024-02-12T01:35:32.316Z 23 Acked 1022 Read 8 Done Done Sent false
27582024-02-12T01:35:32.316Z 24 Acked 1023 Read 1 Done Done Sent false
27592024-02-12T01:35:32.316Z 25 Acked 1024 Write 3 Done Done Sent false
27602024-02-12T01:35:32.316Z 26 Acked 1025 Read 7 Done Done Sent false
27612024-02-12T01:35:32.316Z 27 Acked 1026 Read 7 Done Done Sent false
27622024-02-12T01:35:32.316Z 28 Acked 1027 Write 6 Done Done Sent false
27632024-02-12T01:35:32.316Z 29 Acked 1028 Write 4 Done Done Sent false
27642024-02-12T01:35:32.316Z 30 Acked 1029 Read 8 Sent Done Sent false
27652024-02-12T01:35:32.316Z STATES DS:0 DS:1 DS:2 TOTAL
27662024-02-12T01:35:32.317Z New 0 0 0 0
27672024-02-12T01:35:32.317Z Sent 1 0 29 30
27682024-02-12T01:35:32.317Z Done 29 30 1 60
27692024-02-12T01:35:32.317Z Skipped 0 0 0 0
27702024-02-12T01:35:32.317Z Error 0 0 0 0
27712024-02-12T01:35:32.317ZLast Flush: 1015 1015 0
27722024-02-12T01:35:32.317ZDownstairs last five completed:
27732024-02-12T01:35:32.317ZUpstairs last five completed: 30 29 28 27 26
27742024-02-12T01:35:32.317Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:31.226978531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1231}
27752024-02-12T01:35:32.317ZWrote out file "/tmp/test_repair_verify.data"
27762024-02-12T01:35:32.320Z
27772024-02-12T01:35:32.323ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27782024-02-12T01:35:32.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27792024-02-12T01:35:32.341Z 0 000-019 23 23 23 36 36 36 F F F
27802024-02-12T01:35:32.341Z 1 020-039 20 20 20 29 29 29 T T F <---
27812024-02-12T01:35:32.341Z 2 040-059 22 22 22 33 33 33 T T F <---
27822024-02-12T01:35:32.341Z 3 060-079 22 22 22 34 34 34 T T F <---
27832024-02-12T01:35:32.341Z 4 080-099 22 22 22 34 34 34 F F F
27842024-02-12T01:35:32.341Z 5 100-119 22 22 22 34 34 34 F F F
27852024-02-12T01:35:32.341Z 6 120-139 22 22 22 33 33 33 T T F <---
27862024-02-12T01:35:32.341Z 7 140-159 24 24 19 38 38 28 T T F <---
27872024-02-12T01:35:32.341Z 8 160-179 22 22 22 35 35 35 F F F
27882024-02-12T01:35:32.341Z 9 180-199 15 15 15 21 21 21 F F F
27892024-02-12T01:35:32.345Z 10 200-219 23 23 23 36 36 36 F F F
27902024-02-12T01:35:32.345Z 11 220-239 23 23 23 36 36 36 F F F
27912024-02-12T01:35:32.345Z 12 240-259 18 18 18 26 26 26 F F F
27922024-02-12T01:35:32.345Z 13 260-279 23 23 23 36 36 36 F F F
27932024-02-12T01:35:32.345Z 14 280-299 23 23 23 36 36 36 F F F
27942024-02-12T01:35:32.345Z 15 300-319 24 24 22 38 38 35 F F F <---
27952024-02-12T01:35:32.345Z 16 320-339 21 21 21 31 31 31 F F F
27962024-02-12T01:35:32.345Z 17 340-359 20 20 20 29 29 29 F F F
27972024-02-12T01:35:32.345Z 18 360-379 22 22 22 35 35 35 T T F <---
27982024-02-12T01:35:32.345Z 19 380-399 19 19 19 28 28 28 F F F
27992024-02-12T01:35:32.345Z 20 400-419 19 19 19 28 28 28 T T F <---
28002024-02-12T01:35:32.345Z 21 420-439 21 21 21 31 31 31 F F F
28012024-02-12T01:35:32.345Z 22 440-459 24 24 21 38 38 31 F F F <---
28022024-02-12T01:35:32.345Z 23 460-479 21 21 21 31 31 31 T T F <---
28032024-02-12T01:35:32.345Z 24 480-499 24 24 15 37 37 21 T T T <---
28042024-02-12T01:35:32.345Z 25 500-519 17 17 17 25 25 25 F F F
28052024-02-12T01:35:32.345Z 26 520-539 9 9 9 13 13 13 T T F <---
28062024-02-12T01:35:32.345Z 27 540-559 15 15 15 21 21 21 T T F <---
28072024-02-12T01:35:32.345Z 28 560-579 20 20 20 29 29 29 F F F
28082024-02-12T01:35:32.345Z 29 580-599 17 17 17 25 25 25 F F F
28092024-02-12T01:35:32.345ZMax gen: 24, Max flush: 38
28102024-02-12T01:35:32.345ZError: Difference in extent metadata found!
28112024-02-12T01:35:32.345ZA Difference in extent metadata is expected here
28122024-02-12T01:35:32.346ZOn loop 12
28132024-02-12T01:35:32.346Z
28142024-02-12T01:35:32.349ZVerifying data now
28152024-02-12T01:35:38.946ZNo change after verify
28162024-02-12T01:35:38.946ZLoop: 12 Downstairs dump after verify (and repair):
28172024-02-12T01:35:38.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28182024-02-12T01:35:38.961Z 0 000-019 23 23 23 36 36 36 F F F
28192024-02-12T01:35:38.961Z 1 020-039 25 25 25 39 39 39 F F F
28202024-02-12T01:35:38.961Z 2 040-059 25 25 25 39 39 39 F F F
28212024-02-12T01:35:38.961Z 3 060-079 25 25 25 39 39 39 F F F
28222024-02-12T01:35:38.961Z 4 080-099 22 22 22 34 34 34 F F F
28232024-02-12T01:35:38.961Z 5 100-119 22 22 22 34 34 34 F F F
28242024-02-12T01:35:38.961Z 6 120-139 25 25 25 39 39 39 F F F
28252024-02-12T01:35:38.961Z 7 140-159 25 25 25 39 39 39 F F F
28262024-02-12T01:35:38.961Z 8 160-179 22 22 22 35 35 35 F F F
28272024-02-12T01:35:38.961Z 9 180-199 15 15 15 21 21 21 F F F
28282024-02-12T01:35:38.964Z 10 200-219 23 23 23 36 36 36 F F F
28292024-02-12T01:35:38.964Z 11 220-239 23 23 23 36 36 36 F F F
28302024-02-12T01:35:38.964Z 12 240-259 18 18 18 26 26 26 F F F
28312024-02-12T01:35:38.964Z 13 260-279 23 23 23 36 36 36 F F F
28322024-02-12T01:35:38.964Z 14 280-299 23 23 23 36 36 36 F F F
28332024-02-12T01:35:38.964Z 15 300-319 24 24 24 38 38 38 F F F
28342024-02-12T01:35:38.964Z 16 320-339 21 21 21 31 31 31 F F F
28352024-02-12T01:35:38.964Z 17 340-359 20 20 20 29 29 29 F F F
28362024-02-12T01:35:38.964Z 18 360-379 25 25 25 39 39 39 F F F
28372024-02-12T01:35:38.964Z 19 380-399 19 19 19 28 28 28 F F F
28382024-02-12T01:35:38.964Z 20 400-419 25 25 25 39 39 39 F F F
28392024-02-12T01:35:38.964Z 21 420-439 21 21 21 31 31 31 F F F
28402024-02-12T01:35:38.964Z 22 440-459 24 24 24 38 38 38 F F F
28412024-02-12T01:35:38.964Z 23 460-479 25 25 25 39 39 39 F F F
28422024-02-12T01:35:38.964Z 24 480-499 25 25 25 39 39 39 F F F
28432024-02-12T01:35:38.965Z 25 500-519 17 17 17 25 25 25 F F F
28442024-02-12T01:35:38.965Z 26 520-539 25 25 25 39 39 39 F F F
28452024-02-12T01:35:38.965Z 27 540-559 25 25 25 39 39 39 F F F
28462024-02-12T01:35:38.965Z 28 560-579 20 20 20 29 29 29 F F F
28472024-02-12T01:35:38.965Z 29 580-599 17 17 17 25 25 25 F F F
28482024-02-12T01:35:38.965ZMax gen: 25, Max flush: 39
28492024-02-12T01:35:38.965Z
28502024-02-12T01:35:38.965ZBegin loop 13
28512024-02-12T01:35:38.965ZDownstairs to restart: 0
28522024-02-12T01:35:38.988Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897414561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
28532024-02-12T01:35:38.994Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897709497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
28542024-02-12T01:35:38.994Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897751355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
28552024-02-12T01:35:38.994Z{"msg":"Crucible 0d9e322a-8142-4e3e-9420-3aff498ca49f has session id: c840b20b-528b-4931-b4a7-d8be49600a62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897775783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28562024-02-12T01:35:38.994Z{"msg":"Upstairs opts: Upstairs UUID: 0d9e322a-8142-4e3e-9420-3aff498ca49f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897831641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28572024-02-12T01:35:38.994Z{"msg"Crucible runtime is spawned
28582024-02-12T01:35:38.995Z:"Crucible stats registered with UUID: 0d9e322a-8142-4e3e-9420-3aff498ca49f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897877889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28592024-02-12T01:35:38.995Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.897967514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
28602024-02-12T01:35:38.995Z{"msg":"0d9e322a-8142-4e3e-9420-3aff498ca49f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898110558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28612024-02-12T01:35:38.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898163125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28622024-02-12T01:35:38.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898211573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28632024-02-12T01:35:38.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898237761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28642024-02-12T01:35:38.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.89826197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28652024-02-12T01:35:38.995Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898298669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28662024-02-12T01:35:38.995Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898322207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28672024-02-12T01:35:38.995Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898352766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28682024-02-12T01:35:38.995Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898387084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28692024-02-12T01:35:38.995Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898419263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28702024-02-12T01:35:38.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898443302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28712024-02-12T01:35:38.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.89846609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28722024-02-12T01:35:38.995Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.898567356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28732024-02-12T01:35:38.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899125609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28742024-02-12T01:35:38.995Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899189096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28752024-02-12T01:35:38.995Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899230564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28762024-02-12T01:35:38.995Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899255983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28772024-02-12T01:35:38.995Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899681143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28782024-02-12T01:35:38.995Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.899857974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28792024-02-12T01:35:38.995Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900165089Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28802024-02-12T01:35:38.995Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900207618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28812024-02-12T01:35:38.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900294283Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28822024-02-12T01:35:38.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900338881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28832024-02-12T01:35:38.996Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.90035725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28842024-02-12T01:35:38.996Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900378919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28852024-02-12T01:35:38.996Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900391739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28862024-02-12T01:35:38.996Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900411628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28872024-02-12T01:35:38.996Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.900886375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28882024-02-12T01:35:38.996Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901315705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28892024-02-12T01:35:38.996Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901339564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28902024-02-12T01:35:38.996Z{"msg":"[0]R flush_numbers[0..12]: [36, 39, 39, 39, 34, 34, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901354563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28912024-02-12T01:35:38.996Z{"msg":"[0]R generation[0..12]: [23, 25, 25, 25, 22, 22, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901368812Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28922024-02-12T01:35:38.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901391901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28932024-02-12T01:35:38.996Z{"msg":"[1]R flush_numbers[0..12]: [36, 39, 39, 39, 34, 34, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.90140642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28942024-02-12T01:35:38.996Z{"msg":"[1]R generation[0..12]: [23, 25, 25, 25, 22, 22, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901426679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28952024-02-12T01:35:38.996Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901440748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28962024-02-12T01:35:38.996Z{"msg":"[2]R flush_numbers[0..12]: [36, 39, 39, 39, 34, 34, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901461068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28972024-02-12T01:35:38.996Z{"msg":"[2]R generation[0..12]: [23, 25, 25, 25, 22, 22, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901474747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28982024-02-12T01:35:38.996Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901489036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
28992024-02-12T01:35:38.996Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901509015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO
29002024-02-12T01:35:38.996Z:"c840b20b-528b-4931-b4a7-d8be49600a62"}
29012024-02-12T01:35:38.996Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901526244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29022024-02-12T01:35:38.996Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901539394Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29032024-02-12T01:35:38.996Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901559463Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29042024-02-12T01:35:38.996Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901573012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29052024-02-12T01:35:38.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901591761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29062024-02-12T01:35:38.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901605071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29072024-02-12T01:35:38.997Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.90162555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29082024-02-12T01:35:38.997Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901638989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29092024-02-12T01:35:38.997Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901657738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29102024-02-12T01:35:38.997Z{"msg":"0d9e322a-8142-4e3e-9420-3aff498ca49f is now active with session: c840b20b-528b-4931-b4a7-d8be49600a62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901671497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29112024-02-12T01:35:38.997Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901683967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239,"session_id":"c840b20b-528b-4931-b4a7-d8be49600a62"}
29122024-02-12T01:35:38.997Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:37.901696116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
29132024-02-12T01:35:38.997ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
29142024-02-12T01:35:38.997ZLoading write count information from file "/tmp/test_repair_verify.data"
29152024-02-12T01:35:38.997ZRun Repair workload
29162024-02-12T01:35:38.997Z01/30 Write block 107 len 16384 data: 3 3 2 3
29172024-02-12T01:35:39.001Z02/30 Read block 105 len 8192
29182024-02-12T01:35:39.001Z03/30 Write block 398 len 4096 data: 2
29192024-02-12T01:35:39.001Z04/30 Read block 323 len 20480
29202024-02-12T01:35:39.001Z05/30 Write block 29 len 8192 data: 5 4
29212024-02-12T01:35:39.001Z06/30 Write block 12 len 16384 data: 3 3 4 4
29222024-02-12T01:35:39.001Z07/30 Read block 476 len 36864
29232024-02-12T01:35:39.002Z08/30 Write block 538 len 12288 data: 5 5 4
29242024-02-12T01:35:39.002Z09/30 Read block 25 len 16384
29252024-02-12T01:35:39.002Z10/30 Read block 377 len 20480
29262024-02-12T01:35:39.002Z11/30 Read block 522 len 4096
29272024-02-12T01:35:39.002Z12/30 Write block 47 len 20480 data: 4 3 3 3 3
29282024-02-12T01:35:39.002Z13/30 Flush
29292024-02-12T01:35:39.022Z14/30 Flush
29302024-02-12T01:35:39.025Z15/30 Write block 535 len 32768 data: 4 4 5 6 6 5 3 3
29312024-02-12T01:35:39.025Z16/30 Read block 309 len 40960
29322024-02-12T01:35:39.025Z17/30 Read block 502 len 8192
29332024-02-12T01:35:39.028Z18/30 Read block 587 len 4096
29342024-02-12T01:35:39.028Z19/30 Read block 544 len 32768
29352024-02-12T01:35:39.028Z20/30 Write block 328 len 24576 data: 2 2 2 2 2 2
29362024-02-12T01:35:39.028Z21/30 Read block 258 len 24576
29372024-02-12T01:35:39.031Z22/30 Read block 157 len 28672
29382024-02-12T01:35:39.031Z23/30 Read block 158 len 12288
29392024-02-12T01:35:39.031Z24/30 Write block 596 len 8192 data: 4 3
29402024-02-12T01:35:39.031Z25/30 Read block 462 len 4096
29412024-02-12T01:35:39.035Z26/30 Read block 531 len 24576
29422024-02-12T01:35:39.035Z27/30 Write block 405 len 20480 data: 4 4 4 5 4
29432024-02-12T01:35:39.035Z28/30 Write block 2 len 36864 data: 2 2 2 3 3 3 3 3 3
29442024-02-12T01:35:39.038Z29/30 Read block 177 len 12288
29452024-02-12T01:35:39.038Z30/30 Write block 290 len 32768 data: 2 3 2 2 2 2 2 2
29462024-02-12T01:35:39.038Z----------------------------------------------------------------
29472024-02-12T01:35:39.038Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29482024-02-12T01:35:39.038ZStates: Active Active Active
29492024-02-12T01:35:39.038ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29502024-02-12T01:35:39.038Z 1 Acked 1000 Write 4 Done Done Done false
29512024-02-12T01:35:39.039Z 2 Acked 1001 Read 2 Done Done Done false
29522024-02-12T01:35:39.039Z 3 Acked 1002 Write 1 Sent Done Done false
29532024-02-12T01:35:39.039Z 4 Acked 1003 Read 5 Sent Done Done false
29542024-02-12T01:35:39.039Z 5 Acked 1004 Write 2 Sent Done Done false
29552024-02-12T01:35:39.039Z 6 Acked 1005 Write 4 Sent Done Done false
29562024-02-12T01:35:39.039Z 7 Acked 1006 Read 9 Sent Done Done false
29572024-02-12T01:35:39.039Z 8 Acked 1007 Write 3 Sent Done Done false
29582024-02-12T01:35:39.039Z 9 Acked 1008 Read 4 Sent Done Done false
29592024-02-12T01:35:39.039Z 10 Acked 1009 Read 5 Sent Done Done false
29602024-02-12T01:35:39.039Z 11 Acked 1010 Read 1 Sent Done Done false
29612024-02-12T01:35:39.039Z 12 Acked 1011 Write 5 Sent Done Done false
29622024-02-12T01:35:39.039Z 13 Acked 1012 Flush 0 Sent Done Done false
29632024-02-12T01:35:39.039Z 14 Acked 1013 Flush 0 Sent Done Done false
29642024-02-12T01:35:39.039Z 15 Acked 1014 Write 8 Sent Done Done false
29652024-02-12T01:35:39.039Z 16 Acked 1015 Read 10 Sent Done Done false
29662024-02-12T01:35:39.039Z 17 Acked 1016 Read 2 Sent Done Done false
29672024-02-12T01:35:39.039Z 18 Acked 1017 Read 1 Sent Done Done false
29682024-02-12T01:35:39.039Z 19 Acked 1018 Read 8 Sent Done Done false
29692024-02-12T01:35:39.039Z 20 Acked 1019 Write 6 Sent Done Done false
29702024-02-12T01:35:39.039Z 21 Acked 1020 Read 6 Sent Done Done false
29712024-02-12T01:35:39.039Z 22 Acked 1021 Read 7 Sent Done Done false
29722024-02-12T01:35:39.039Z 23 Acked 1022 Read 3 Sent Done Done false
29732024-02-12T01:35:39.039Z 24 Acked 1023 Write 2 Sent Done Done false
29742024-02-12T01:35:39.039Z 25 Acked 1024 Read 1 Sent Done Done false
29752024-02-12T01:35:39.040Z 26 Acked 1025 Read 6 Sent Done Done false
29762024-02-12T01:35:39.040Z 27 Acked 1026 Write 5 Sent Done Done false
29772024-02-12T01:35:39.040Z 28 Acked 1027 Write 9 Sent Done Done false
29782024-02-12T01:35:39.040Z 29 Acked 1028 Read 3 Sent Done Done false
29792024-02-12T01:35:39.040Z 30 Acked 1029 Write 8 Sent Sent Sent false
29802024-02-12T01:35:39.040Z STATES DS:0 DS:1 DS:2 TOTAL
29812024-02-12T01:35:39.040Z New 0 0 0 0
29822024-02-12T01:35:39.040Z Sent 28 1 1 30
29832024-02-12T01:35:39.040Z Done 2 29 29 60
29842024-02-12T01:35:39.040Z Skipped 0 0 0 0
29852024-02-12T01:35:39.040Z Error 0 0 0 0
29862024-02-12T01:35:39.040ZLast Flush: 0 1013 1013
29872024-02-12T01:35:39.040ZDownstairs last five completed:
29882024-02-12T01:35:39.040ZUpstairs last five completed: 30 29 28 27 26
29892024-02-12T01:35:39.040Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:37.949229619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1239}
29902024-02-12T01:35:39.040ZWrote out file "/tmp/test_repair_verify.data"
29912024-02-12T01:35:39.044Z
29922024-02-12T01:35:39.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29932024-02-12T01:35:39.061ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29942024-02-12T01:35:39.062Z 0 000-019 23 26 26 36 40 40 F T T <---
29952024-02-12T01:35:39.062Z 1 020-039 25 26 26 39 40 40 F F F <---
29962024-02-12T01:35:39.062Z 2 040-059 25 26 26 39 40 40 F F F <---
29972024-02-12T01:35:39.062Z 3 060-079 25 25 25 39 39 39 F F F
29982024-02-12T01:35:39.062Z 4 080-099 22 22 22 34 34 34 F F F
29992024-02-12T01:35:39.065Z 5 100-119 22 26 26 34 40 40 T F F <---
30002024-02-12T01:35:39.065Z 6 120-139 25 25 25 39 39 39 F F F
30012024-02-12T01:35:39.065Z 7 140-159 25 25 25 39 39 39 F F F
30022024-02-12T01:35:39.065Z 8 160-179 22 22 22 35 35 35 F F F
30032024-02-12T01:35:39.065Z 9 180-199 15 15 15 21 21 21 F F F
30042024-02-12T01:35:39.065Z 10 200-219 23 23 23 36 36 36 F F F
30052024-02-12T01:35:39.065Z 11 220-239 23 23 23 36 36 36 F F F
30062024-02-12T01:35:39.065Z 12 240-259 18 18 18 26 26 26 F F F
30072024-02-12T01:35:39.065Z 13 260-279 23 23 23 36 36 36 F F F
30082024-02-12T01:35:39.065Z 14 280-299 23 23 23 36 36 36 F T T <---
30092024-02-12T01:35:39.065Z 15 300-319 24 24 24 38 38 38 F F F
30102024-02-12T01:35:39.065Z 16 320-339 21 21 21 31 31 31 F T T <---
30112024-02-12T01:35:39.065Z 17 340-359 20 20 20 29 29 29 F F F
30122024-02-12T01:35:39.066Z 18 360-379 25 25 25 39 39 39 F F F
30132024-02-12T01:35:39.066Z 19 380-399 19 26 26 28 40 40 F F F <---
30142024-02-12T01:35:39.066Z 20 400-419 25 25 25 39 39 39 F T T <---
30152024-02-12T01:35:39.066Z 21 420-439 21 21 21 31 31 31 F F F
30162024-02-12T01:35:39.066Z 22 440-459 24 24 24 38 38 38 F F F
30172024-02-12T01:35:39.066Z 23 460-479 25 25 25 39 39 39 F F F
30182024-02-12T01:35:39.066Z 24 480-499 25 25 25 39 39 39 F F F
30192024-02-12T01:35:39.066Z 25 500-519 17 17 17 25 25 25 F F F
30202024-02-12T01:35:39.066Z 26 520-539 25 26 26 39 40 40 F T T <---
30212024-02-12T01:35:39.066Z 27 540-559 25 26 26 39 40 40 F T T <---
30222024-02-12T01:35:39.066Z 28 560-579 20 20 20 29 29 29 F F F
30232024-02-12T01:35:39.066Z 29 580-599 17 17 17 25 25 25 F T T <---
30242024-02-12T01:35:39.066ZMax gen: 26, Max flush: 40
30252024-02-12T01:35:39.066ZError: Difference in extent metadata found!
30262024-02-12T01:35:39.066ZA Difference in extent metadata is expected here
30272024-02-12T01:35:39.066ZOn loop 13
30282024-02-12T01:35:39.066Z
30292024-02-12T01:35:39.069ZVerifying data now
30302024-02-12T01:35:45.347ZNo change after verify
30312024-02-12T01:35:45.347ZLoop: 13 Downstairs dump after verify (and repair):
30322024-02-12T01:35:45.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30332024-02-12T01:35:45.362Z 0 000-019 27 27 27 41 41 41 F F F
30342024-02-12T01:35:45.362Z 1 020-039 26 26 26 40 40 40 F F F
30352024-02-12T01:35:45.362Z 2 040-059 26 26 26 40 40 40 F F F
30362024-02-12T01:35:45.362Z 3 060-079 25 25 25 39 39 39 F F F
30372024-02-12T01:35:45.363Z 4 080-099 22 22 22 34 34 34 F F F
30382024-02-12T01:35:45.363Z 5 100-119 26 26 26 40 40 40 F F F
30392024-02-12T01:35:45.363Z 6 120-139 25 25 25 39 39 39 F F F
30402024-02-12T01:35:45.363Z 7 140-159 25 25 25 39 39 39 F F F
30412024-02-12T01:35:45.366Z 8 160-179 22 22 22 35 35 35 F F F
30422024-02-12T01:35:45.366Z 9 180-199 15 15 15 21 21 21 F F F
30432024-02-12T01:35:45.366Z 10 200-219 23 23 23 36 36 36 F F F
30442024-02-12T01:35:45.366Z 11 220-239 23 23 23 36 36 36 F F F
30452024-02-12T01:35:45.366Z 12 240-259 18 18 18 26 26 26 F F F
30462024-02-12T01:35:45.366Z 13 260-279 23 23 23 36 36 36 F F F
30472024-02-12T01:35:45.366Z 14 280-299 27 27 27 41 41 41 F F F
30482024-02-12T01:35:45.366Z 15 300-319 24 24 24 38 38 38 F F F
30492024-02-12T01:35:45.366Z 16 320-339 27 27 27 41 41 41 F F F
30502024-02-12T01:35:45.366Z 17 340-359 20 20 20 29 29 29 F F F
30512024-02-12T01:35:45.366Z 18 360-379 25 25 25 39 39 39 F F F
30522024-02-12T01:35:45.366Z 19 380-399 26 26 26 40 40 40 F F F
30532024-02-12T01:35:45.366Z 20 400-419 27 27 27 41 41 41 F F F
30542024-02-12T01:35:45.367Z 21 420-439 21 21 21 31 31 31 F F F
30552024-02-12T01:35:45.367Z 22 440-459 24 24 24 38 38 38 F F F
30562024-02-12T01:35:45.367Z 23 460-479 25 25 25 39 39 39 F F F
30572024-02-12T01:35:45.367Z 24 480-499 25 25 25 39 39 39 F F F
30582024-02-12T01:35:45.367Z 25 500-519 17 17 17 25 25 25 F F F
30592024-02-12T01:35:45.367Z 26 520-539 27 27 27 41 41 41 F F F
30602024-02-12T01:35:45.367Z 27 540-559 27 27 27 41 41 41 F F F
30612024-02-12T01:35:45.367Z 28 560-579 20 20 20 29 29 29 F F F
30622024-02-12T01:35:45.367Z 29 580-599 27 27 27 41 41 41 F F F
30632024-02-12T01:35:45.367ZMax gen: 27, Max flush: 41
30642024-02-12T01:35:45.367Z
30652024-02-12T01:35:45.367ZBegin loop 14
30662024-02-12T01:35:45.367ZDownstairs to restart: 2
30672024-02-12T01:35:45.397Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.296545954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
30682024-02-12T01:35:45.397Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\"Crucible runtime is spawned
30692024-02-12T01:35:45.405Z,\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.296899307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
30702024-02-12T01:35:45.405Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.296949455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
30712024-02-12T01:35:45.405Z{"msg":"Crucible 2cd65ba8-73a0-4297-9d99-f86df2d5e060 has session id: 1140759c-81ef-45be-9e26-3f564fb7b7a8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.296980154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30722024-02-12T01:35:45.405Z{"msg":"Upstairs opts: Upstairs UUID: 2cd65ba8-73a0-4297-9d99-f86df2d5e060, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297024661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30732024-02-12T01:35:45.405Z{"msg":"Crucible stats registered with UUID: 2cd65ba8-73a0-4297-9d99-f86df2d5e060","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297067949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30742024-02-12T01:35:45.405Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297103688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
30752024-02-12T01:35:45.405Z{"msg":"2cd65ba8-73a0-4297-9d99-f86df2d5e060 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297136436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30762024-02-12T01:35:45.405Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297156415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30772024-02-12T01:35:45.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297184664Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30782024-02-12T01:35:45.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297216342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30792024-02-12T01:35:45.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.29725434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30802024-02-12T01:35:45.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297308618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30812024-02-12T01:35:45.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297338756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30822024-02-12T01:35:45.406Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297368305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30832024-02-12T01:35:45.406Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297422642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30842024-02-12T01:35:45.406Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.29746623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30852024-02-12T01:35:45.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.29748211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30862024-02-12T01:35:45.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297562666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30872024-02-12T01:35:45.406Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.297590764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30882024-02-12T01:35:45.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.2981042Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30892024-02-12T01:35:45.406Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.298145558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30902024-02-12T01:35:45.406Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.298162037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30912024-02-12T01:35:45.406Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.298180276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30922024-02-12T01:35:45.406Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.29873638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30932024-02-12T01:35:45.407Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.298905072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30942024-02-12T01:35:45.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299192118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30952024-02-12T01:35:45.407Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299238876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30962024-02-12T01:35:45.407Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299266844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30972024-02-12T01:35:45.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299336651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30982024-02-12T01:35:45.407Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299375299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
30992024-02-12T01:35:45.407Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299391559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31002024-02-12T01:35:45.407Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.299532412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31012024-02-12T01:35:45.407Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.29955947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31022024-02-12T01:35:45.407Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300082186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31032024-02-12T01:35:45.407Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300500946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31042024-02-12T01:35:45.407Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300546043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31052024-02-12T01:35:45.407Z{"msg":"[0]R flush_numbers[0..12]: [41, 40, 40, 39, 34, 40, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300565572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31062024-02-12T01:35:45.407Z{"msg":"[0]R generation[0..12]: [27, 26, 26, 25, 22, 26, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300591541Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31072024-02-12T01:35:45.408Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.30060976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31082024-02-12T01:35:45.408Z{"msg":"[1]R flush_numbers[0..12]: [41, 40, 40, 39, 34, 40, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300635619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31092024-02-12T01:35:45.408Z{"msg":"[1]R generation[0..12]: [27, 26, 26, 25, 22, 26, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300653058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31102024-02-12T01:35:45.408Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300678657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31112024-02-12T01:35:45.408Z{"msg":"[2]R flush_numbers[0..12]: [41, 40, 40, 39, 34, 40, 39, 39, 35, 21, 36, 36]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
31122024-02-12T01:35:45.408Z,"time":"2024-02-12T01:35:44.300696136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31132024-02-12T01:35:45.408Z{"msg":"[2]R generation[0..12]: [27, 26, 26, 25, 22, 26, 25, 25, 22, 15, 23, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300722025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31142024-02-12T01:35:45.408Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300739844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31152024-02-12T01:35:45.408Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300757203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31162024-02-12T01:35:45.408Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300774302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31172024-02-12T01:35:45.408Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300805511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31182024-02-12T01:35:45.408Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.30082227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31192024-02-12T01:35:45.408Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300839019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31202024-02-12T01:35:45.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300861778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31212024-02-12T01:35:45.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300878557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31222024-02-12T01:35:45.408Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300902796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31232024-02-12T01:35:45.408Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300919626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31242024-02-12T01:35:45.408Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300942764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31252024-02-12T01:35:45.409Z{"msg":"2cd65ba8-73a0-4297-9d99-f86df2d5e060 is now active with session: 1140759c-81ef-45be-9e26-3f564fb7b7a8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300958904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31262024-02-12T01:35:45.409Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300974263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247,"session_id":"1140759c-81ef-45be-9e26-3f564fb7b7a8"}
31272024-02-12T01:35:45.409Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:44.300989702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
31282024-02-12T01:35:45.409ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
31292024-02-12T01:35:45.409ZLoading write count information from file "/tmp/test_repair_verify.data"
31302024-02-12T01:35:45.409ZRun Repair workload
31312024-02-12T01:35:45.409Z01/30 Write block 16 len 16384 data: 3 3 3 3
31322024-02-12T01:35:45.409Z02/30 Read block 498 len 36864
31332024-02-12T01:35:45.409Z03/30 Write block 187 len 28672 data: 4 6 6 5 4 3 2
31342024-02-12T01:35:45.409Z04/30 Read block 108 len 4096
31352024-02-12T01:35:45.409Z05/30 Read block 412 len 28672
31362024-02-12T01:35:45.409Z06/30 Flush
31372024-02-12T01:35:45.409Z07/30 Write block 364 len 24576 data: 4 6 6 7 5 5
31382024-02-12T01:35:45.409Z08/30 Read block 37 len 28672
31392024-02-12T01:35:45.409Z09/30 Read block 593 len 4096
31402024-02-12T01:35:45.409Z10/30 Read block 324 len 4096
31412024-02-12T01:35:45.413Z11/30 Read block 128 len 20480
31422024-02-12T01:35:45.413Z12/30 Read block 499 len 36864
31432024-02-12T01:35:45.413Z13/30 Flush
31442024-02-12T01:35:45.413Z14/30 Write block 485 len 24576 data: 3 3 4 5 4 4
31452024-02-12T01:35:45.413Z15/30 Read block 425 len 32768
31462024-02-12T01:35:45.413Z16/30 Read block 386 len 32768
31472024-02-12T01:35:45.413Z17/30 Read block 434 len 8192
31482024-02-12T01:35:45.413Z18/30 Read block 305 len 4096
31492024-02-12T01:35:45.413Z19/30 Read block 540 len 4096
31502024-02-12T01:35:45.413Z20/30 Read block 417 len 36864
31512024-02-12T01:35:45.414Z21/30 Read block 589 len 32768
31522024-02-12T01:35:45.414Z22/30 Write block 129 len 36864 data: 4 4 5 4 3 3 2 2 2
31532024-02-12T01:35:45.414Z23/30 Write block 409 len 20480 data: 5 4 5 4 5
31542024-02-12T01:35:45.414Z24/30 Read block 199 len 8192
31552024-02-12T01:35:45.417Z25/30 Write block 206 len 24576 data: 4 5 5 5 2 3
31562024-02-12T01:35:45.417Z26/30 Write block 556 len 8192 data: 3 3
31572024-02-12T01:35:45.417Z27/30 Read block 388 len 40960
31582024-02-12T01:35:45.417Z28/30 Read block 89 len 32768
31592024-02-12T01:35:45.417Z29/30 Read block 519 len 28672
31602024-02-12T01:35:45.421Z30/30 Write block 479 len 24576 data: 4 5 3 3 3 3
31612024-02-12T01:35:45.421Z----------------------------------------------------------------
31622024-02-12T01:35:45.421Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:24
31632024-02-12T01:35:45.421ZStates: Active Active Active
31642024-02-12T01:35:45.421ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31652024-02-12T01:35:45.421Z 7 Acked 1006 Write 6 Done Done Done false
31662024-02-12T01:35:45.421Z 8 Acked 1007 Read 7 Done Done Done false
31672024-02-12T01:35:45.421Z 9 Acked 1008 Read 1 Done Done Done false
31682024-02-12T01:35:45.421Z 10 Acked 1009 Read 1 Done Done Done false
31692024-02-12T01:35:45.421Z 11 Acked 1010 Read 5 Done Done Done false
31702024-02-12T01:35:45.421Z 12 Acked 1011 Read 9 Done Done Sent false
31712024-02-12T01:35:45.421Z 13 Acked 1012 Flush 0 Done Done Sent false
31722024-02-12T01:35:45.421Z 14 Acked 1013 Write 6 Done Done Sent false
31732024-02-12T01:35:45.421Z 15 Acked 1014 Read 8 Done Done Sent false
31742024-02-12T01:35:45.421Z 16 Acked 1015 Read 8 Done Done Sent false
31752024-02-12T01:35:45.422Z 17 Acked 1016 Read 2 Done Done Sent false
31762024-02-12T01:35:45.422Z 18 Acked 1017 Read 1 Done Done Sent false
31772024-02-12T01:35:45.422Z 19 Acked 1018 Read 1 Done Done Sent false
31782024-02-12T01:35:45.422Z 20 Acked 1019 Read 9 Done Done Sent false
31792024-02-12T01:35:45.422Z 21 Acked 1020 Read 8 Done Done Sent false
31802024-02-12T01:35:45.422Z 22 Acked 1021 Write 9 Done Done Sent false
31812024-02-12T01:35:45.422Z 23 Acked 1022 Write 5 Done Done Sent false
31822024-02-12T01:35:45.422Z 24 Acked 1023 Read 2 Done Done Sent false
31832024-02-12T01:35:45.422Z 25 Acked 1024 Write 6 Done Done Sent false
31842024-02-12T01:35:45.422Z 26 Acked 1025 Write 2 Done Done Sent false
31852024-02-12T01:35:45.422Z 27 Acked 1026 Read 10 Done Done Sent false
31862024-02-12T01:35:45.423Z 28 Acked 1027 Read 8 Done Done Sent false
31872024-02-12T01:35:45.423Z 29 Acked 1028 Read 7 Done Done Sent false
31882024-02-12T01:35:45.423Z 30 Acked 1029 Write 6 Sent Sent Sent false
31892024-02-12T01:35:45.423Z STATES DS:0 DS:1 DS:2 TOTAL
31902024-02-12T01:35:45.423Z New 0 0 0 0
31912024-02-12T01:35:45.423Z Sent 1 1 19 21
31922024-02-12T01:35:45.423Z Done 23 23 5 51
31932024-02-12T01:35:45.423Z Skipped 0 0 0 0
31942024-02-12T01:35:45.423Z Error 0 0 0 0
31952024-02-12T01:35:45.423ZLast Flush: 1012 1012 1005
31962024-02-12T01:35:45.423ZDownstairs last five completed: 1005 1004 1003 1002 1001
31972024-02-12T01:35:45.423ZUpstairs last five completed: 30 29 28 27 26
31982024-02-12T01:35:45.423Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:44.330297414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1247}
31992024-02-12T01:35:45.424ZWrote out file "/tmp/test_repair_verify.data"
32002024-02-12T01:35:45.427Z
32012024-02-12T01:35:45.427ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
32022024-02-12T01:35:45.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32032024-02-12T01:35:45.443Z 0 000-019 28 28 28 42 42 42 F F F
32042024-02-12T01:35:45.443Z 1 020-039 26 26 26 40 40 40 F F F
32052024-02-12T01:35:45.443Z 2 040-059 26 26 26 40 40 40 F F F
32062024-02-12T01:35:45.443Z 3 060-079 25 25 25 39 39 39 F F F
32072024-02-12T01:35:45.449Z 4 080-099 22 22 22 34 34 34 F F F
32082024-02-12T01:35:45.449Z 5 100-119 26 26 26 40 40 40 F F F
32092024-02-12T01:35:45.449Z 6 120-139 25 25 25 39 39 39 T T F <---
32102024-02-12T01:35:45.449Z 7 140-159 25 25 25 39 39 39 F F F
32112024-02-12T01:35:45.449Z 8 160-179 22 22 22 35 35 35 F F F
32122024-02-12T01:35:45.449Z 9 180-199 28 28 28 42 42 42 F F F
32132024-02-12T01:35:45.449Z 10 200-219 23 23 23 36 36 36 T T F <---
32142024-02-12T01:35:45.449Z 11 220-239 23 23 23 36 36 36 F F F
32152024-02-12T01:35:45.449Z 12 240-259 18 18 18 26 26 26 F F F
32162024-02-12T01:35:45.450Z 13 260-279 23 23 23 36 36 36 F F F
32172024-02-12T01:35:45.450Z 14 280-299 27 27 27 41 41 41 F F F
32182024-02-12T01:35:45.450Z 15 300-319 24 24 24 38 38 38 F F F
32192024-02-12T01:35:45.450Z 16 320-339 27 27 27 41 41 41 F F F
32202024-02-12T01:35:45.450Z 17 340-359 20 20 20 29 29 29 F F F
32212024-02-12T01:35:45.450Z 18 360-379 28 28 25 43 43 39 F F T <---
32222024-02-12T01:35:45.450Z 19 380-399 26 26 26 40 40 40 F F F
32232024-02-12T01:35:45.450Z 20 400-419 27 27 27 41 41 41 T T F <---
32242024-02-12T01:35:45.450Z 21 420-439 21 21 21 31 31 31 F F F
32252024-02-12T01:35:45.450Z 22 440-459 24 24 24 38 38 38 F F F
32262024-02-12T01:35:45.450Z 23 460-479 25 25 25 39 39 39 T T F <---
32272024-02-12T01:35:45.450Z 24 480-499 25 25 25 39 39 39 T T F <---
32282024-02-12T01:35:45.450Z 25 500-519 17 17 17 25 25 25 F F F
32292024-02-12T01:35:45.450Z 26 520-539 27 27 27 41 41 41 F F F
32302024-02-12T01:35:45.451Z 27 540-559 27 27 27 41 41 41 T T F <---
32312024-02-12T01:35:45.451Z 28 560-579 20 20 20 29 29 29 F F F
32322024-02-12T01:35:45.451Z 29 580-599 27 27 27 41 41 41 F F F
32332024-02-12T01:35:45.451ZMax gen: 28, Max flush: 43
32342024-02-12T01:35:45.451ZError: Difference in extent metadata found!
32352024-02-12T01:35:45.451ZA Difference in extent metadata is expected here
32362024-02-12T01:35:45.451ZOn loop 14
32372024-02-12T01:35:45.451Z
32382024-02-12T01:35:45.455ZVerifying data now
32392024-02-12T01:35:51.313ZNo change after verify
32402024-02-12T01:35:51.313ZLoop: 14 Downstairs dump after verify (and repair):
32412024-02-12T01:35:51.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32422024-02-12T01:35:51.328Z 0 000-019 28 28 28 42 42 42 F F F
32432024-02-12T01:35:51.328Z 1 020-039 26 26 26 40 40 40 F F F
32442024-02-12T01:35:51.328Z 2 040-059 26 26 26 40 40 40 F F F
32452024-02-12T01:35:51.329Z 3 060-079 25 25 25 39 39 39 F F F
32462024-02-12T01:35:51.329Z 4 080-099 22 22 22 34 34 34 F F F
32472024-02-12T01:35:51.332Z 5 100-119 26 26 26 40 40 40 F F F
32482024-02-12T01:35:51.332Z 6 120-139 29 29 29 44 44 44 F F F
32492024-02-12T01:35:51.332Z 7 140-159 25 25 25 39 39 39 F F F
32502024-02-12T01:35:51.332Z 8 160-179 22 22 22 35 35 35 F F F
32512024-02-12T01:35:51.332Z 9 180-199 28 28 28 42 42 42 F F F
32522024-02-12T01:35:51.332Z 10 200-219 29 29 29 44 44 44 F F F
32532024-02-12T01:35:51.332Z 11 220-239 23 23 23 36 36 36 F F F
32542024-02-12T01:35:51.332Z 12 240-259 18 18 18 26 26 26 F F F
32552024-02-12T01:35:51.332Z 13 260-279 23 23 23 36 36 36 F F F
32562024-02-12T01:35:51.332Z 14 280-299 27 27 27 41 41 41 F F F
32572024-02-12T01:35:51.332Z 15 300-319 24 24 24 38 38 38 F F F
32582024-02-12T01:35:51.332Z 16 320-339 27 27 27 41 41 41 F F F
32592024-02-12T01:35:51.332Z 17 340-359 20 20 20 29 29 29 F F F
32602024-02-12T01:35:51.332Z 18 360-379 28 28 28 43 43 43 F F F
32612024-02-12T01:35:51.332Z 19 380-399 26 26 26 40 40 40 F F F
32622024-02-12T01:35:51.332Z 20 400-419 29 29 29 44 44 44 F F F
32632024-02-12T01:35:51.332Z 21 420-439 21 21 21 31 31 31 F F F
32642024-02-12T01:35:51.332Z 22 440-459 24 24 24 38 38 38 F F F
32652024-02-12T01:35:51.333Z 23 460-479 29 29 29 44 44 44 F F F
32662024-02-12T01:35:51.333Z 24 480-499 29 29 29 44 44 44 F F F
32672024-02-12T01:35:51.333Z 25 500-519 17 17 17 25 25 25 F F F
32682024-02-12T01:35:51.333Z 26 520-539 27 27 27 41 41 41 F F F
32692024-02-12T01:35:51.333Z 27 540-559 29 29 29 44 44 44 F F F
32702024-02-12T01:35:51.333Z 28 560-579 20 20 20 29 29 29 F F F
32712024-02-12T01:35:51.333Z 29 580-599 27 27 27 41 41 41 F F F
32722024-02-12T01:35:51.333ZMax gen: 29, Max flush: 44
32732024-02-12T01:35:51.336Z
32742024-02-12T01:35:51.336ZBegin loop 15
32752024-02-12T01:35:51.336ZDownstairs to restart: 1
32762024-02-12T01:35:51.351Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262097603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
32772024-02-12T01:35:51.351Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262331652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
32782024-02-12T01:35:51.356Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262349041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
32792024-02-12T01:35:51.356Z{"msg":"Crucible 649962e2-3042-492e-b432-98c4def87fa0 has session id: 3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2","v":0,"name":"crucible","level":30Crucible runtime is spawned
32802024-02-12T01:35:51.356Z,"time":"2024-02-12T01:35:50.262391119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32812024-02-12T01:35:51.356Z{"msg":"Upstairs opts: Upstairs UUID: 649962e2-3042-492e-b432-98c4def87fa0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262492884Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32822024-02-12T01:35:51.356Z{"msg":"Crucible stats registered with UUID: 649962e2-3042-492e-b432-98c4def87fa0","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.26256538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32832024-02-12T01:35:51.356Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262603859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
32842024-02-12T01:35:51.356Z{"msg":"649962e2-3042-492e-b432-98c4def87fa0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262625678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32852024-02-12T01:35:51.356Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262659526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32862024-02-12T01:35:51.356Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262690894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32872024-02-12T01:35:51.356Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262720393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32882024-02-12T01:35:51.356Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262750932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32892024-02-12T01:35:51.356Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.26277643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32902024-02-12T01:35:51.356Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262811909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32912024-02-12T01:35:51.356Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262833928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32922024-02-12T01:35:51.357Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262870236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32932024-02-12T01:35:51.357Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262902714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32942024-02-12T01:35:51.357Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.262927943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32952024-02-12T01:35:51.357Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263122494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32962024-02-12T01:35:51.357Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:50.263191751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32972024-02-12T01:35:51.357Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:50.263223869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32982024-02-12T01:35:51.357Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:50.263266797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
32992024-02-12T01:35:51.357Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263299985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33002024-02-12T01:35:51.357Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263327814Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33012024-02-12T01:35:51.357Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.26341694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33022024-02-12T01:35:51.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263763553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33032024-02-12T01:35:51.357Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263801071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33042024-02-12T01:35:51.357Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.26381983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33052024-02-12T01:35:51.357Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.263899377Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33062024-02-12T01:35:51.357Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.264506068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33072024-02-12T01:35:51.357Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.264555635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33082024-02-12T01:35:51.357Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.265052462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33092024-02-12T01:35:51.357Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.26509203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33102024-02-12T01:35:51.357Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.265114649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33112024-02-12T01:35:51.357Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.265148027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33122024-02-12T01:35:51.357Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.265173726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33132024-02-12T01:35:51.357Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:50.265188335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33142024-02-12T01:35:52.354Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.264814704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33152024-02-12T01:35:52.354Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.26489904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33162024-02-12T01:35:52.358Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.265714941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33172024-02-12T01:35:52.358Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.265746659Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33182024-02-12T01:35:52.358Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266246755Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33192024-02-12T01:35:52.358Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266743981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33202024-02-12T01:35:52.358Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266794089Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33212024-02-12T01:35:52.358Z{"msg":"[0]R flush_numbers[0..12]: [42, 40, 40, 39, 34, 40, 44, 39, 35, 42, 44, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266821398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33222024-02-12T01:35:52.358Z{"msg":"[0]R generation[0..12]: [28, 26, 26, 25, 22, 26, 29, 25, 22, 28, 29, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266845267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33232024-02-12T01:35:52.358Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266860386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33242024-02-12T01:35:52.358Z{"msg":"[1]R flush_numbers[0..12]: [42, 40, 40, 39, 34, 40, 44, 39, 35, 42, 44, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266875225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33252024-02-12T01:35:52.358Z{"msg":"[1]R generation[0..12]: [28, 26, 26, 25, 22, 26, 29, 25, 22, 28, 29, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266889025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33262024-02-12T01:35:52.358Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266910324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33272024-02-12T01:35:52.358Z{"msg":"[2]R flush_numbers[0..12]: [42, 40, 40, 39, 34, 40, 44, 39, 35, 42, 44, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266924893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33282024-02-12T01:35:52.358ZWait for a query_work_queue command to finish before sending IO
33292024-02-12T01:35:52.358Z{"msg":"[2]R generation[0..12]: [28, 26, 26, 25, 22, 26, 29, 25, 22, 28, 29, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266951912Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33302024-02-12T01:35:52.358Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.266966471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33312024-02-12T01:35:52.358Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.26698595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33322024-02-12T01:35:52.358Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267000839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33332024-02-12T01:35:52.359Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267013819Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33342024-02-12T01:35:52.359Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267027288Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33352024-02-12T01:35:52.359Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267041507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33362024-02-12T01:35:52.359Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267060626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33372024-02-12T01:35:52.359Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267074586Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33382024-02-12T01:35:52.359Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267087805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33392024-02-12T01:35:52.359Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267101524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33402024-02-12T01:35:52.359Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267122343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33412024-02-12T01:35:52.359Z{"msg":"649962e2-3042-492e-b432-98c4def87fa0 is now active with session: 3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267135893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33422024-02-12T01:35:52.359Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267148052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255,"session_id":"3e58bb50-7778-4ef3-b5a6-8bd7a42f2dc2"}
33432024-02-12T01:35:52.359Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:51.267167151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
33442024-02-12T01:35:52.359ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33452024-02-12T01:35:52.359ZLoading write count information from file "/tmp/test_repair_verify.data"
33462024-02-12T01:35:52.359ZRun Repair workload
33472024-02-12T01:35:52.362Z01/30 Write block 573 len 4096 data: 4
33482024-02-12T01:35:52.362Z02/30 Write block 214 len 12288 data: 3 3 3
33492024-02-12T01:35:52.362Z03/30 Read block 351 len 40960
33502024-02-12T01:35:52.362Z04/30 Write block 547 len 16384 data: 4 4 4 5
33512024-02-12T01:35:52.362Z05/30 Write block 462 len 36864 data: 3 3 3 4 5 6 6 6 5
33522024-02-12T01:35:52.362Z06/30 Write block 278 len 24576 data: 4 4 4 3 3 3
33532024-02-12T01:35:52.365Z07/30 Write block 165 len 20480 data: 4 4 5 5 5
33542024-02-12T01:35:52.365Z08/30 Read block 215 len 20480
33552024-02-12T01:35:52.365Z09/30 Read block 390 len 20480
33562024-02-12T01:35:52.365Z10/30 Read block 512 len 16384
33572024-02-12T01:35:52.368Z11/30 Write block 152 len 36864 data: 4 3 3 2 2 2 2 2 2
33582024-02-12T01:35:52.368Z12/30 Read block 106 len 4096
33592024-02-12T01:35:52.368Z13/30 Write block 532 len 28672 data: 3 3 3 5 5 6 7
33602024-02-12T01:35:52.371Z14/30 Write block 176 len 12288 data: 3 3 4
33612024-02-12T01:35:52.371Z15/30 Read block 483 len 20480
33622024-02-12T01:35:52.371Z16/30 Write block 36 len 24576 data: 3 3 3 3 2 3
33632024-02-12T01:35:52.371Z17/30 Read block 171 len 24576
33642024-02-12T01:35:52.374Z18/30 Write block 452 len 16384 data: 3 3 4 4
33652024-02-12T01:35:52.374Z19/30 Write block 502 len 12288 data: 4 3 3
33662024-02-12T01:35:52.374Z20/30 Read block 449 len 40960
33672024-02-12T01:35:52.374Z21/30 Read block 327 len 24576
33682024-02-12T01:35:52.377Z22/30 Flush
33692024-02-12T01:35:52.418Z23/30 Write block 136 len 36864 data: 3 3 4 4 4 5 6 6 6
33702024-02-12T01:35:52.421Z24/30 Read block 561 len 20480
33712024-02-12T01:35:52.421Z25/30 Write block 176 len 24576 data: 4 4 5 4 4 4
33722024-02-12T01:35:52.421Z26/30 Read block 91 len 12288
33732024-02-12T01:35:52.424Z27/30 Read block 435 len 40960
33742024-02-12T01:35:52.424Z28/30 Write block 546 len 12288 data: 4 5 5
33752024-02-12T01:35:52.424Z29/30 Write block 559 len 4096 data: 4
33762024-02-12T01:35:52.424Z30/30 Read block 521 len 32768
33772024-02-12T01:35:52.428Z----------------------------------------------------------------
33782024-02-12T01:35:52.428Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33792024-02-12T01:35:52.428ZStates: Active Active Active
33802024-02-12T01:35:52.428ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33812024-02-12T01:35:52.428Z 1 Acked 1000 Write 1 Done Sent Done false
33822024-02-12T01:35:52.428Z 2 Acked 1001 Write 3 Done Sent Done false
33832024-02-12T01:35:52.428Z 3 Acked 1002 Read 10 Done Sent Done false
33842024-02-12T01:35:52.428Z 4 Acked 1003 Write 4 Done Sent Done false
33852024-02-12T01:35:52.428Z 5 Acked 1004 Write 9 Done Sent Done false
33862024-02-12T01:35:52.428Z 6 Acked 1005 Write 6 Done Sent Done false
33872024-02-12T01:35:52.428Z 7 Acked 1006 Write 5 Done Sent Done false
33882024-02-12T01:35:52.428Z 8 Acked 1007 Read 5 Done Sent Done false
33892024-02-12T01:35:52.428Z 9 Acked 1008 Read 5 Done Sent Done false
33902024-02-12T01:35:52.428Z 10 Acked 1009 Read 4 Done Sent Done false
33912024-02-12T01:35:52.428Z 11 Acked 1010 Write 9 Done Sent Done false
33922024-02-12T01:35:52.428Z 12 Acked 1011 Read 1 Done Sent Done false
33932024-02-12T01:35:52.429Z 13 Acked 1012 Write 7 Done Sent Done false
33942024-02-12T01:35:52.429Z 14 Acked 1013 Write 3 Done Sent Done false
33952024-02-12T01:35:52.429Z 15 Acked 1014 Read 5 Done Sent Done false
33962024-02-12T01:35:52.429Z 16 Acked 1015 Write 6 Done Sent Done false
33972024-02-12T01:35:52.429Z 17 Acked 1016 Read 6 Done Sent Done false
33982024-02-12T01:35:52.429Z 18 Acked 1017 Write 4 Done Sent Done false
33992024-02-12T01:35:52.429Z 19 Acked 1018 Write 3 Done Sent Done false
34002024-02-12T01:35:52.429Z 20 Acked 1019 Read 10 Done Sent Done false
34012024-02-12T01:35:52.429Z 21 Acked 1020 Read 6 Done Sent Done false
34022024-02-12T01:35:52.429Z 22 Acked 1021 Flush 0 Done Sent Done false
34032024-02-12T01:35:52.429Z 23 Acked 1022 Write 9 Done Sent Done false
34042024-02-12T01:35:52.429Z 24 Acked 1023 Read 5 Done Sent Done false
34052024-02-12T01:35:52.429Z 25 Acked 1024 Write 6 Done Sent Done false
34062024-02-12T01:35:52.429Z 26 Acked 1025 Read 3 Done Sent Done false
34072024-02-12T01:35:52.429Z 27 Acked 1026 Read 10 Done Sent Done false
34082024-02-12T01:35:52.429Z 28 Acked 1027 Write 3 Done Sent Done false
34092024-02-12T01:35:52.429Z 29 Acked 1028 Write 1 Done Sent Done false
34102024-02-12T01:35:52.429Z 30 Acked 1029 Read 8 Done Sent Sent false
34112024-02-12T01:35:52.429Z STATES DS:0 DS:1 DS:2 TOTAL
34122024-02-12T01:35:52.429Z New 0 0 0 0
34132024-02-12T01:35:52.429Z Sent 0 30 1 31
34142024-02-12T01:35:52.429Z Done 30 0 29 59
34152024-02-12T01:35:52.429Z Skipped 0 0 0 0
34162024-02-12T01:35:52.429Z Error 0 0 0 0
34172024-02-12T01:35:52.430ZLast Flush: 1021 0 1021
34182024-02-12T01:35:52.430ZDownstairs last five completed:
34192024-02-12T01:35:52.430ZUpstairs last five completed: 30 29 28 27 26
34202024-02-12T01:35:52.430Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:51.336379299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1255}
34212024-02-12T01:35:52.430ZWrote out file "/tmp/test_repair_verify.data"
34222024-02-12T01:35:52.433Z
34232024-02-12T01:35:52.436ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
34242024-02-12T01:35:52.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34252024-02-12T01:35:52.451Z 0 000-019 28 28 28 42 42 42 F F F
34262024-02-12T01:35:52.451Z 1 020-039 30 26 30 45 40 45 F F F <---
34272024-02-12T01:35:52.451Z 2 040-059 30 26 30 45 40 45 F F F <---
34282024-02-12T01:35:52.451Z 3 060-079 25 25 25 39 39 39 F F F
34292024-02-12T01:35:52.451Z 4 080-099 22 22 22 34 34 34 F F F
34302024-02-12T01:35:52.451Z 5 100-119 26 26 26 40 40 40 F F F
34312024-02-12T01:35:52.451Z 6 120-139 29 29 29 44 44 44 T F T <---
34322024-02-12T01:35:52.451Z 7 140-159 30 25 30 45 39 45 T F T <---
34332024-02-12T01:35:52.451Z 8 160-179 30 22 30 45 35 45 T F T <---
34342024-02-12T01:35:52.451Z 9 180-199 28 28 28 42 42 42 T F T <---
34352024-02-12T01:35:52.451Z 10 200-219 30 29 30 45 44 45 F F F <---
34362024-02-12T01:35:52.454Z 11 220-239 23 23 23 36 36 36 F F F
34372024-02-12T01:35:52.455Z 12 240-259 18 18 18 26 26 26 F F F
34382024-02-12T01:35:52.455Z 13 260-279 30 23 30 45 36 45 F F F <---
34392024-02-12T01:35:52.455Z 14 280-299 30 27 30 45 41 45 F F F <---
34402024-02-12T01:35:52.455Z 15 300-319 24 24 24 38 38 38 F F F
34412024-02-12T01:35:52.455Z 16 320-339 27 27 27 41 41 41 F F F
34422024-02-12T01:35:52.455Z 17 340-359 20 20 20 29 29 29 F F F
34432024-02-12T01:35:52.455Z 18 360-379 28 28 28 43 43 43 F F F
34442024-02-12T01:35:52.455Z 19 380-399 26 26 26 40 40 40 F F F
34452024-02-12T01:35:52.455Z 20 400-419 29 29 29 44 44 44 F F F
34462024-02-12T01:35:52.455Z 21 420-439 21 21 21 31 31 31 F F F
34472024-02-12T01:35:52.455Z 22 440-459 30 24 30 45 38 45 F F F <---
34482024-02-12T01:35:52.455Z 23 460-479 30 29 30 45 44 45 F F F <---
34492024-02-12T01:35:52.455Z 24 480-499 29 29 29 44 44 44 F F F
34502024-02-12T01:35:52.455Z 25 500-519 30 17 30 45 25 45 F F F <---
34512024-02-12T01:35:52.455Z 26 520-539 30 27 30 45 41 45 F F F <---
34522024-02-12T01:35:52.455Z 27 540-559 30 29 30 45 44 45 T F T <---
34532024-02-12T01:35:52.455Z 28 560-579 30 20 30 45 29 45 F F F <---
34542024-02-12T01:35:52.455Z 29 580-599 27 27 27 41 41 41 F F F
34552024-02-12T01:35:52.455ZMax gen: 30, Max flush: 45
34562024-02-12T01:35:52.455ZError: Difference in extent metadata found!
34572024-02-12T01:35:52.455ZA Difference in extent metadata is expected here
34582024-02-12T01:35:52.455ZOn loop 15
34592024-02-12T01:35:52.455Z
34602024-02-12T01:35:52.459ZVerifying data now
34612024-02-12T01:35:58.008ZNo change after verify
34622024-02-12T01:35:58.008ZLoop: 15 Downstairs dump after verify (and repair):
34632024-02-12T01:35:58.023ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34642024-02-12T01:35:58.023Z 0 000-019 28 28 28 42 42 42 F F F
34652024-02-12T01:35:58.024Z 1 020-039 30 30 30 45 45 45 F F F
34662024-02-12T01:35:58.024Z 2 040-059 30 30 30 45 45 45 F F F
34672024-02-12T01:35:58.024Z 3 060-079 25 25 25 39 39 39 F F F
34682024-02-12T01:35:58.024Z 4 080-099 22 22 22 34 34 34 F F F
34692024-02-12T01:35:58.024Z 5 100-119 26 26 26 40 40 40 F F F
34702024-02-12T01:35:58.024Z 6 120-139 31 31 31 46 46 46 F F F
34712024-02-12T01:35:58.024Z 7 140-159 31 31 31 46 46 46 F F F
34722024-02-12T01:35:58.024Z 8 160-179 31 31 31 46 46 46 F F F
34732024-02-12T01:35:58.024Z 9 180-199 31 31 31 46 46 46 F F F
34742024-02-12T01:35:58.028Z 10 200-219 30 30 30 45 45 45 F F F
34752024-02-12T01:35:58.028Z 11 220-239 23 23 23 36 36 36 F F F
34762024-02-12T01:35:58.028Z 12 240-259 18 18 18 26 26 26 F F F
34772024-02-12T01:35:58.028Z 13 260-279 30 30 30 45 45 45 F F F
34782024-02-12T01:35:58.028Z 14 280-299 30 30 30 45 45 45 F F F
34792024-02-12T01:35:58.028Z 15 300-319 24 24 24 38 38 38 F F F
34802024-02-12T01:35:58.028Z 16 320-339 27 27 27 41 41 41 F F F
34812024-02-12T01:35:58.028Z 17 340-359 20 20 20 29 29 29 F F F
34822024-02-12T01:35:58.028Z 18 360-379 28 28 28 43 43 43 F F F
34832024-02-12T01:35:58.028Z 19 380-399 26 26 26 40 40 40 F F F
34842024-02-12T01:35:58.028Z 20 400-419 29 29 29 44 44 44 F F F
34852024-02-12T01:35:58.028Z 21 420-439 21 21 21 31 31 31 F F F
34862024-02-12T01:35:58.028Z 22 440-459 30 30 30 45 45 45 F F F
34872024-02-12T01:35:58.028Z 23 460-479 30 30 30 45 45 45 F F F
34882024-02-12T01:35:58.028Z 24 480-499 29 29 29 44 44 44 F F F
34892024-02-12T01:35:58.028Z 25 500-519 30 30 30 45 45 45 F F F
34902024-02-12T01:35:58.028Z 26 520-539 30 30 30 45 45 45 F F F
34912024-02-12T01:35:58.028Z 27 540-559 31 31 31 46 46 46 F F F
34922024-02-12T01:35:58.028Z 28 560-579 30 30 30 45 45 45 F F F
34932024-02-12T01:35:58.028Z 29 580-599 27 27 27 41 41 41 F F F
34942024-02-12T01:35:58.028ZMax gen: 31, Max flush: 46
34952024-02-12T01:35:58.028Z
34962024-02-12T01:35:58.028ZBegin loop 16
34972024-02-12T01:35:58.029ZDownstairs to restart: 1
34982024-02-12T01:35:58.044Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954394808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
34992024-02-12T01:35:58.049Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954640927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
35002024-02-12T01:35:58.049Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
35012024-02-12T01:35:58.049Z,"time":"2024-02-12T01:35:56.954678375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
35022024-02-12T01:35:58.049Z{"msg":"Crucible 1af6e050-5a64-41ca-a17d-d4463f8483ab has session id: 971da79f-5fec-479e-ae4d-e691c5419f5b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954706733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35032024-02-12T01:35:58.049Z{"msg":"Upstairs opts: Upstairs UUID: 1af6e050-5a64-41ca-a17d-d4463f8483ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954751021Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35042024-02-12T01:35:58.049Z{"msg":"Crucible stats registered with UUID: 1af6e050-5a64-41ca-a17d-d4463f8483ab","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.95478709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35052024-02-12T01:35:58.049Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954810719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
35062024-02-12T01:35:58.049Z{"msg":"1af6e050-5a64-41ca-a17d-d4463f8483ab active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954922783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35072024-02-12T01:35:58.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.954975321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35082024-02-12T01:35:58.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955063456Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35092024-02-12T01:35:58.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955095745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35102024-02-12T01:35:58.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955122044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35112024-02-12T01:35:58.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955159502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35122024-02-12T01:35:58.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955185891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35132024-02-12T01:35:58.050Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955218569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35142024-02-12T01:35:58.050Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955255137Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35152024-02-12T01:35:58.050Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955293565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35162024-02-12T01:35:58.050Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:56.955326484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35172024-02-12T01:35:58.050Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:56.955375512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35182024-02-12T01:35:58.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.95540412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35192024-02-12T01:35:58.050Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:56.955427409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35202024-02-12T01:35:58.050Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955463807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35212024-02-12T01:35:58.050Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955490706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35222024-02-12T01:35:58.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955515595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35232024-02-12T01:35:58.050Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.955540684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35242024-02-12T01:35:58.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.956091127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35252024-02-12T01:35:58.050Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.956122256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35262024-02-12T01:35:58.050Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.956137915Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35272024-02-12T01:35:58.050Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.956189713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35282024-02-12T01:35:58.050Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.95665752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35292024-02-12T01:35:58.050Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.956786074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35302024-02-12T01:35:58.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957101069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35312024-02-12T01:35:58.050Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957148917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35322024-02-12T01:35:58.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957202654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35332024-02-12T01:35:58.050Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957318659Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35342024-02-12T01:35:58.050Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957354767Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35352024-02-12T01:35:58.050Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:56.957371836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35362024-02-12T01:35:59.047Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.95626053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35372024-02-12T01:35:59.047Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.956419313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35382024-02-12T01:35:59.052Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.957501601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35392024-02-12T01:35:59.052Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.957536179Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35402024-02-12T01:35:59.052Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958012797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35412024-02-12T01:35:59.052Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.95858065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35422024-02-12T01:35:59.052Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958621797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35432024-02-12T01:35:59.052Z{"msg":"[0]R flush_numbers[0..12]: [42, 45, 45, 39, 34, 40, 46, 46, 46, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958649126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35442024-02-12T01:35:59.052Z{"msg":"[0]R generation[0..12]: [28, 30, 30, 25, 22, 26, 31, 31, 31, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958680155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35452024-02-12T01:35:59.052Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958707393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35462024-02-12T01:35:59.052Z{"msg":"Wait for a query_work_queue command to finish before sending IO
35472024-02-12T01:35:59.052Z[1]R flush_numbers[0..12]: [42, 45, 45, 39, 34, 40, 46, 46, 46, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958740852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35482024-02-12T01:35:59.052Z{"msg":"[1]R generation[0..12]: [28, 30, 30, 25, 22, 26, 31, 31, 31, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.95877375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35492024-02-12T01:35:59.052Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958798959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35502024-02-12T01:35:59.052Z{"msg":"[2]R flush_numbers[0..12]: [42, 45, 45, 39, 34, 40, 46, 46, 46, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958823608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35512024-02-12T01:35:59.052Z{"msg":"[2]R generation[0..12]: [28, 30, 30, 25, 22, 26, 31, 31, 31, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958854506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35522024-02-12T01:35:59.052Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958887995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35532024-02-12T01:35:59.052Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958917603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35542024-02-12T01:35:59.052Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958942232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35552024-02-12T01:35:59.053Z{"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.958969111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35562024-02-12T01:35:59.053Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.9589929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35572024-02-12T01:35:59.053Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959023768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35582024-02-12T01:35:59.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959045257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35592024-02-12T01:35:59.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959073066Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35602024-02-12T01:35:59.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959096725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35612024-02-12T01:35:59.053Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959128403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35622024-02-12T01:35:59.053Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35632024-02-12T01:35:59.053Z,"time":"2024-02-12T01:35:57.959151172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35642024-02-12T01:35:59.053Z{"msg":"1af6e050-5a64-41ca-a17d-d4463f8483ab is now active with session: 971da79f-5fec-479e-ae4d-e691c5419f5b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.95918937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35652024-02-12T01:35:59.053Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959219639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263,"session_id":"971da79f-5fec-479e-ae4d-e691c5419f5b"}
35662024-02-12T01:35:59.053Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:35:57.959242248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
35672024-02-12T01:35:59.053ZLoading write count information from file "/tmp/test_repair_verify.data"
35682024-02-12T01:35:59.053ZRun Repair workload
35692024-02-12T01:35:59.053Z01/30 Write block 279 len 32768 data: 5 5 4 4 4 4 4 4
35702024-02-12T01:35:59.056Z02/30 Write block 81 len 8192 data: 6 5
35712024-02-12T01:35:59.056Z03/30 Read block 102 len 40960
35722024-02-12T01:35:59.056Z04/30 Read block 430 len 16384
35732024-02-12T01:35:59.056Z05/30 Flush
35742024-02-12T01:35:59.059Z06/30 Write block 373 len 36864 data: 4 4 5 5 5 4 3 3 3
35752024-02-12T01:35:59.059Z07/30 Flush
35762024-02-12T01:35:59.059Z08/30 Write block 113 len 8192 data: 4 6
35772024-02-12T01:35:59.062Z09/30 Write block 256 len 20480 data: 3 4 4 3 4
35782024-02-12T01:35:59.062Z10/30 Read block 78 len 32768
35792024-02-12T01:35:59.062Z11/30 Write block 445 len 24576 data: 3 2 2 2 3 3
35802024-02-12T01:35:59.068Z12/30 Write block 479 len 36864 data: 5 6 4 4 4 4 4 4 5
35812024-02-12T01:35:59.068Z13/30 Read block 256 len 24576
35822024-02-12T01:35:59.068Z14/30 Read block 145 len 4096
35832024-02-12T01:35:59.068Z15/30 Write block 37 len 4096 data: 4
35842024-02-12T01:35:59.070Z16/30 Read block 398 len 12288
35852024-02-12T01:35:59.070Z17/30 Write block 568 len 36864 data: 3 3 3 3 4 5 4 6 5
35862024-02-12T01:35:59.070Z18/30 Read block 245 len 40960
35872024-02-12T01:35:59.070Z19/30 Write block 528 len 32768 data: 3 3 3 3 4 4 4 6
35882024-02-12T01:35:59.070Z20/30 Write block 179 len 4096 data: 5
35892024-02-12T01:35:59.071Z21/30 Flush
35902024-02-12T01:35:59.098Z22/30 Write block 104 len 20480 data: 3 3 3 4 4
35912024-02-12T01:35:59.100Z23/30 Read block 526 len 36864
35922024-02-12T01:35:59.100Z24/30 Read block 75 len 8192
35932024-02-12T01:35:59.100Z25/30 Write block 132 len 28672 data: 5 4 4 3 4 4 5
35942024-02-12T01:35:59.103Z26/30 Read block 395 len 12288
35952024-02-12T01:35:59.103Z27/30 Write block 483 len 24576 data: 5 5 5 5 6 6
35962024-02-12T01:35:59.103Z28/30 Read block 140 len 36864
35972024-02-12T01:35:59.106Z29/30 Read block 532 len 12288
35982024-02-12T01:35:59.107Z30/30 Write block 447 len 8192 data: 3 3
35992024-02-12T01:35:59.107Z----------------------------------------------------------------
36002024-02-12T01:35:59.107Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
36012024-02-12T01:35:59.107ZStates: Active Active Active
36022024-02-12T01:35:59.107ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
36032024-02-12T01:35:59.107Z 1 Acked 1000 Write 8 Done Done Done false
36042024-02-12T01:35:59.107Z 2 Acked 1001 Write 2 Done Sent Done false
36052024-02-12T01:35:59.107Z 3 Acked 1002 Read 10 Done Sent Done false
36062024-02-12T01:35:59.107Z 4 Acked 1003 Read 4 Done Sent Done false
36072024-02-12T01:35:59.107Z 5 Acked 1004 Flush 0 Done Sent Done false
36082024-02-12T01:35:59.107Z 6 Acked 1005 Write 9 Done Sent Done false
36092024-02-12T01:35:59.107Z 7 Acked 1006 Flush 0 Done Sent Done false
36102024-02-12T01:35:59.107Z 8 Acked 1007 Write 2 Done Sent Done false
36112024-02-12T01:35:59.107Z 9 Acked 1008 Write 5 Done Sent Done false
36122024-02-12T01:35:59.107Z 10 Acked 1009 Read 8 Done Sent Done false
36132024-02-12T01:35:59.107Z 11 Acked 1010 Write 6 Done Sent Done false
36142024-02-12T01:35:59.107Z 12 Acked 1011 Write 9 Done Sent Done false
36152024-02-12T01:35:59.107Z 13 Acked 1012 Read 6 Done Sent Done false
36162024-02-12T01:35:59.107Z 14 Acked 1013 Read 1 Done Sent Done false
36172024-02-12T01:35:59.107Z 15 Acked 1014 Write 1 Done Sent Done false
36182024-02-12T01:35:59.107Z 16 Acked 1015 Read 3 Done Sent Done false
36192024-02-12T01:35:59.107Z 17 Acked 1016 Write 9 Done Sent Done false
36202024-02-12T01:35:59.107Z 18 Acked 1017 Read 10 Done Sent Done false
36212024-02-12T01:35:59.108Z 19 Acked 1018 Write 8 Done Sent Done false
36222024-02-12T01:35:59.108Z 20 Acked 1019 Write 1 Done Sent Done false
36232024-02-12T01:35:59.108Z 21 Acked 1020 Flush 0 Done Sent Done false
36242024-02-12T01:35:59.108Z 22 Acked 1021 Write 5 Done Sent Done false
36252024-02-12T01:35:59.108Z 23 Acked 1022 Read 9 Done Sent Done false
36262024-02-12T01:35:59.108Z 24 Acked 1023 Read 2 Done Sent Done false
36272024-02-12T01:35:59.108Z 25 Acked 1024 Write 7 Done Sent Done false
36282024-02-12T01:35:59.108Z 26 Acked 1025 Read 3 Done Sent Done false
36292024-02-12T01:35:59.108Z 27 Acked 1026 Write 6 Done Sent Done false
36302024-02-12T01:35:59.108Z 28 Acked 1027 Read 9 Done Sent Done false
36312024-02-12T01:35:59.108Z 29 Acked 1028 Read 3 Done Sent Done false
36322024-02-12T01:35:59.108Z 30 Acked 1029 Write 2 Sent Sent Sent false
36332024-02-12T01:35:59.108Z STATES DS:0 DS:1 DS:2 TOTAL
36342024-02-12T01:35:59.108Z New 0 0 0 0
36352024-02-12T01:35:59.108Z Sent 1 29 1 31
36362024-02-12T01:35:59.108Z Done 29 1 29 59
36372024-02-12T01:35:59.108Z Skipped 0 0 0 0
36382024-02-12T01:35:59.108Z Error 0 0 0 0
36392024-02-12T01:35:59.108ZLast Flush: 1020 0 1020
36402024-02-12T01:35:59.108ZDownstairs last five completed:
36412024-02-12T01:35:59.108ZUpstairs last five completed: 30 29 28 27 26
36422024-02-12T01:35:59.108Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:35:58.015852717Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1263}
36432024-02-12T01:35:59.108ZWrote out file "/tmp/test_repair_verify.data"
36442024-02-12T01:35:59.111Z
36452024-02-12T01:35:59.114ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36462024-02-12T01:35:59.131ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36472024-02-12T01:35:59.131Z 0 000-019 28 28 28 42 42 42 F F F
36482024-02-12T01:35:59.131Z 1 020-039 32 30 32 49 45 49 F F F <---
36492024-02-12T01:35:59.131Z 2 040-059 30 30 30 45 45 45 F F F
36502024-02-12T01:35:59.131Z 3 060-079 25 25 25 39 39 39 F F F
36512024-02-12T01:35:59.131Z 4 080-099 32 22 32 47 34 47 F F F <---
36522024-02-12T01:35:59.131Z 5 100-119 32 26 32 49 40 49 T F T <---
36532024-02-12T01:35:59.131Z 6 120-139 31 31 31 46 46 46 T F T <---
36542024-02-12T01:35:59.131Z 7 140-159 31 31 31 46 46 46 F F F
36552024-02-12T01:35:59.131Z 8 160-179 32 31 32 49 46 49 F F F <---
36562024-02-12T01:35:59.131Z 9 180-199 31 31 31 46 46 46 F F F
36572024-02-12T01:35:59.131Z 10 200-219 30 30 30 45 45 45 F F F
36582024-02-12T01:35:59.131Z 11 220-239 23 23 23 36 36 36 F F F
36592024-02-12T01:35:59.135Z 12 240-259 32 18 32 49 26 49 F F F <---
36602024-02-12T01:35:59.135Z 13 260-279 32 30 32 49 45 49 F T F <---
36612024-02-12T01:35:59.135Z 14 280-299 32 30 32 47 45 47 F T F <---
36622024-02-12T01:35:59.135Z 15 300-319 24 24 24 38 38 38 F F F
36632024-02-12T01:35:59.135Z 16 320-339 27 27 27 41 41 41 F F F
36642024-02-12T01:35:59.135Z 17 340-359 20 20 20 29 29 29 F F F
36652024-02-12T01:35:59.135Z 18 360-379 32 28 32 48 43 48 F F F <---
36662024-02-12T01:35:59.135Z 19 380-399 32 26 32 48 40 48 F F F <---
36672024-02-12T01:35:59.135Z 20 400-419 29 29 29 44 44 44 F F F
36682024-02-12T01:35:59.135Z 21 420-439 21 21 21 31 31 31 F F F
36692024-02-12T01:35:59.135Z 22 440-459 32 30 32 49 45 49 T F T <---
36702024-02-12T01:35:59.135Z 23 460-479 32 30 32 49 45 49 F F F <---
36712024-02-12T01:35:59.135Z 24 480-499 32 29 32 49 44 49 T F T <---
36722024-02-12T01:35:59.135Z 25 500-519 30 30 30 45 45 45 F F F
36732024-02-12T01:35:59.135Z 26 520-539 32 30 32 49 45 49 F F F <---
36742024-02-12T01:35:59.135Z 27 540-559 31 31 31 46 46 46 F F F
36752024-02-12T01:35:59.135Z 28 560-579 32 30 32 49 45 49 F F F <---
36762024-02-12T01:35:59.135Z 29 580-599 27 27 27 41 41 41 F F F
36772024-02-12T01:35:59.135ZMax gen: 32, Max flush: 49
36782024-02-12T01:35:59.136ZError: Difference in extent metadata found!
36792024-02-12T01:35:59.136ZA Difference in extent metadata is expected here
36802024-02-12T01:35:59.136ZOn loop 16
36812024-02-12T01:35:59.136Z
36822024-02-12T01:35:59.139ZVerifying data now
36832024-02-12T01:36:05.831ZNo change after verify
36842024-02-12T01:36:05.831ZLoop: 16 Downstairs dump after verify (and repair):
36852024-02-12T01:36:05.847ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36862024-02-12T01:36:05.847Z 0 000-019 28 28 28 42 42 42 F F F
36872024-02-12T01:36:05.847Z 1 020-039 32 32 32 49 49 49 F F F
36882024-02-12T01:36:05.847Z 2 040-059 30 30 30 45 45 45 F F F
36892024-02-12T01:36:05.847Z 3 060-079 25 25 25 39 39 39 F F F
36902024-02-12T01:36:05.847Z 4 080-099 32 32 32 47 47 47 F F F
36912024-02-12T01:36:05.847Z 5 100-119 33 33 33 50 50 50 F F F
36922024-02-12T01:36:05.847Z 6 120-139 33 33 33 50 50 50 F F F
36932024-02-12T01:36:05.847Z 7 140-159 31 31 31 46 46 46 F F F
36942024-02-12T01:36:05.847Z 8 160-179 32 32 32 49 49 49 F F F
36952024-02-12T01:36:05.847Z 9 180-199 31 31 31 46 46 46 F F F
36962024-02-12T01:36:05.847Z 10 200-219 30 30 30 45 45 45 F F F
36972024-02-12T01:36:05.847Z 11 220-239 23 23 23 36 36 36 F F F
36982024-02-12T01:36:05.853Z 12 240-259 32 32 32 49 49 49 F F F
36992024-02-12T01:36:05.853Z 13 260-279 32 32 32 49 49 49 F F F
37002024-02-12T01:36:05.853Z 14 280-299 32 32 32 47 47 47 F F F
37012024-02-12T01:36:05.853Z 15 300-319 24 24 24 38 38 38 F F F
37022024-02-12T01:36:05.853Z 16 320-339 27 27 27 41 41 41 F F F
37032024-02-12T01:36:05.853Z 17 340-359 20 20 20 29 29 29 F F F
37042024-02-12T01:36:05.853Z 18 360-379 32 32 32 48 48 48 F F F
37052024-02-12T01:36:05.853Z 19 380-399 32 32 32 48 48 48 F F F
37062024-02-12T01:36:05.853Z 20 400-419 29 29 29 44 44 44 F F F
37072024-02-12T01:36:05.853Z 21 420-439 21 21 21 31 31 31 F F F
37082024-02-12T01:36:05.853Z 22 440-459 33 33 33 50 50 50 F F F
37092024-02-12T01:36:05.853Z 23 460-479 32 32 32 49 49 49 F F F
37102024-02-12T01:36:05.854Z 24 480-499 33 33 33 50 50 50 F F F
37112024-02-12T01:36:05.854Z 25 500-519 30 30 30 45 45 45 F F F
37122024-02-12T01:36:05.854Z 26 520-539 32 32 32 49 49 49 F F F
37132024-02-12T01:36:05.854Z 27 540-559 31 31 31 46 46 46 F F F
37142024-02-12T01:36:05.854Z 28 560-579 32 32 32 49 49 49 F F F
37152024-02-12T01:36:05.854Z 29 580-599 27 27 27 41 41 41 F F F
37162024-02-12T01:36:05.854ZMax gen: 33, Max flush: 50
37172024-02-12T01:36:05.854Z
37182024-02-12T01:36:05.854ZBegin loop 17
37192024-02-12T01:36:05.854ZDownstairs to restart: 0
37202024-02-12T01:36:05.868Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.777778302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
37212024-02-12T01:36:05.874Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
37222024-02-12T01:36:05.874Z,"time":"2024-02-12T01:36:04.778138065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
37232024-02-12T01:36:05.874Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778181143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
37242024-02-12T01:36:05.874Z{"msg":"Crucible 17dc2937-fd0d-4e86-8cdd-14aafb7b732b has session id: dede075c-a978-4b77-a3d1-fd148b106d75","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778210732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37252024-02-12T01:36:05.874Z{"msg":"Upstairs opts: Upstairs UUID: 17dc2937-fd0d-4e86-8cdd-14aafb7b732b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778265969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37262024-02-12T01:36:05.874Z{"msg":"Crucible stats registered with UUID: 17dc2937-fd0d-4e86-8cdd-14aafb7b732b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778311387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37272024-02-12T01:36:05.874Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778343065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
37282024-02-12T01:36:05.874Z{"msg":"17dc2937-fd0d-4e86-8cdd-14aafb7b732b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778371724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37292024-02-12T01:36:05.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778411512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37302024-02-12T01:36:05.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.77845478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37312024-02-12T01:36:05.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778500398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37322024-02-12T01:36:05.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778555465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37332024-02-12T01:36:05.874Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778593443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37342024-02-12T01:36:05.874Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778627602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37352024-02-12T01:36:05.874Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.77866929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37362024-02-12T01:36:05.874Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778701858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37372024-02-12T01:36:05.874Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778737827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37382024-02-12T01:36:05.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778763045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37392024-02-12T01:36:05.875Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:04.778799594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37402024-02-12T01:36:05.875Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:04.778828062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37412024-02-12T01:36:05.875Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:04.778861161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37422024-02-12T01:36:05.875Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.77888087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37432024-02-12T01:36:05.875Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778911138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37442024-02-12T01:36:05.875Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778926867Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37452024-02-12T01:36:05.875Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.778958596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37462024-02-12T01:36:05.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.779273871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37472024-02-12T01:36:05.875Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.779316039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37482024-02-12T01:36:05.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.779336138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37492024-02-12T01:36:05.875Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.779359247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37502024-02-12T01:36:05.875Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.779868822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37512024-02-12T01:36:05.875Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780037774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37522024-02-12T01:36:05.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780286473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37532024-02-12T01:36:05.875Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780328451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37542024-02-12T01:36:05.875Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780357189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37552024-02-12T01:36:05.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780463834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37562024-02-12T01:36:05.875Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780504582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37572024-02-12T01:36:05.875Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:04.780528051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37582024-02-12T01:36:06.871Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.780129171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37592024-02-12T01:36:06.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.780262465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37602024-02-12T01:36:06.875Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.781308835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37612024-02-12T01:36:06.875Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.781346733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37622024-02-12T01:36:06.875Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.781820571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37632024-02-12T01:36:06.875Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782346835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37642024-02-12T01:36:06.875Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782393693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37652024-02-12T01:36:06.875Z{"msg":"[0]R flush_numbers[0..12]: [42, 49, 45, 39, 47, 50, 50, 46, 49, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782413132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37662024-02-12T01:36:06.875Z{"msg":"[0]R generation[0..12]: [28, 32, 30, 25, 32, 33, 33, 31, 32, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.78245307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37672024-02-12T01:36:06.875Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782496378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37682024-02-12T01:36:06.875Z{"msg":"[1]R flush_numbers[0..12]: [42, 49, 45, 39, 47, 50, 50, 46, 49, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782537476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37692024-02-12T01:36:06.876Z{"msg":"Wait for a query_work_queue command to finish before sending IO
37702024-02-12T01:36:06.876Z[1]R generation[0..12]: [28, 32, 30, 25, 32, 33, 33, 31, 32, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782578214Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37712024-02-12T01:36:06.876Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782606403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37722024-02-12T01:36:06.876Z{"msg":"[2]R flush_numbers[0..12]: [42, 49, 45, 39, 47, 50, 50, 46, 49, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782641401Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37732024-02-12T01:36:06.876Z{"msg":"[2]R generation[0..12]: [28, 32, 30, 25, 32, 33, 33, 31, 32, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.78267777Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37742024-02-12T01:36:06.876Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782707908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37752024-02-12T01:36:06.876Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782722657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37762024-02-12T01:36:06.876Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782752956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37772024-02-12T01:36:06.876Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782775115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37782024-02-12T01:36:06.876Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782802753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37792024-02-12T01:36:06.876Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782818603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37802024-02-12T01:36:06.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782848161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37812024-02-12T01:36:06.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.78286575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37822024-02-12T01:36:06.876Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782895579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37832024-02-12T01:36:06.876Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782913458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37842024-02-12T01:36:06.876Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782941947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37852024-02-12T01:36:06.876Z{"msg":"17dc2937-fd0d-4e86-8cdd-14aafb7b732b is now active with session: dede075c-a978-4b77-a3d1-fd148b106d75","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782956656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37862024-02-12T01:36:06.876ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37872024-02-12T01:36:06.876Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.782987815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271,"session_id":"dede075c-a978-4b77-a3d1-fd148b106d75"}
37882024-02-12T01:36:06.876Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:05.783003584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
37892024-02-12T01:36:06.880ZLoading write count information from file "/tmp/test_repair_verify.data"
37902024-02-12T01:36:06.880ZRun Repair workload
37912024-02-12T01:36:06.880Z01/30 Write block 456 len 20480 data: 3 3 4 4 3
37922024-02-12T01:36:06.880Z02/30 Read block 226 len 40960
37932024-02-12T01:36:06.880Z03/30 Read block 531 len 28672
37942024-02-12T01:36:06.880Z04/30 Write block 551 len 32768 data: 4 4 3 3 2 4 4 3
37952024-02-12T01:36:06.880Z05/30 Write block 72 len 20480 data: 3 3 3 4 6
37962024-02-12T01:36:06.880Z06/30 Read block 83 len 32768
37972024-02-12T01:36:06.883Z07/30 Write block 299 len 8192 data: 3 3
37982024-02-12T01:36:06.883Z08/30 Read block 527 len 24576
37992024-02-12T01:36:06.883Z09/30 Read block 512 len 20480
38002024-02-12T01:36:06.885Z10/30 Write block 363 len 12288 data: 3 5 7
38012024-02-12T01:36:06.886Z11/30 Read block 407 len 28672
38022024-02-12T01:36:06.886Z12/30 Read block 322 len 4096
38032024-02-12T01:36:06.886Z13/30 Read block 519 len 32768
38042024-02-12T01:36:06.889Z14/30 Write block 573 len 4096 data: 6
38052024-02-12T01:36:06.889Z15/30 Write block 283 len 24576 data: 5 5 5 5 3 3
38062024-02-12T01:36:06.889Z16/30 Write block 24 len 36864 data: 3 3 3 4 5 6 5 4 6
38072024-02-12T01:36:06.889Z17/30 Write block 54 len 4096 data: 2
38082024-02-12T01:36:06.892Z18/30 Write block 123 len 40960 data: 4 4 4 6 6 5 5 5 6 6
38092024-02-12T01:36:06.892Z19/30 Flush
38102024-02-12T01:36:06.917Z20/30 Write block 58 len 32768 data: 5 5 5 5 6 5 5 5
38112024-02-12T01:36:06.921Z21/30 Write block 88 len 16384 data: 5 5 4 4
38122024-02-12T01:36:06.921Z22/30 Flush
38132024-02-12T01:36:06.924Z23/30 Write block 98 len 8192 data: 3 2
38142024-02-12T01:36:06.924Z24/30 Write block 166 len 4096 data: 5
38152024-02-12T01:36:06.924Z25/30 Flush
38162024-02-12T01:36:06.926Z26/30 Write block 161 len 20480 data: 2 2 2 3 5
38172024-02-12T01:36:06.927Z27/30 Write block 345 len 12288 data: 4 4 4
38182024-02-12T01:36:06.927Z28/30 Write block 381 len 16384 data: 4 3 4 3
38192024-02-12T01:36:06.927Z29/30 Write block 574 len 8192 data: 5 7
38202024-02-12T01:36:06.927Z30/30 Read block 372 len 28672
38212024-02-12T01:36:06.930Z----------------------------------------------------------------
38222024-02-12T01:36:06.930Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
38232024-02-12T01:36:06.930ZStates: Active Active Active
38242024-02-12T01:36:06.930ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38252024-02-12T01:36:06.930Z 1 Acked 1000 Write 5 Done Done Done false
38262024-02-12T01:36:06.930Z 2 Acked 1001 Read 10 Done Done Done false
38272024-02-12T01:36:06.930Z 3 Acked 1002 Read 7 Done Done Done false
38282024-02-12T01:36:06.930Z 4 Acked 1003 Write 8 Done Done Done false
38292024-02-12T01:36:06.930Z 5 Acked 1004 Write 5 Done Done Done false
38302024-02-12T01:36:06.930Z 6 Acked 1005 Read 8 Done Done Done false
38312024-02-12T01:36:06.930Z 7 Acked 1006 Write 2 Done Done Done false
38322024-02-12T01:36:06.931Z 8 Acked 1007 Read 6 Sent Done Done false
38332024-02-12T01:36:06.931Z 9 Acked 1008 Read 5 Sent Done Done false
38342024-02-12T01:36:06.931Z 10 Acked 1009 Write 3 Sent Done Done false
38352024-02-12T01:36:06.931Z 11 Acked 1010 Read 7 Sent Done Done false
38362024-02-12T01:36:06.931Z 12 Acked 1011 Read 1 Sent Done Done false
38372024-02-12T01:36:06.931Z 13 Acked 1012 Read 8 Sent Done Done false
38382024-02-12T01:36:06.931Z 14 Acked 1013 Write 1 Sent Done Done false
38392024-02-12T01:36:06.931Z 15 Acked 1014 Write 6 Sent Done Done false
38402024-02-12T01:36:06.931Z 16 Acked 1015 Write 9 Sent Done Done false
38412024-02-12T01:36:06.931Z 17 Acked 1016 Write 1 Sent Done Done false
38422024-02-12T01:36:06.931Z 18 Acked 1017 Write 10 Sent Done Done false
38432024-02-12T01:36:06.931Z 19 Acked 1018 Flush 0 Sent Done Done false
38442024-02-12T01:36:06.931Z 20 Acked 1019 Write 8 Sent Done Done false
38452024-02-12T01:36:06.931Z 21 Acked 1020 Write 4 Sent Done Done false
38462024-02-12T01:36:06.931Z 22 Acked 1021 Flush 0 Sent Done Done false
38472024-02-12T01:36:06.931Z 23 Acked 1022 Write 2 Sent Done Done false
38482024-02-12T01:36:06.931Z 24 Acked 1023 Write 1 Sent Done Done false
38492024-02-12T01:36:06.931Z 25 Acked 1024 Flush 0 Sent Done Done false
38502024-02-12T01:36:06.931Z 26 Acked 1025 Write 5 Sent Done Done false
38512024-02-12T01:36:06.931Z 27 Acked 1026 Write 3 Sent Done Done false
38522024-02-12T01:36:06.931Z 28 Acked 1027 Write 4 Sent Done Done false
38532024-02-12T01:36:06.931Z 29 Acked 1028 Write 2 Sent Done Done false
38542024-02-12T01:36:06.931Z 30 Acked 1029 Read 7 Sent Done Sent false
38552024-02-12T01:36:06.931Z STATES DS:0 DS:1 DS:2 TOTAL
38562024-02-12T01:36:06.931Z New 0 0 0 0
38572024-02-12T01:36:06.932Z Sent 23 0 1 24
38582024-02-12T01:36:06.932Z Done 7 30 29 66
38592024-02-12T01:36:06.932Z Skipped 0 0 0 0
38602024-02-12T01:36:06.932Z Error 0 0 0 0
38612024-02-12T01:36:06.932ZLast Flush: 0 1024 1024
38622024-02-12T01:36:06.932ZDownstairs last five completed:
38632024-02-12T01:36:06.932ZUpstairs last five completed: 30 29 28 27 26
38642024-02-12T01:36:06.932Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:05.836996598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1271}
38652024-02-12T01:36:06.932ZWrote out file "/tmp/test_repair_verify.data"
38662024-02-12T01:36:06.938Z
38672024-02-12T01:36:06.942ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38682024-02-12T01:36:06.953ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38692024-02-12T01:36:06.954Z 0 000-019 28 28 28 42 42 42 F F F
38702024-02-12T01:36:06.954Z 1 020-039 32 34 34 49 51 51 F F F <---
38712024-02-12T01:36:06.954Z 2 040-059 30 34 34 45 52 52 F F F <---
38722024-02-12T01:36:06.954Z 3 060-079 25 34 34 39 52 52 T F F <---
38732024-02-12T01:36:06.954Z 4 080-099 32 34 34 47 53 53 F F F <---
38742024-02-12T01:36:06.954Z 5 100-119 33 33 33 50 50 50 F F F
38752024-02-12T01:36:06.957Z 6 120-139 33 34 34 50 51 51 F F F <---
38762024-02-12T01:36:06.957Z 7 140-159 31 31 31 46 46 46 F F F
38772024-02-12T01:36:06.957Z 8 160-179 32 34 34 49 53 53 F T T <---
38782024-02-12T01:36:06.957Z 9 180-199 31 31 31 46 46 46 F F F
38792024-02-12T01:36:06.957Z 10 200-219 30 30 30 45 45 45 F F F
38802024-02-12T01:36:06.957Z 11 220-239 23 23 23 36 36 36 F F F
38812024-02-12T01:36:06.957Z 12 240-259 32 32 32 49 49 49 F F F
38822024-02-12T01:36:06.957Z 13 260-279 32 32 32 49 49 49 F F F
38832024-02-12T01:36:06.957Z 14 280-299 32 34 34 47 51 51 T F F <---
38842024-02-12T01:36:06.957Z 15 300-319 24 34 34 38 51 51 T F F <---
38852024-02-12T01:36:06.957Z 16 320-339 27 27 27 41 41 41 F F F
38862024-02-12T01:36:06.957Z 17 340-359 20 20 20 29 29 29 F T T <---
38872024-02-12T01:36:06.957Z 18 360-379 32 34 34 48 51 51 F F F <---
38882024-02-12T01:36:06.957Z 19 380-399 32 32 32 48 48 48 F T T <---
38892024-02-12T01:36:06.957Z 20 400-419 29 29 29 44 44 44 F F F
38902024-02-12T01:36:06.957Z 21 420-439 21 21 21 31 31 31 F F F
38912024-02-12T01:36:06.957Z 22 440-459 33 34 34 50 51 51 T F F <---
38922024-02-12T01:36:06.957Z 23 460-479 32 34 34 49 51 51 T F F <---
38932024-02-12T01:36:06.957Z 24 480-499 33 33 33 50 50 50 F F F
38942024-02-12T01:36:06.957Z 25 500-519 30 30 30 45 45 45 F F F
38952024-02-12T01:36:06.958Z 26 520-539 32 32 32 49 49 49 F F F
38962024-02-12T01:36:06.958Z 27 540-559 31 34 34 46 51 51 T F F <---
38972024-02-12T01:36:06.958Z 28 560-579 32 34 34 49 51 51 F T T <---
38982024-02-12T01:36:06.958Z 29 580-599 27 27 27 41 41 41 F F F
38992024-02-12T01:36:06.958ZMax gen: 34, Max flush: 53
39002024-02-12T01:36:06.958ZError: Difference in extent metadata found!
39012024-02-12T01:36:06.958ZA Difference in extent metadata is expected here
39022024-02-12T01:36:06.958ZOn loop 17
39032024-02-12T01:36:06.958Z
39042024-02-12T01:36:06.961ZVerifying data now
39052024-02-12T01:36:13.139ZNo change after verify
39062024-02-12T01:36:13.139ZLoop: 17 Downstairs dump after verify (and repair):
39072024-02-12T01:36:13.155ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39082024-02-12T01:36:13.155Z 0 000-019 28 28 28 42 42 42 F F F
39092024-02-12T01:36:13.155Z 1 020-039 34 34 34 51 51 51 F F F
39102024-02-12T01:36:13.155Z 2 040-059 34 34 34 52 52 52 F F F
39112024-02-12T01:36:13.155Z 3 060-079 34 34 34 52 52 52 F F F
39122024-02-12T01:36:13.155Z 4 080-099 34 34 34 53 53 53 F F F
39132024-02-12T01:36:13.155Z 5 100-119 33 33 33 50 50 50 F F F
39142024-02-12T01:36:13.155Z 6 120-139 34 34 34 51 51 51 F F F
39152024-02-12T01:36:13.155Z 7 140-159 31 31 31 46 46 46 F F F
39162024-02-12T01:36:13.155Z 8 160-179 35 35 35 54 54 54 F F F
39172024-02-12T01:36:13.155Z 9 180-199 31 31 31 46 46 46 F F F
39182024-02-12T01:36:13.161Z 10 200-219 30 30 30 45 45 45 F F F
39192024-02-12T01:36:13.161Z 11 220-239 23 23 23 36 36 36 F F F
39202024-02-12T01:36:13.161Z 12 240-259 32 32 32 49 49 49 F F F
39212024-02-12T01:36:13.162Z 13 260-279 32 32 32 49 49 49 F F F
39222024-02-12T01:36:13.162Z 14 280-299 34 34 34 51 51 51 F F F
39232024-02-12T01:36:13.162Z 15 300-319 34 34 34 51 51 51 F F F
39242024-02-12T01:36:13.162Z 16 320-339 27 27 27 41 41 41 F F F
39252024-02-12T01:36:13.162Z 17 340-359 35 35 35 54 54 54 F F F
39262024-02-12T01:36:13.162Z 18 360-379 34 34 34 51 51 51 F F F
39272024-02-12T01:36:13.162Z 19 380-399 35 35 35 54 54 54 F F F
39282024-02-12T01:36:13.162Z 20 400-419 29 29 29 44 44 44 F F F
39292024-02-12T01:36:13.162Z 21 420-439 21 21 21 31 31 31 F F F
39302024-02-12T01:36:13.162Z 22 440-459 34 34 34 51 51 51 F F F
39312024-02-12T01:36:13.162Z 23 460-479 34 34 34 51 51 51 F F F
39322024-02-12T01:36:13.162Z 24 480-499 33 33 33 50 50 50 F F F
39332024-02-12T01:36:13.162Z 25 500-519 30 30 30 45 45 45 F F F
39342024-02-12T01:36:13.162Z 26 520-539 32 32 32 49 49 49 F F F
39352024-02-12T01:36:13.162Z 27 540-559 34 34 34 51 51 51 F F F
39362024-02-12T01:36:13.162Z 28 560-579 35 35 35 54 54 54 F F F
39372024-02-12T01:36:13.162Z 29 580-599 27 27 27 41 41 41 F F F
39382024-02-12T01:36:13.162ZMax gen: 35, Max flush: 54
39392024-02-12T01:36:13.162Z
39402024-02-12T01:36:13.162ZBegin loop 18
39412024-02-12T01:36:13.162ZDownstairs to restart: 0
39422024-02-12T01:36:13.177Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085136946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
39432024-02-12T01:36:13.177Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085419662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
39442024-02-12T01:36:13.177ZCrucible runtime is spawned
39452024-02-12T01:36:13.182Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.08546743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
39462024-02-12T01:36:13.182Z{"msg":"Crucible 6a1fc15e-a7c6-4551-aef8-1b2745b63349 has session id: 0161d25c-7e8f-4dae-9a70-545160592f3b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085490149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39472024-02-12T01:36:13.182Z{"msg":"Upstairs opts: Upstairs UUID: 6a1fc15e-a7c6-4551-aef8-1b2745b63349, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085522657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39482024-02-12T01:36:13.182Z{"msg":"Crucible stats registered with UUID: 6a1fc15e-a7c6-4551-aef8-1b2745b63349","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085563805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39492024-02-12T01:36:13.182Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085584704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
39502024-02-12T01:36:13.182Z{"msg":"6a1fc15e-a7c6-4551-aef8-1b2745b63349 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085600564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39512024-02-12T01:36:13.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085629762Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39522024-02-12T01:36:13.182Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085653081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39532024-02-12T01:36:13.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085691219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39542024-02-12T01:36:13.182Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085715088Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39552024-02-12T01:36:13.182Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085753376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39562024-02-12T01:36:13.183Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085779345Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39572024-02-12T01:36:13.183Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085821073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39582024-02-12T01:36:13.183Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085853421Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39592024-02-12T01:36:13.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085967176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39602024-02-12T01:36:13.183Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.085999505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39612024-02-12T01:36:13.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.086024813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39622024-02-12T01:36:13.183Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.086184426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39632024-02-12T01:36:13.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.086650683Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39642024-02-12T01:36:13.183Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.086691672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39652024-02-12T01:36:13.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.08671887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39662024-02-12T01:36:13.183Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.086753978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39672024-02-12T01:36:13.183Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087170129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39682024-02-12T01:36:13.183Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087289943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39692024-02-12T01:36:13.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087655746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39702024-02-12T01:36:13.183Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087701283Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39712024-02-12T01:36:13.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087754401Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39722024-02-12T01:36:13.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087801019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39732024-02-12T01:36:13.183Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087820387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39742024-02-12T01:36:13.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.087843277Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39752024-02-12T01:36:13.183Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088047797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39762024-02-12T01:36:13.183Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088083595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39772024-02-12T01:36:13.183Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088540493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39782024-02-12T01:36:13.188Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088941484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39792024-02-12T01:36:13.188Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088975032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39802024-02-12T01:36:13.188Z{"msg":"[0]R flush_numbers[0..12]: [42, 51, 52, 52, 53, 50, 51, 46, 54, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.088989962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39812024-02-12T01:36:13.188Z{"msg":"[0]R generation[0..12]: [28, 34, 34, 34, 34, 33, 34, 31, 35, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089004411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39822024-02-12T01:36:13.188Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.08902141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39832024-02-12T01:36:13.188Z{"msg":"[1]R flush_numbers[0..12]: [42, 51, 52, 52, 53, 50, 51, 46, 54, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089036149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39842024-02-12T01:36:13.188Z{"msg":"[1]R generation[0..12]: [28, 34, 34, 34, 34, 33, 34, 31, 35, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089050879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39852024-02-12T01:36:13.188Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089065328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39862024-02-12T01:36:13.188Z{"msg":"[2]R flush_numbers[0..12]: [42, 51, 52, 52, 53, 50, 51, 46, 54, 46, 45, 36]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089086967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39872024-02-12T01:36:13.188Z{"msg":"[2]R generation[0..12]: [28, 34, 34, 34, 34, 33, 34, 31, 35, 31, 30, 23]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089103666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39882024-02-12T01:36:13.189Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39892024-02-12T01:36:13.189Z,"time":"2024-02-12T01:36:12.089139555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39902024-02-12T01:36:13.189Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089177073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39912024-02-12T01:36:13.189Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089199462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39922024-02-12T01:36:13.189Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089213281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39932024-02-12T01:36:13.189Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089227531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39942024-02-12T01:36:13.189Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089253029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39952024-02-12T01:36:13.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089295837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39962024-02-12T01:36:13.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089338205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39972024-02-12T01:36:13.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089383503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39982024-02-12T01:36:13.189Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089412292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
39992024-02-12T01:36:13.189Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.08944561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
40002024-02-12T01:36:13.189Z{"msg":"6a1fc15e-a7c6-4551-aef8-1b2745b63349 is now active with session: 0161d25c-7e8f-4dae-9a70-545160592f3b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089470809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
40012024-02-12T01:36:13.189Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089502107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279,"session_id":"0161d25c-7e8f-4dae-9a70-545160592f3b"}
40022024-02-12T01:36:13.189Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:12.089537126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
40032024-02-12T01:36:13.189ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
40042024-02-12T01:36:13.189ZLoading write count information from file "/tmp/test_repair_verify.data"
40052024-02-12T01:36:13.189ZRun Repair workload
40062024-02-12T01:36:13.189Z01/30 Write block 388 len 4096 data: 6
40072024-02-12T01:36:13.189Z02/30 Write block 410 len 32768 data: 5 6 5 6 4 5 4 4
40082024-02-12T01:36:13.189Z03/30 Write block 30 len 28672 data: 6 5 7 6 6 3 4
40092024-02-12T01:36:13.189Z04/30 Read block 385 len 16384
40102024-02-12T01:36:13.189Z05/30 Read block 196 len 4096
40112024-02-12T01:36:13.190Z06/30 Write block 581 len 12288 data: 3 3 3
40122024-02-12T01:36:13.190Z07/30 Read block 559 len 40960
40132024-02-12T01:36:13.193Z08/30 Read block 65 len 32768
40142024-02-12T01:36:13.193Z09/30 Write block 550 len 24576 data: 6 5 5 4 4 3
40152024-02-12T01:36:13.193Z10/30 Read block 372 len 12288
40162024-02-12T01:36:13.193Z11/30 Write block 187 len 12288 data: 5 7 7
40172024-02-12T01:36:13.193Z12/30 Read block 59 len 40960
40182024-02-12T01:36:13.193Z13/30 Read block 214 len 36864
40192024-02-12T01:36:13.196Z14/30 Write block 285 len 4096 data: 6
40202024-02-12T01:36:13.196Z15/30 Read block 218 len 8192
40212024-02-12T01:36:13.196Z16/30 Flush
40222024-02-12T01:36:13.200Z17/30 Write block 69 len 28672 data: 3 2 3 4 4 4 5
40232024-02-12T01:36:13.200Z18/30 Write block 390 len 4096 data: 8
40242024-02-12T01:36:13.200Z19/30 Flush
40252024-02-12T01:36:13.203Z20/30 Write block 265 len 28672 data: 4 4 4 3 2 2 3
40262024-02-12T01:36:13.203Z21/30 Write block 559 len 20480 data: 5 4 4 4 5
40272024-02-12T01:36:13.203Z22/30 Read block 370 len 20480
40282024-02-12T01:36:13.203Z23/30 Read block 85 len 40960
40292024-02-12T01:36:13.203Z24/30 Read block 114 len 8192
40302024-02-12T01:36:13.206Z25/30 Write block 233 len 8192 data: 3 3
40312024-02-12T01:36:13.207Z26/30 Write block 395 len 12288 data: 3 2 2
40322024-02-12T01:36:13.207Z27/30 Write block 11 len 8192 data: 3 4
40332024-02-12T01:36:13.207Z28/30 Read block 542 len 24576
40342024-02-12T01:36:13.207Z29/30 Read block 385 len 12288
40352024-02-12T01:36:13.210Z30/30 Write block 124 len 20480 data: 5 5 7 7 6
40362024-02-12T01:36:13.210Z----------------------------------------------------------------
40372024-02-12T01:36:13.210Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
40382024-02-12T01:36:13.210ZStates: Active Active Active
40392024-02-12T01:36:13.210ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40402024-02-12T01:36:13.210Z 1 Acked 1000 Write 1 Done Done Done false
40412024-02-12T01:36:13.210Z 2 Acked 1001 Write 8 Sent Done Done false
40422024-02-12T01:36:13.210Z 3 Acked 1002 Write 7 Sent Done Done false
40432024-02-12T01:36:13.210Z 4 Acked 1003 Read 4 Sent Done Done false
40442024-02-12T01:36:13.210Z 5 Acked 1004 Read 1 Sent Done Done false
40452024-02-12T01:36:13.210Z 6 Acked 1005 Write 3 Sent Done Done false
40462024-02-12T01:36:13.210Z 7 Acked 1006 Read 10 Sent Done Done false
40472024-02-12T01:36:13.210Z 8 Acked 1007 Read 8 Sent Done Done false
40482024-02-12T01:36:13.210Z 9 Acked 1008 Write 6 Sent Done Done false
40492024-02-12T01:36:13.210Z 10 Acked 1009 Read 3 Sent Done Done false
40502024-02-12T01:36:13.211Z 11 Acked 1010 Write 3 Sent Done Done false
40512024-02-12T01:36:13.211Z 12 Acked 1011 Read 10 Sent Done Done false
40522024-02-12T01:36:13.211Z 13 Acked 1012 Read 9 Sent Done Done false
40532024-02-12T01:36:13.211Z 14 Acked 1013 Write 1 Sent Done Done false
40542024-02-12T01:36:13.211Z 15 Acked 1014 Read 2 Sent Done Done false
40552024-02-12T01:36:13.211Z 16 Acked 1015 Flush 0 Sent Done Done false
40562024-02-12T01:36:13.211Z 17 Acked 1016 Write 7 Sent Done Done false
40572024-02-12T01:36:13.211Z 18 Acked 1017 Write 1 Sent Done Done false
40582024-02-12T01:36:13.211Z 19 Acked 1018 Flush 0 Sent Done Done false
40592024-02-12T01:36:13.211Z 20 Acked 1019 Write 7 Sent Done Done false
40602024-02-12T01:36:13.211Z 21 Acked 1020 Write 5 Sent Done Done false
40612024-02-12T01:36:13.211Z 22 Acked 1021 Read 5 Sent Done Done false
40622024-02-12T01:36:13.211Z 23 Acked 1022 Read 10 Sent Done Done false
40632024-02-12T01:36:13.211Z 24 Acked 1023 Read 2 Sent Done Done false
40642024-02-12T01:36:13.211Z 25 Acked 1024 Write 2 Sent Done Done false
40652024-02-12T01:36:13.211Z 26 Acked 1025 Write 3 Sent Done Done false
40662024-02-12T01:36:13.211Z 27 Acked 1026 Write 2 Sent Done Done false
40672024-02-12T01:36:13.211Z 28 Acked 1027 Read 6 Sent Done Done false
40682024-02-12T01:36:13.211Z 29 Acked 1028 Read 3 Sent Done Done false
40692024-02-12T01:36:13.211Z 30 Acked 1029 Write 5 Sent Sent Sent false
40702024-02-12T01:36:13.211Z STATES DS:0 DS:1 DS:2 TOTAL
40712024-02-12T01:36:13.211Z New 0 0 0 0
40722024-02-12T01:36:13.211Z Sent 29 1 1 31
40732024-02-12T01:36:13.211Z Done 1 29 29 59
40742024-02-12T01:36:13.211Z Skipped 0 0 0 0
40752024-02-12T01:36:13.212Z Error 0 0 0 0
40762024-02-12T01:36:13.212ZLast Flush: 0 1018 1018
40772024-02-12T01:36:13.212ZDownstairs last five completed:
40782024-02-12T01:36:13.212ZUpstairs last five completed: 30 29 28 27 26
40792024-02-12T01:36:13.212Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:12.116945438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1279}
40802024-02-12T01:36:13.212ZWrote out file "/tmp/test_repair_verify.data"
40812024-02-12T01:36:13.215Z
40822024-02-12T01:36:13.217ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40832024-02-12T01:36:13.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40842024-02-12T01:36:13.233Z 0 000-019 28 28 28 42 42 42 F T T <---
40852024-02-12T01:36:13.233Z 1 020-039 34 36 36 51 55 55 F F F <---
40862024-02-12T01:36:13.233Z 2 040-059 34 34 34 52 52 52 F F F
40872024-02-12T01:36:13.234Z 3 060-079 34 36 36 52 56 56 F F F <---
40882024-02-12T01:36:13.234Z 4 080-099 34 34 34 53 53 53 F F F
40892024-02-12T01:36:13.237Z 5 100-119 33 33 33 50 50 50 F F F
40902024-02-12T01:36:13.237Z 6 120-139 34 34 34 51 51 51 F T T <---
40912024-02-12T01:36:13.237Z 7 140-159 31 31 31 46 46 46 F F F
40922024-02-12T01:36:13.237Z 8 160-179 35 35 35 54 54 54 F F F
40932024-02-12T01:36:13.237Z 9 180-199 31 36 36 46 55 55 F F F <---
40942024-02-12T01:36:13.237Z 10 200-219 30 30 30 45 45 45 F F F
40952024-02-12T01:36:13.237Z 11 220-239 23 23 23 36 36 36 F T T <---
40962024-02-12T01:36:13.237Z 12 240-259 32 32 32 49 49 49 F F F
40972024-02-12T01:36:13.237Z 13 260-279 32 32 32 49 49 49 F T T <---
40982024-02-12T01:36:13.237Z 14 280-299 34 36 36 51 55 55 F F F <---
40992024-02-12T01:36:13.237Z 15 300-319 34 34 34 51 51 51 F F F
41002024-02-12T01:36:13.237Z 16 320-339 27 27 27 41 41 41 F F F
41012024-02-12T01:36:13.237Z 17 340-359 35 35 35 54 54 54 F F F
41022024-02-12T01:36:13.238Z 18 360-379 34 34 34 51 51 51 F F F
41032024-02-12T01:36:13.238Z 19 380-399 35 36 36 54 56 56 T T T <---
41042024-02-12T01:36:13.238Z 20 400-419 29 36 36 44 55 55 F F F <---
41052024-02-12T01:36:13.238Z 21 420-439 21 21 21 31 31 31 F F F
41062024-02-12T01:36:13.238Z 22 440-459 34 34 34 51 51 51 F F F
41072024-02-12T01:36:13.238Z 23 460-479 34 34 34 51 51 51 F F F
41082024-02-12T01:36:13.238Z 24 480-499 33 33 33 50 50 50 F F F
41092024-02-12T01:36:13.238Z 25 500-519 30 30 30 45 45 45 F F F
41102024-02-12T01:36:13.238Z 26 520-539 32 32 32 49 49 49 F F F
41112024-02-12T01:36:13.238Z 27 540-559 34 36 36 51 55 55 F T T <---
41122024-02-12T01:36:13.238Z 28 560-579 35 35 35 54 54 54 F T T <---
41132024-02-12T01:36:13.238Z 29 580-599 27 36 36 41 55 55 F F F <---
41142024-02-12T01:36:13.238ZMax gen: 36, Max flush: 56
41152024-02-12T01:36:13.238ZError: Difference in extent metadata found!
41162024-02-12T01:36:13.238ZA Difference in extent metadata is expected here
41172024-02-12T01:36:13.238ZOn loop 18
41182024-02-12T01:36:13.238Z
41192024-02-12T01:36:13.245ZVerifying data now
41202024-02-12T01:36:17.870ZNo change after verify
41212024-02-12T01:36:17.871ZLoop: 18 Downstairs dump after verify (and repair):
41222024-02-12T01:36:17.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41232024-02-12T01:36:17.886Z 0 000-019 37 37 37 57 57 57 F F F
41242024-02-12T01:36:17.886Z 1 020-039 36 36 36 55 55 55 F F F
41252024-02-12T01:36:17.886Z 2 040-059 34 34 34 52 52 52 F F F
41262024-02-12T01:36:17.886Z 3 060-079 36 36 36 56 56 56 F F F
41272024-02-12T01:36:17.886Z 4 080-099 34 34 34 53 53 53 F F F
41282024-02-12T01:36:17.886Z 5 100-119 33 33 33 50 50 50 F F F
41292024-02-12T01:36:17.886Z 6 120-139 37 37 37 57 57 57 F F F
41302024-02-12T01:36:17.887Z 7 140-159 31 31 31 46 46 46 F F F
41312024-02-12T01:36:17.887Z 8 160-179 35 35 35 54 54 54 F F F
41322024-02-12T01:36:17.887Z 9 180-199 36 36 36 55 55 55 F F F
41332024-02-12T01:36:17.887Z 10 200-219 30 30 30 45 45 45 F F F
41342024-02-12T01:36:17.887Z 11 220-239 37 37 37 57 57 57 F F F
41352024-02-12T01:36:17.890Z 12 240-259 32 32 32 49 49 49 F F F
41362024-02-12T01:36:17.890Z 13 260-279 37 37 37 57 57 57 F F F
41372024-02-12T01:36:17.890Z 14 280-299 36 36 36 55 55 55 F F F
41382024-02-12T01:36:17.890Z 15 300-319 34 34 34 51 51 51 F F F
41392024-02-12T01:36:17.890Z 16 320-339 27 27 27 41 41 41 F F F
41402024-02-12T01:36:17.890Z 17 340-359 35 35 35 54 54 54 F F F
41412024-02-12T01:36:17.890Z 18 360-379 34 34 34 51 51 51 F F F
41422024-02-12T01:36:17.890Z 19 380-399 37 37 37 57 57 57 F F F
41432024-02-12T01:36:17.890Z 20 400-419 36 36 36 55 55 55 F F F
41442024-02-12T01:36:17.890Z 21 420-439 21 21 21 31 31 31 F F F
41452024-02-12T01:36:17.890Z 22 440-459 34 34 34 51 51 51 F F F
41462024-02-12T01:36:17.890Z 23 460-479 34 34 34 51 51 51 F F F
41472024-02-12T01:36:17.890Z 24 480-499 33 33 33 50 50 50 F F F
41482024-02-12T01:36:17.890Z 25 500-519 30 30 30 45 45 45 F F F
41492024-02-12T01:36:17.890Z 26 520-539 32 32 32 49 49 49 F F F
41502024-02-12T01:36:17.890Z 27 540-559 37 37 37 57 57 57 F F F
41512024-02-12T01:36:17.890Z 28 560-579 37 37 37 57 57 57 F F F
41522024-02-12T01:36:17.891Z 29 580-599 36 36 36 55 55 55 F F F
41532024-02-12T01:36:17.891ZMax gen: 37, Max flush: 57
41542024-02-12T01:36:17.891Z
41552024-02-12T01:36:17.891ZBegin loop 19
41562024-02-12T01:36:17.891ZDownstairs to restart: 1
41572024-02-12T01:36:17.908Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816169882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
41582024-02-12T01:36:17.908Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.81642107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
41592024-02-12T01:36:17.913Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816442389Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
41602024-02-12T01:36:17.913Z{"msg":"Crucible 3e612187-0a2e-4a82-a51d-f934fdb342e6 has session id: 29fa057e-b563-428d-b30b-80b72d68042a","v":0,"name":"crucible","level":30Crucible runtime is spawned
41612024-02-12T01:36:17.913Z,"time":"2024-02-12T01:36:16.816499206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41622024-02-12T01:36:17.913Z{"msg":"Upstairs opts: Upstairs UUID: 3e612187-0a2e-4a82-a51d-f934fdb342e6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816537354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41632024-02-12T01:36:17.913Z{"msg":"Crucible stats registered with UUID: 3e612187-0a2e-4a82-a51d-f934fdb342e6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816578782Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41642024-02-12T01:36:17.913Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816602181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
41652024-02-12T01:36:17.913Z{"msg":"3e612187-0a2e-4a82-a51d-f934fdb342e6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816752244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41662024-02-12T01:36:17.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816819911Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41672024-02-12T01:36:17.913Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816863539Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41682024-02-12T01:36:17.913Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816935595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41692024-02-12T01:36:17.914Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.816984253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41702024-02-12T01:36:17.914Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817111837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41712024-02-12T01:36:17.914Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817172424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41722024-02-12T01:36:17.914Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817219812Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41732024-02-12T01:36:17.914Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.81724689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41742024-02-12T01:36:17.914Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817271119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41752024-02-12T01:36:17.914Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817295318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41762024-02-12T01:36:17.914Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817329157Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41772024-02-12T01:36:17.914Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:16.817357305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41782024-02-12T01:36:17.914Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:16.817393674Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41792024-02-12T01:36:17.914Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:16.817420842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41802024-02-12T01:36:17.914Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817440661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41812024-02-12T01:36:17.914Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.81746219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41822024-02-12T01:36:17.914Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817493879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41832024-02-12T01:36:17.914Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817715928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41842024-02-12T01:36:17.914Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817754266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41852024-02-12T01:36:17.914Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817914069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41862024-02-12T01:36:17.914Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.817966836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41872024-02-12T01:36:17.914Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818285371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41882024-02-12T01:36:17.914Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818412545Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41892024-02-12T01:36:17.914Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818779667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41902024-02-12T01:36:17.914Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818831495Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41912024-02-12T01:36:17.914Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818866663Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41922024-02-12T01:36:17.915Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818907051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41932024-02-12T01:36:17.915Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.81893495Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41942024-02-12T01:36:17.915Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:16.818949609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41952024-02-12T01:36:18.910Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.818554459Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41962024-02-12T01:36:18.911Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.818640365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41972024-02-12T01:36:18.916Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.819516053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41982024-02-12T01:36:18.916Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.81958786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
41992024-02-12T01:36:18.916Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820014079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42002024-02-12T01:36:18.916Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820556993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42012024-02-12T01:36:18.916Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820596391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42022024-02-12T01:36:18.916Z{"msg":"[0]R flush_numbers[0..12]: [57, 55, 52, 56, 53, 50, 57, 46, 54, 55, 45, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.82062062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42032024-02-12T01:36:18.916Z{"msg":"[0]R generation[0..12]: [37, 36, 34, 36, 34, 33, 37, 31, 35, 36, 30, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820651989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42042024-02-12T01:36:18.916Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820668818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42052024-02-12T01:36:18.916Z{"msg":"[1]R flush_numbers[0..12]: [57, 55, 52, 56, 53, 50, 57, 46, 54, 55, 45, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820702306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42062024-02-12T01:36:18.916Z{"msg":"[1]R generation[0..12]: [37, 36, 34, 36, 34, 33, 37, 31, 35, 36, 30, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820722665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42072024-02-12T01:36:18.916Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820752734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42082024-02-12T01:36:18.916Z{"msg":"[2]R flush_numbers[0..12]: [57, 55, 52, 56, 53, 50, 57, 46, 54, 55, 45, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820768353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42092024-02-12T01:36:18.916Z{"msg":"[2]R generation[0..12]: [37, 36, 34, 36, 34, 33, 37, 31, 35, 36, 30, 37]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
42102024-02-12T01:36:18.916Ztime":"2024-02-12T01:36:17.820782552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42112024-02-12T01:36:18.916Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820812281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42122024-02-12T01:36:18.916Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.82083537Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42132024-02-12T01:36:18.916Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820866388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42142024-02-12T01:36:18.916Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820880888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42152024-02-12T01:36:18.916Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820899407Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42162024-02-12T01:36:18.916Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820927796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42172024-02-12T01:36:18.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820941985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42182024-02-12T01:36:18.916Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820958634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42192024-02-12T01:36:18.917Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.820988443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42202024-02-12T01:36:18.917Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.821003552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42212024-02-12T01:36:18.917Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.821020201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42222024-02-12T01:36:18.917Z{"msg":"3e612187-0a2e-4a82-a51d-f934fdb342e6 is now active with session: 29fa057e-b563-428d-b30b-80b72d68042a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.82104875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42232024-02-12T01:36:18.917Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.821061879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287,"session_id":"29fa057e-b563-428d-b30b-80b72d68042a"}
42242024-02-12T01:36:18.917Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:17.821090758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
42252024-02-12T01:36:18.917ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42262024-02-12T01:36:18.917ZLoading write count information from file "/tmp/test_repair_verify.data"
42272024-02-12T01:36:18.917ZRun Repair workload
42282024-02-12T01:36:18.917Z01/30 Flush
42292024-02-12T01:36:18.923Z02/30 Write block 214 len 12288 data: 4 4 4
42302024-02-12T01:36:18.923Z03/30 Read block 61 len 24576
42312024-02-12T01:36:18.923Z04/30 Write block 82 len 4096 data: 6
42322024-02-12T01:36:18.923Z05/30 Write block 4 len 36864 data: 3 4 4 4 4 4 4 4 5
42332024-02-12T01:36:18.923Z06/30 Flush
42342024-02-12T01:36:18.923Z07/30 Write block 454 len 20480 data: 5 5 4 4 5
42352024-02-12T01:36:18.923Z08/30 Write block 116 len 32768 data: 4 5 6 6 5 5 4 5
42362024-02-12T01:36:18.923Z09/30 Read block 60 len 4096
42372024-02-12T01:36:18.923Z10/30 Read block 4 len 12288
42382024-02-12T01:36:18.927Z11/30 Write block 482 len 36864 data: 5 6 6 6 6 7 7 5 5
42392024-02-12T01:36:18.927Z12/30 Read block 408 len 28672
42402024-02-12T01:36:18.927Z13/30 Write block 79 len 32768 data: 6 7 7 7 4 5 4 4
42412024-02-12T01:36:18.930Z14/30 Write block 589 len 8192 data: 5 5
42422024-02-12T01:36:18.930Z15/30 Write block 322 len 4096 data: 5
42432024-02-12T01:36:18.930Z16/30 Read block 60 len 40960
42442024-02-12T01:36:18.930Z17/30 Write block 38 len 4096 data: 4
42452024-02-12T01:36:18.930Z18/30 Write block 467 len 8192 data: 7 7
42462024-02-12T01:36:18.930Z19/30 Read block 550 len 20480
42472024-02-12T01:36:18.933Z20/30 Read block 23 len 24576
42482024-02-12T01:36:18.933Z21/30 Read block 253 len 16384
42492024-02-12T01:36:18.933Z22/30 Read block 469 len 16384
42502024-02-12T01:36:18.936Z23/30 Write block 43 len 40960 data: 5 5 4 3 5 4 4 4 4 2
42512024-02-12T01:36:18.936Z24/30 Write block 515 len 40960 data: 3 3 2 2 2 2 2 2 2 2
42522024-02-12T01:36:18.936Z25/30 Write block 181 len 28672 data: 5 3 3 4 4 4 6
42532024-02-12T01:36:18.936Z26/30 Read block 146 len 24576
42542024-02-12T01:36:18.939Z27/30 Read block 379 len 16384
42552024-02-12T01:36:18.939Z28/30 Read block 477 len 8192
42562024-02-12T01:36:18.939Z29/30 Write block 387 len 4096 data: 6
42572024-02-12T01:36:18.939Z30/30 Read block 365 len 20480
42582024-02-12T01:36:18.943Z----------------------------------------------------------------
42592024-02-12T01:36:18.943Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:29
42602024-02-12T01:36:18.943ZStates: Active Active Active
42612024-02-12T01:36:18.943ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42622024-02-12T01:36:18.943Z 2 Acked 1001 Write 3 Done Sent Done false
42632024-02-12T01:36:18.943Z 3 Acked 1002 Read 6 Done Sent Done false
42642024-02-12T01:36:18.943Z 4 Acked 1003 Write 1 Done Sent Done false
42652024-02-12T01:36:18.943Z 5 Acked 1004 Write 9 Done Sent Done false
42662024-02-12T01:36:18.943Z 6 Acked 1005 Flush 0 Done Sent Done false
42672024-02-12T01:36:18.943Z 7 Acked 1006 Write 5 Done Sent Done false
42682024-02-12T01:36:18.943Z 8 Acked 1007 Write 8 Done Sent Done false
42692024-02-12T01:36:18.943Z 9 Acked 1008 Read 1 Done Sent Done false
42702024-02-12T01:36:18.943Z 10 Acked 1009 Read 3 Done Sent Done false
42712024-02-12T01:36:18.943Z 11 Acked 1010 Write 9 Done Sent Done false
42722024-02-12T01:36:18.943Z 12 Acked 1011 Read 7 Done Sent Done false
42732024-02-12T01:36:18.943Z 13 Acked 1012 Write 8 Done Sent Done false
42742024-02-12T01:36:18.944Z 14 Acked 1013 Write 2 Done Sent Done false
42752024-02-12T01:36:18.944Z 15 Acked 1014 Write 1 Done Sent Done false
42762024-02-12T01:36:18.944Z 16 Acked 1015 Read 10 Done Sent Done false
42772024-02-12T01:36:18.944Z 17 Acked 1016 Write 1 Done Sent Done false
42782024-02-12T01:36:18.944Z 18 Acked 1017 Write 2 Done Sent Done false
42792024-02-12T01:36:18.944Z 19 Acked 1018 Read 5 Done Sent Done false
42802024-02-12T01:36:18.944Z 20 Acked 1019 Read 6 Done Sent Done false
42812024-02-12T01:36:18.944Z 21 Acked 1020 Read 4 Done Sent Done false
42822024-02-12T01:36:18.944Z 22 Acked 1021 Read 4 Done Sent Done false
42832024-02-12T01:36:18.944Z 23 Acked 1022 Write 10 Done Sent Done false
42842024-02-12T01:36:18.944Z 24 Acked 1023 Write 10 Done Sent Done false
42852024-02-12T01:36:18.944Z 25 Acked 1024 Write 7 Done Sent Done false
42862024-02-12T01:36:18.944Z 26 Acked 1025 Read 6 Done Sent Done false
42872024-02-12T01:36:18.944Z 27 Acked 1026 Read 4 Done Sent Done false
42882024-02-12T01:36:18.944Z 28 Acked 1027 Read 2 Done Sent Done false
42892024-02-12T01:36:18.944Z 29 Acked 1028 Write 1 Done Sent Done false
42902024-02-12T01:36:18.944Z 30 Acked 1029 Read 5 Done Sent Done false
42912024-02-12T01:36:18.944Z STATES DS:0 DS:1 DS:2 TOTAL
42922024-02-12T01:36:18.944Z New 0 0 0 0
42932024-02-12T01:36:18.944Z Sent 0 29 0 29
42942024-02-12T01:36:18.944Z Done 29 0 29 58
42952024-02-12T01:36:18.944Z Skipped 0 0 0 0
42962024-02-12T01:36:18.944Z Error 0 0 0 0
42972024-02-12T01:36:18.944ZLast Flush: 1005 1000 1005
42982024-02-12T01:36:18.944ZDownstairs last five completed: 1000
42992024-02-12T01:36:18.945ZUpstairs last five completed: 30 29 28 27 26
43002024-02-12T01:36:18.945Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:17.848701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1287}
43012024-02-12T01:36:18.945ZWrote out file "/tmp/test_repair_verify.data"
43022024-02-12T01:36:18.948Z
43032024-02-12T01:36:18.953ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
43042024-02-12T01:36:18.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43052024-02-12T01:36:18.970Z 0 000-019 38 37 38 59 57 59 F F F <---
43062024-02-12T01:36:18.970Z 1 020-039 36 36 36 55 55 55 T F T <---
43072024-02-12T01:36:18.970Z 2 040-059 34 34 34 52 52 52 T F T <---
43082024-02-12T01:36:18.970Z 3 060-079 36 36 36 56 56 56 T F T <---
43092024-02-12T01:36:18.970Z 4 080-099 38 34 38 59 53 59 T F T <---
43102024-02-12T01:36:18.970Z 5 100-119 33 33 33 50 50 50 T F T <---
43112024-02-12T01:36:18.970Z 6 120-139 37 37 37 57 57 57 T F T <---
43122024-02-12T01:36:18.970Z 7 140-159 31 31 31 46 46 46 F F F
43132024-02-12T01:36:18.973Z 8 160-179 35 35 35 54 54 54 F F F
43142024-02-12T01:36:18.974Z 9 180-199 36 36 36 55 55 55 T F T <---
43152024-02-12T01:36:18.974Z 10 200-219 38 30 38 59 45 59 F F F <---
43162024-02-12T01:36:18.974Z 11 220-239 37 37 37 57 57 57 F F F
43172024-02-12T01:36:18.974Z 12 240-259 32 32 32 49 49 49 F F F
43182024-02-12T01:36:18.974Z 13 260-279 37 37 37 57 57 57 F F F
43192024-02-12T01:36:18.974Z 14 280-299 36 36 36 55 55 55 F F F
43202024-02-12T01:36:18.974Z 15 300-319 34 34 34 51 51 51 F F F
43212024-02-12T01:36:18.974Z 16 320-339 27 27 27 41 41 41 T F T <---
43222024-02-12T01:36:18.974Z 17 340-359 35 35 35 54 54 54 F F F
43232024-02-12T01:36:18.974Z 18 360-379 34 34 34 51 51 51 F F F
43242024-02-12T01:36:18.974Z 19 380-399 37 37 37 57 57 57 T F T <---
43252024-02-12T01:36:18.974Z 20 400-419 36 36 36 55 55 55 F F F
43262024-02-12T01:36:18.974Z 21 420-439 21 21 21 31 31 31 F F F
43272024-02-12T01:36:18.974Z 22 440-459 34 34 34 51 51 51 T F T <---
43282024-02-12T01:36:18.974Z 23 460-479 34 34 34 51 51 51 T F T <---
43292024-02-12T01:36:18.974Z 24 480-499 33 33 33 50 50 50 T F T <---
43302024-02-12T01:36:18.974Z 25 500-519 30 30 30 45 45 45 T F T <---
43312024-02-12T01:36:18.974Z 26 520-539 32 32 32 49 49 49 T F T <---
43322024-02-12T01:36:18.974Z 27 540-559 37 37 37 57 57 57 F F F
43332024-02-12T01:36:18.974Z 28 560-579 37 37 37 57 57 57 F F F
43342024-02-12T01:36:18.974Z 29 580-599 36 36 36 55 55 55 T F T <---
43352024-02-12T01:36:18.974ZMax gen: 38, Max flush: 59
43362024-02-12T01:36:18.974ZError: Difference in extent metadata found!
43372024-02-12T01:36:18.975ZA Difference in extent metadata is expected here
43382024-02-12T01:36:18.975ZOn loop 19
43392024-02-12T01:36:18.975Z
43402024-02-12T01:36:18.979ZVerifying data now
43412024-02-12T01:36:23.773ZNo change after verify
43422024-02-12T01:36:23.773ZLoop: 19 Downstairs dump after verify (and repair):
43432024-02-12T01:36:23.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43442024-02-12T01:36:23.789Z 0 000-019 38 38 38 59 59 59 F F F
43452024-02-12T01:36:23.789Z 1 020-039 39 39 39 60 60 60 F F F
43462024-02-12T01:36:23.789Z 2 040-059 39 39 39 60 60 60 F F F
43472024-02-12T01:36:23.789Z 3 060-079 39 39 39 60 60 60 F F F
43482024-02-12T01:36:23.789Z 4 080-099 39 39 39 60 60 60 F F F
43492024-02-12T01:36:23.789Z 5 100-119 39 39 39 60 60 60 F F F
43502024-02-12T01:36:23.789Z 6 120-139 39 39 39 60 60 60 F F F
43512024-02-12T01:36:23.789Z 7 140-159 31 31 31 46 46 46 F F F
43522024-02-12T01:36:23.789Z 8 160-179 35 35 35 54 54 54 F F F
43532024-02-12T01:36:23.793Z 9 180-199 39 39 39 60 60 60 F F F
43542024-02-12T01:36:23.793Z 10 200-219 38 38 38 59 59 59 F F F
43552024-02-12T01:36:23.793Z 11 220-239 37 37 37 57 57 57 F F F
43562024-02-12T01:36:23.793Z 12 240-259 32 32 32 49 49 49 F F F
43572024-02-12T01:36:23.793Z 13 260-279 37 37 37 57 57 57 F F F
43582024-02-12T01:36:23.793Z 14 280-299 36 36 36 55 55 55 F F F
43592024-02-12T01:36:23.793Z 15 300-319 34 34 34 51 51 51 F F F
43602024-02-12T01:36:23.793Z 16 320-339 39 39 39 60 60 60 F F F
43612024-02-12T01:36:23.793Z 17 340-359 35 35 35 54 54 54 F F F
43622024-02-12T01:36:23.793Z 18 360-379 34 34 34 51 51 51 F F F
43632024-02-12T01:36:23.793Z 19 380-399 39 39 39 60 60 60 F F F
43642024-02-12T01:36:23.793Z 20 400-419 36 36 36 55 55 55 F F F
43652024-02-12T01:36:23.793Z 21 420-439 21 21 21 31 31 31 F F F
43662024-02-12T01:36:23.793Z 22 440-459 39 39 39 60 60 60 F F F
43672024-02-12T01:36:23.793Z 23 460-479 39 39 39 60 60 60 F F F
43682024-02-12T01:36:23.793Z 24 480-499 39 39 39 60 60 60 F F F
43692024-02-12T01:36:23.793Z 25 500-519 39 39 39 60 60 60 F F F
43702024-02-12T01:36:23.793Z 26 520-539 39 39 39 60 60 60 F F F
43712024-02-12T01:36:23.793Z 27 540-559 37 37 37 57 57 57 F F F
43722024-02-12T01:36:23.794Z 28 560-579 37 37 37 57 57 57 F F F
43732024-02-12T01:36:23.794Z 29 580-599 39 39 39 60 60 60 F F F
43742024-02-12T01:36:23.794ZMax gen: 39, Max flush: 60
43752024-02-12T01:36:23.794Z
43762024-02-12T01:36:23.794ZBegin loop 20
43772024-02-12T01:36:23.794ZDownstairs to restart: 1
43782024-02-12T01:36:23.810Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.717627869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
43792024-02-12T01:36:23.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"Crucible runtime is spawned
43802024-02-12T01:36:23.810Ztime":"2024-02-12T01:36:22.717891416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
43812024-02-12T01:36:23.815Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.717950344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
43822024-02-12T01:36:23.815Z{"msg":"Crucible 8ada709c-8b7e-4f36-b15f-220a402a5c48 has session id: d293d41c-59ef-423a-8031-d6169591bda0","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.717989331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43832024-02-12T01:36:23.815Z{"msg":"Upstairs opts: Upstairs UUID: 8ada709c-8b7e-4f36-b15f-220a402a5c48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718081117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43842024-02-12T01:36:23.815Z{"msg":"Crucible stats registered with UUID: 8ada709c-8b7e-4f36-b15f-220a402a5c48","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718101276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43852024-02-12T01:36:23.816Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718131835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
43862024-02-12T01:36:23.816Z{"msg":"8ada709c-8b7e-4f36-b15f-220a402a5c48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718145554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43872024-02-12T01:36:23.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718175163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43882024-02-12T01:36:23.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718192282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43892024-02-12T01:36:23.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718206331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43902024-02-12T01:36:23.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.71824431Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43912024-02-12T01:36:23.816Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718276918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43922024-02-12T01:36:23.816Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718297807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43932024-02-12T01:36:23.816Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718318436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43942024-02-12T01:36:23.816Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718344325Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43952024-02-12T01:36:23.816Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718367604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43962024-02-12T01:36:23.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718424001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43972024-02-12T01:36:23.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.718549945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43982024-02-12T01:36:23.816Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.71864112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
43992024-02-12T01:36:23.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719265701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44002024-02-12T01:36:23.816Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719305139Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44012024-02-12T01:36:23.816Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719394444Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44022024-02-12T01:36:23.817Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719433713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44032024-02-12T01:36:23.817Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719736088Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44042024-02-12T01:36:23.817Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.719852273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44052024-02-12T01:36:23.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720242124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44062024-02-12T01:36:23.817Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720297081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44072024-02-12T01:36:23.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.72033689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44082024-02-12T01:36:23.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720353139Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44092024-02-12T01:36:23.817Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720375328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44102024-02-12T01:36:23.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720390387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44112024-02-12T01:36:23.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720663834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44122024-02-12T01:36:23.817Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.720696042Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44132024-02-12T01:36:23.817Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721152531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44142024-02-12T01:36:23.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721570241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44152024-02-12T01:36:23.823Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721609609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44162024-02-12T01:36:23.823Z{"msg":"[0]R flush_numbers[0..12]: [59, 60, 60, 60, 60, 60, 60, 46, 54, 60, 59, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721626618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44172024-02-12T01:36:23.823Z{"msg":"[0]R generation[0..12]: [38, 39, 39, 39, 39, 39, 39, 31, 35, 39, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721648017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44182024-02-12T01:36:23.823Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721662576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44192024-02-12T01:36:23.823Z{"msg":"[1]R flush_numbers[0..12]: [59, 60, 60, 60, 60, 60, 60, 46, 54, 60, 59, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721683355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44202024-02-12T01:36:23.823Z{"msg":"[1]R generation[0..12]: [38, 39, 39, 39, 39, 39, 39, 31, 35, 39, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721697815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44212024-02-12T01:36:23.823Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721719634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44222024-02-12T01:36:23.823Z{"msg":"[2]R flush_numbers[0..12]: [59, 60, 60, 60, 60, 60, 60, 46, 54, 60, 59, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721733813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44232024-02-12T01:36:23.823Z{"msg":"[2]R generation[0..12]: [38, 39, 39, 39, 39, 39, 39, 31, 35, 39, 38, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721753992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
44242024-02-12T01:36:23.823Zd293d41c-59ef-423a-8031-d6169591bda0"}
44252024-02-12T01:36:23.823Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721771951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44262024-02-12T01:36:23.823Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.72179238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44272024-02-12T01:36:23.823Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.7218065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44282024-02-12T01:36:23.823Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721824769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44292024-02-12T01:36:23.823Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721838148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44302024-02-12T01:36:23.823Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721863437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44312024-02-12T01:36:23.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721875956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44322024-02-12T01:36:23.824Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721896515Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44332024-02-12T01:36:23.824Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721910724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44342024-02-12T01:36:23.824Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721937763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44352024-02-12T01:36:23.824Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721951393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44362024-02-12T01:36:23.824Z{"msg":"8ada709c-8b7e-4f36-b15f-220a402a5c48 is now active with session: d293d41c-59ef-423a-8031-d6169591bda0","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721970792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44372024-02-12T01:36:23.824Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.721983071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295,"session_id":"d293d41c-59ef-423a-8031-d6169591bda0"}
44382024-02-12T01:36:23.824Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:22.72200129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
44392024-02-12T01:36:23.824ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
44402024-02-12T01:36:23.824ZLoading write count information from file "/tmp/test_repair_verify.data"
44412024-02-12T01:36:23.824ZRun Repair workload
44422024-02-12T01:36:23.824Z01/30 Write block 29 len 32768 data: 7 7 6 8 7 7 4 5
44432024-02-12T01:36:23.824Z02/30 Write block 445 len 24576 data: 4 3 4 4 4 4
44442024-02-12T01:36:23.824Z03/30 Read block 288 len 4096
44452024-02-12T01:36:23.824Z04/30 Read block 105 len 28672
44462024-02-12T01:36:23.824Z05/30 Read block 301 len 24576
44472024-02-12T01:36:23.824Z06/30 Read block 252 len 4096
44482024-02-12T01:36:23.828Z07/30 Read block 452 len 28672
44492024-02-12T01:36:23.828Z08/30 Write block 295 len 12288 data: 3 3 3
44502024-02-12T01:36:23.828Z09/30 Write block 71 len 4096 data: 4
44512024-02-12T01:36:23.828Z10/30 Write block 352 len 4096 data: 6
44522024-02-12T01:36:23.828Z11/30 Write block 367 len 28672 data: 8 6 6 5 4 4 5
44532024-02-12T01:36:23.828Z12/30 Read block 334 len 4096
44542024-02-12T01:36:23.828Z13/30 Write block 101 len 36864 data: 3 3 3 4 4 4 5 5 3
44552024-02-12T01:36:23.828Z14/30 Write block 278 len 28672 data: 5 6 6 5 5 6 6
44562024-02-12T01:36:23.828Z15/30 Write block 581 len 32768 data: 4 4 4 3 4 4 4 4
44572024-02-12T01:36:23.828Z16/30 Flush
44582024-02-12T01:36:23.832Z17/30 Write block 55 len 16384 data: 2 2 3 6
44592024-02-12T01:36:23.835Z18/30 Read block 586 len 32768
44602024-02-12T01:36:23.835Z19/30 Write block 197 len 32768 data: 2 2 3 3 3 3 4 4
44612024-02-12T01:36:23.835Z20/30 Write block 76 len 32768 data: 7 7 6 7 8 8 8 5
44622024-02-12T01:36:23.837Z21/30 Write block 175 len 12288 data: 4 5 5
44632024-02-12T01:36:23.837Z22/30 Read block 443 len 36864
44642024-02-12T01:36:23.837Z23/30 Write block 287 len 20480 data: 4 4 3 3 4
44652024-02-12T01:36:23.840Z24/30 Read block 250 len 8192
44662024-02-12T01:36:23.840Z25/30 Write block 157 len 20480 data: 3 3 3 3 3
44672024-02-12T01:36:23.840Z26/30 Write block 207 len 32768 data: 6 6 6 3 4 3 3 5
44682024-02-12T01:36:23.840Z27/30 Write block 58 len 4096 data: 7
44692024-02-12T01:36:23.846Z28/30 Read block 338 len 28672
44702024-02-12T01:36:23.846Z29/30 Write block 515 len 4096 data: 4
44712024-02-12T01:36:23.846Z30/30 Write block 192 len 12288 data: 4 3 2
44722024-02-12T01:36:23.846Z----------------------------------------------------------------
44732024-02-12T01:36:23.846Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44742024-02-12T01:36:23.847ZStates: Active Active Active
44752024-02-12T01:36:23.847ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44762024-02-12T01:36:23.847Z 1 Acked 1000 Write 8 Done Done Done false
44772024-02-12T01:36:23.847Z 2 Acked 1001 Write 6 Done Done Done false
44782024-02-12T01:36:23.847Z 3 Acked 1002 Read 1 Done Done Done false
44792024-02-12T01:36:23.847Z 4 Acked 1003 Read 7 Done Sent Done false
44802024-02-12T01:36:23.847Z 5 Acked 1004 Read 6 Done Sent Done false
44812024-02-12T01:36:23.847Z 6 Acked 1005 Read 1 Done Sent Done false
44822024-02-12T01:36:23.847Z 7 Acked 1006 Read 7 Done Sent Done false
44832024-02-12T01:36:23.847Z 8 Acked 1007 Write 3 Done Sent Done false
44842024-02-12T01:36:23.847Z 9 Acked 1008 Write 1 Done Sent Done false
44852024-02-12T01:36:23.847Z 10 Acked 1009 Write 1 Done Sent Done false
44862024-02-12T01:36:23.847Z 11 Acked 1010 Write 7 Done Sent Done false
44872024-02-12T01:36:23.847Z 12 Acked 1011 Read 1 Done Sent Done false
44882024-02-12T01:36:23.847Z 13 Acked 1012 Write 9 Done Sent Done false
44892024-02-12T01:36:23.847Z 14 Acked 1013 Write 7 Done Sent Done false
44902024-02-12T01:36:23.847Z 15 Acked 1014 Write 8 Done Sent Done false
44912024-02-12T01:36:23.847Z 16 Acked 1015 Flush 0 Done Sent Done false
44922024-02-12T01:36:23.847Z 17 Acked 1016 Write 4 Done Sent Done false
44932024-02-12T01:36:23.847Z 18 Acked 1017 Read 8 Done Sent Done false
44942024-02-12T01:36:23.847Z 19 Acked 1018 Write 8 Done Sent Done false
44952024-02-12T01:36:23.847Z 20 Acked 1019 Write 8 Done Sent Done false
44962024-02-12T01:36:23.847Z 21 Acked 1020 Write 3 Done Sent Done false
44972024-02-12T01:36:23.848Z 22 Acked 1021 Read 9 Done Sent Done false
44982024-02-12T01:36:23.848Z 23 Acked 1022 Write 5 Done Sent Done false
44992024-02-12T01:36:23.848Z 24 Acked 1023 Read 2 Done Sent Done false
45002024-02-12T01:36:23.848Z 25 Acked 1024 Write 5 Done Sent Done false
45012024-02-12T01:36:23.848Z 26 Acked 1025 Write 8 Done Sent Done false
45022024-02-12T01:36:23.848Z 27 Acked 1026 Write 1 Done Sent Done false
45032024-02-12T01:36:23.848Z 28 Acked 1027 Read 7 Done Sent Done false
45042024-02-12T01:36:23.848Z 29 Acked 1028 Write 1 Done Sent Done false
45052024-02-12T01:36:23.848Z 30 Acked 1029 Write 3 Sent Sent Sent false
45062024-02-12T01:36:23.848Z STATES DS:0 DS:1 DS:2 TOTAL
45072024-02-12T01:36:23.848Z New 0 0 0 0
45082024-02-12T01:36:23.848Z Sent 1 27 1 29
45092024-02-12T01:36:23.848Z Done 29 3 29 61
45102024-02-12T01:36:23.848Z Skipped 0 0 0 0
45112024-02-12T01:36:23.848Z Error 0 0 0 0
45122024-02-12T01:36:23.848ZLast Flush: 1015 0 1015
45132024-02-12T01:36:23.848ZDownstairs last five completed:
45142024-02-12T01:36:23.848ZUpstairs last five completed: 30 29 28 27 26
45152024-02-12T01:36:23.848Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:22.750529199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1295}
45162024-02-12T01:36:23.848ZWrote out file "/tmp/test_repair_verify.data"
45172024-02-12T01:36:23.852Z
45182024-02-12T01:36:23.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
45192024-02-12T01:36:23.870ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45202024-02-12T01:36:23.870Z 0 000-019 38 38 38 59 59 59 F F F
45212024-02-12T01:36:23.870Z 1 020-039 40 39 40 61 60 61 F T F <---
45222024-02-12T01:36:23.871Z 2 040-059 39 39 39 60 60 60 T F T <---
45232024-02-12T01:36:23.871Z 3 060-079 40 39 40 61 60 61 T F T <---
45242024-02-12T01:36:23.871Z 4 080-099 39 39 39 60 60 60 T F T <---
45252024-02-12T01:36:23.871Z 5 100-119 40 39 40 61 60 61 F F F <---
45262024-02-12T01:36:23.871Z 6 120-139 39 39 39 60 60 60 F F F
45272024-02-12T01:36:23.871Z 7 140-159 31 31 31 46 46 46 T F T <---
45282024-02-12T01:36:23.871Z 8 160-179 35 35 35 54 54 54 T F T <---
45292024-02-12T01:36:23.871Z 9 180-199 39 39 39 60 60 60 T F T <---
45302024-02-12T01:36:23.874Z 10 200-219 38 38 38 59 59 59 T F T <---
45312024-02-12T01:36:23.874Z 11 220-239 37 37 37 57 57 57 F F F
45322024-02-12T01:36:23.874Z 12 240-259 32 32 32 49 49 49 F F F
45332024-02-12T01:36:23.874Z 13 260-279 40 37 40 61 57 61 F F F <---
45342024-02-12T01:36:23.874Z 14 280-299 40 36 40 61 55 61 T F T <---
45352024-02-12T01:36:23.874Z 15 300-319 34 34 34 51 51 51 F F F
45362024-02-12T01:36:23.874Z 16 320-339 39 39 39 60 60 60 F F F
45372024-02-12T01:36:23.874Z 17 340-359 40 35 40 61 54 61 F F F <---
45382024-02-12T01:36:23.874Z 18 360-379 40 34 40 61 51 61 F F F <---
45392024-02-12T01:36:23.874Z 19 380-399 39 39 39 60 60 60 F F F
45402024-02-12T01:36:23.874Z 20 400-419 36 36 36 55 55 55 F F F
45412024-02-12T01:36:23.874Z 21 420-439 21 21 21 31 31 31 F F F
45422024-02-12T01:36:23.875Z 22 440-459 40 39 40 61 60 61 F T F <---
45432024-02-12T01:36:23.875Z 23 460-479 39 39 39 60 60 60 F F F
45442024-02-12T01:36:23.875Z 24 480-499 39 39 39 60 60 60 F F F
45452024-02-12T01:36:23.875Z 25 500-519 39 39 39 60 60 60 T F T <---
45462024-02-12T01:36:23.875Z 26 520-539 39 39 39 60 60 60 F F F
45472024-02-12T01:36:23.875Z 27 540-559 37 37 37 57 57 57 F F F
45482024-02-12T01:36:23.875Z 28 560-579 37 37 37 57 57 57 F F F
45492024-02-12T01:36:23.875Z 29 580-599 40 39 40 61 60 61 F F F <---
45502024-02-12T01:36:23.875ZMax gen: 40, Max flush: 61
45512024-02-12T01:36:23.875ZError: Difference in extent metadata found!
45522024-02-12T01:36:23.875ZA Difference in extent metadata is expected here
45532024-02-12T01:36:23.875ZOn loop 20
45542024-02-12T01:36:23.875Z
45552024-02-12T01:36:23.878ZVerifying data now
45562024-02-12T01:36:29.621ZNo change after verify
45572024-02-12T01:36:29.621ZLoop: 20 Downstairs dump after verify (and repair):
45582024-02-12T01:36:29.637ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45592024-02-12T01:36:29.637Z 0 000-019 38 38 38 59 59 59 F F F
45602024-02-12T01:36:29.638Z 1 020-039 40 40 40 61 61 61 F F F
45612024-02-12T01:36:29.638Z 2 040-059 41 41 41 62 62 62 F F F
45622024-02-12T01:36:29.638Z 3 060-079 41 41 41 62 62 62 F F F
45632024-02-12T01:36:29.638Z 4 080-099 41 41 41 62 62 62 F F F
45642024-02-12T01:36:29.638Z 5 100-119 40 40 40 61 61 61 F F F
45652024-02-12T01:36:29.639Z 6 120-139 39 39 39 60 60 60 F F F
45662024-02-12T01:36:29.639Z 7 140-159 41 41 41 62 62 62 F F F
45672024-02-12T01:36:29.639Z 8 160-179 41 41 41 62 62 62 F F F
45682024-02-12T01:36:29.639Z 9 180-199 41 41 41 62 62 62 F F F
45692024-02-12T01:36:29.640Z 10 200-219 41 41 41 62 62 62 F F F
45702024-02-12T01:36:29.640Z 11 220-239 37 37 37 57 57 57 F F F
45712024-02-12T01:36:29.640Z 12 240-259 32 32 32 49 49 49 F F F
45722024-02-12T01:36:29.640Z 13 260-279 40 40 40 61 61 61 F F F
45732024-02-12T01:36:29.653Z 14 280-299 41 41 41 62 62 62 F F F
45742024-02-12T01:36:29.654Z 15 300-319 34 34 34 51 51 51 F F F
45752024-02-12T01:36:29.654Z 16 320-339 39 39 39 60 60 60 F F F
45762024-02-12T01:36:29.654Z 17 340-359 40 40 40 61 61 61 F F F
45772024-02-12T01:36:29.654Z 18 360-379 40 40 40 61 61 61 F F F
45782024-02-12T01:36:29.654Z 19 380-399 39 39 39 60 60 60 F F F
45792024-02-12T01:36:29.654Z 20 400-419 36 36 36 55 55 55 F F F
45802024-02-12T01:36:29.654Z 21 420-439 21 21 21 31 31 31 F F F
45812024-02-12T01:36:29.654Z 22 440-459 40 40 40 61 61 61 F F F
45822024-02-12T01:36:29.654Z 23 460-479 39 39 39 60 60 60 F F F
45832024-02-12T01:36:29.654Z 24 480-499 39 39 39 60 60 60 F F F
45842024-02-12T01:36:29.654Z 25 500-519 41 41 41 62 62 62 F F F
45852024-02-12T01:36:29.654Z 26 520-539 39 39 39 60 60 60 F F F
45862024-02-12T01:36:29.654Z 27 540-559 37 37 37 57 57 57 F F F
45872024-02-12T01:36:29.655Z 28 560-579 37 37 37 57 57 57 F F F
45882024-02-12T01:36:29.655Z 29 580-599 40 40 40 61 61 61 F F F
45892024-02-12T01:36:29.655ZMax gen: 41, Max flush: 62
45902024-02-12T01:36:29.655Z
45912024-02-12T01:36:29.655ZBegin loop 21
45922024-02-12T01:36:29.655ZDownstairs to restart: 1
45932024-02-12T01:36:29.676Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.562678767Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
45942024-02-12T01:36:29.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.562920615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
45952024-02-12T01:36:29.677Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.562938074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
45962024-02-12T01:36:29.677Z{"msg":"Crucible 20dd9b08-c852-43df-986c-8d00fe61f1d5 has session id: 5e326c0c-f629-441d-80c4-c88495233a3c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.562959963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
45972024-02-12T01:36:29.677Z{"msg":"Upstairs opts: Upstairs UUID: 20dd9b08-c852-43df-986c-8d00fe61f1d5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
45982024-02-12T01:36:29.677Z,"time":"2024-02-12T01:36:28.562995602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
45992024-02-12T01:36:29.677Z{"msg":"Crucible stats registered with UUID: 20dd9b08-c852-43df-986c-8d00fe61f1d5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.56302773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46002024-02-12T01:36:29.678Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563080758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
46012024-02-12T01:36:29.678Z{"msg":"20dd9b08-c852-43df-986c-8d00fe61f1d5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563166774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46022024-02-12T01:36:29.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563202942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46032024-02-12T01:36:29.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563220091Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46042024-02-12T01:36:29.678Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563254349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46052024-02-12T01:36:29.678Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563291497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46062024-02-12T01:36:29.679Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563327666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46072024-02-12T01:36:29.679Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563362444Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46082024-02-12T01:36:29.679Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563395823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46092024-02-12T01:36:29.679Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563440041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46102024-02-12T01:36:29.679Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563477729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46112024-02-12T01:36:29.680Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563539606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46122024-02-12T01:36:29.680Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563591623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46132024-02-12T01:36:29.680Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.563619992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46142024-02-12T01:36:29.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.564272331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46152024-02-12T01:36:29.680Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.564311439Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46162024-02-12T01:36:29.680Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.564337848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46172024-02-12T01:36:29.681Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.564368936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46182024-02-12T01:36:29.681Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.564951978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46192024-02-12T01:36:29.681Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565003806Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46202024-02-12T01:36:29.681Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.56554146Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46212024-02-12T01:36:29.681Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565631496Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46222024-02-12T01:36:29.682Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565675964Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46232024-02-12T01:36:29.682Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565715132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46242024-02-12T01:36:29.682Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.56575456Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46252024-02-12T01:36:29.682Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565788809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46262024-02-12T01:36:29.683Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565804978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46272024-02-12T01:36:29.683Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.565825677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46282024-02-12T01:36:29.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566157161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46292024-02-12T01:36:29.683Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566564412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46302024-02-12T01:36:29.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566604379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46312024-02-12T01:36:29.684Z{"msg":"[0]R flush_numbers[0..12]: [59, 61, 62, 62, 62, 61, 60, 62, 62, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566620889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46322024-02-12T01:36:29.684Z{"msg":"[0]R generation[0..12]: [38, 40, 41, 41, 41, 40, 39, 41, 41, 41, 41, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566642188Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46332024-02-12T01:36:29.684Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566657317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46342024-02-12T01:36:29.684Z{"msg":"[1]R flush_numbers[0..12]: [59, 61, 62, 62, 62, 61, 60, 62, 62, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566671846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46352024-02-12T01:36:29.685Z{"msg":"[1]R generation[0..12]: [38, 40, 41, 41, 41, 40, 39, 41, 41, 41, 41, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566685756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46362024-02-12T01:36:29.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566715464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46372024-02-12T01:36:29.685Z{"msg":"[2]R flush_numbers[0..12]: [59, 61, 62, 62, 62, 61, 60, 62, 62, 62, 62, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566746083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46382024-02-12T01:36:29.685Z{"msg":"[2]R generation[0..12]: [38, 40, 41, 41, 41, 40, 39, 41, 41, 41, 41, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566769012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
46392024-02-12T01:36:29.685Z,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46402024-02-12T01:36:29.685Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.56680753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46412024-02-12T01:36:29.685Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566840068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46422024-02-12T01:36:29.686Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566858797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46432024-02-12T01:36:29.699Z{"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566883266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46442024-02-12T01:36:29.699Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566897256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46452024-02-12T01:36:29.700Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566927424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46462024-02-12T01:36:29.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566959822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46472024-02-12T01:36:29.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.566976892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46482024-02-12T01:36:29.700Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.56700714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46492024-02-12T01:36:29.700Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.567032479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46502024-02-12T01:36:29.701Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.567056168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46512024-02-12T01:36:29.701Z{"msg":"20dd9b08-c852-43df-986c-8d00fe61f1d5 is now active with session: 5e326c0c-f629-441d-80c4-c88495233a3c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.567086956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46522024-02-12T01:36:29.701Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.567100746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303,"session_id":"5e326c0c-f629-441d-80c4-c88495233a3c"}
46532024-02-12T01:36:29.701Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:28.567113185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
46542024-02-12T01:36:29.701ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
46552024-02-12T01:36:29.701ZLoading write count information from file "/tmp/test_repair_verify.data"
46562024-02-12T01:36:29.702ZRun Repair workload
46572024-02-12T01:36:29.702Z01/30 Write block 27 len 32768 data: 5 6 8 8 7 9 8 8
46582024-02-12T01:36:29.702Z02/30 Read block 43 len 20480
46592024-02-12T01:36:29.702Z03/30 Write block 416 len 24576 data: 5 5 3 2 3 3
46602024-02-12T01:36:29.702Z04/30 Flush
46612024-02-12T01:36:29.703Z05/30 Write block 11 len 36864 data: 5 6 4 5 5 4 4 4 4
46622024-02-12T01:36:29.703Z06/30 Read block 333 len 28672
46632024-02-12T01:36:29.703Z07/30 Read block 251 len 8192
46642024-02-12T01:36:29.703Z08/30 Read block 333 len 32768
46652024-02-12T01:36:29.703Z09/30 Flush
46662024-02-12T01:36:29.704Z10/30 Write block 334 len 24576 data: 2 2 2 3 3 3
46672024-02-12T01:36:29.704Z11/30 Write block 175 len 40960 data: 5 6 6 6 6 5 6 4 4 5
46682024-02-12T01:36:29.704Z12/30 Read block 470 len 16384
46692024-02-12T01:36:29.704Z13/30 Write block 532 len 24576 data: 5 5 5 7 6 7
46702024-02-12T01:36:29.704Z14/30 Read block 257 len 8192
46712024-02-12T01:36:29.705Z15/30 Write block 222 len 12288 data: 2 3 3
46722024-02-12T01:36:29.705Z16/30 Read block 324 len 28672
46732024-02-12T01:36:29.705Z17/30 Read block 1 len 28672
46742024-02-12T01:36:29.705Z18/30 Read block 562 len 16384
46752024-02-12T01:36:29.705Z19/30 Write block 517 len 36864 data: 3 3 3 3 3 3 3 3 2
46762024-02-12T01:36:29.706Z20/30 Write block 162 len 8192 data: 3 3
46772024-02-12T01:36:29.706Z21/30 Read block 118 len 4096
46782024-02-12T01:36:29.706Z22/30 Flush
46792024-02-12T01:36:29.706Z23/30 Flush
46802024-02-12T01:36:29.706Z24/30 Write block 298 len 16384 data: 3 4 4 4
46812024-02-12T01:36:29.707Z25/30 Write block 232 len 32768 data: 4 4 4 3 3 4 4 4
46822024-02-12T01:36:29.707Z26/30 Read block 444 len 16384
46832024-02-12T01:36:29.707Z27/30 Read block 59 len 32768
46842024-02-12T01:36:29.707Z28/30 Write block 282 len 4096 data: 6
46852024-02-12T01:36:29.707Z29/30 Read block 566 len 8192
46862024-02-12T01:36:29.707Z30/30 Read block 218 len 40960
46872024-02-12T01:36:29.718Z----------------------------------------------------------------
46882024-02-12T01:36:29.718Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46892024-02-12T01:36:29.719ZStates: Active Active Active
46902024-02-12T01:36:29.719ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46912024-02-12T01:36:29.719Z 1 Acked 1000 Write 8 Done Sent Done false
46922024-02-12T01:36:29.719Z 2 Acked 1001 Read 5 Done Sent Done false
46932024-02-12T01:36:29.719Z 3 Acked 1002 Write 6 Done Sent Done false
46942024-02-12T01:36:29.719Z 4 Acked 1003 Flush 0 Done Sent Done false
46952024-02-12T01:36:29.719Z 5 Acked 1004 Write 9 Done Sent Done false
46962024-02-12T01:36:29.719Z 6 Acked 1005 Read 7 Done Sent Done false
46972024-02-12T01:36:29.719Z 7 Acked 1006 Read 2 Done Sent Done false
46982024-02-12T01:36:29.719Z 8 Acked 1007 Read 8 Done Sent Done false
46992024-02-12T01:36:29.719Z 9 Acked 1008 Flush 0 Done Sent Done false
47002024-02-12T01:36:29.719Z 10 Acked 1009 Write 6 Done Sent Done false
47012024-02-12T01:36:29.719Z 11 Acked 1010 Write 10 Done Sent Done false
47022024-02-12T01:36:29.719Z 12 Acked 1011 Read 4 Done Sent Done false
47032024-02-12T01:36:29.719Z 13 Acked 1012 Write 6 Done Sent Done false
47042024-02-12T01:36:29.720Z 14 Acked 1013 Read 2 Done Sent Done false
47052024-02-12T01:36:29.720Z 15 Acked 1014 Write 3 Done Sent Done false
47062024-02-12T01:36:29.720Z 16 Acked 1015 Read 7 Done Sent Done false
47072024-02-12T01:36:29.720Z 17 Acked 1016 Read 7 Done Sent Done false
47082024-02-12T01:36:29.720Z 18 Acked 1017 Read 4 Done Sent Done false
47092024-02-12T01:36:29.720Z 19 Acked 1018 Write 9 Done Sent Done false
47102024-02-12T01:36:29.720Z 20 Acked 1019 Write 2 Done Sent Done false
47112024-02-12T01:36:29.720Z 21 Acked 1020 Read 1 Done Sent Done false
47122024-02-12T01:36:29.720Z 22 Acked 1021 Flush 0 Done Sent Done false
47132024-02-12T01:36:29.720Z 23 Acked 1022 Flush 0 Done Sent Done false
47142024-02-12T01:36:29.720Z 24 Acked 1023 Write 4 Done Sent Done false
47152024-02-12T01:36:29.720Z 25 Acked 1024 Write 8 Done Sent Done false
47162024-02-12T01:36:29.720Z 26 Acked 1025 Read 4 Done Sent Done false
47172024-02-12T01:36:29.720Z 27 Acked 1026 Read 8 Done Sent Done false
47182024-02-12T01:36:29.720Z 28 Acked 1027 Write 1 Done Sent Done false
47192024-02-12T01:36:29.720Z 29 Acked 1028 Read 2 Done Sent Done false
47202024-02-12T01:36:29.721Z 30 Acked 1029 Read 10 Done Sent Sent false
47212024-02-12T01:36:29.721Z STATES DS:0 DS:1 DS:2 TOTAL
47222024-02-12T01:36:29.721Z New 0 0 0 0
47232024-02-12T01:36:29.721Z Sent 0 30 1 31
47242024-02-12T01:36:29.721Z Done 30 0 29 59
47252024-02-12T01:36:29.721Z Skipped 0 0 0 0
47262024-02-12T01:36:29.721Z Error 0 0 0 0
47272024-02-12T01:36:29.721ZLast Flush: 1022 0 1022
47282024-02-12T01:36:29.721ZDownstairs last five completed:
47292024-02-12T01:36:29.721ZUpstairs last five completed: 30 29 28 27 26
47302024-02-12T01:36:29.722Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:28.599720409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1303}
47312024-02-12T01:36:29.722ZWrote out file "/tmp/test_repair_verify.data"
47322024-02-12T01:36:29.722Z
47332024-02-12T01:36:29.722ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
47342024-02-12T01:36:29.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47352024-02-12T01:36:29.731Z 0 000-019 42 38 42 64 59 64 F F F <---
47362024-02-12T01:36:29.731Z 1 020-039 42 40 42 63 61 63 F F F <---
47372024-02-12T01:36:29.732Z 2 040-059 41 41 41 62 62 62 F F F
47382024-02-12T01:36:29.732Z 3 060-079 41 41 41 62 62 62 F F F
47392024-02-12T01:36:29.732Z 4 080-099 41 41 41 62 62 62 F F F
47402024-02-12T01:36:29.732Z 5 100-119 40 40 40 61 61 61 F F F
47412024-02-12T01:36:29.732Z 6 120-139 39 39 39 60 60 60 F F F
47422024-02-12T01:36:29.732Z 7 140-159 41 41 41 62 62 62 F F F
47432024-02-12T01:36:29.732Z 8 160-179 42 41 42 65 62 65 F F F <---
47442024-02-12T01:36:29.733Z 9 180-199 42 41 42 65 62 65 F F F <---
47452024-02-12T01:36:29.733Z 10 200-219 41 41 41 62 62 62 F F F
47462024-02-12T01:36:29.733Z 11 220-239 42 37 42 65 57 65 T F T <---
47472024-02-12T01:36:29.733Z 12 240-259 32 32 32 49 49 49 F F F
47482024-02-12T01:36:29.733Z 13 260-279 40 40 40 61 61 61 F F F
47492024-02-12T01:36:29.733Z 14 280-299 41 41 41 62 62 62 T F T <---
47502024-02-12T01:36:29.733Z 15 300-319 34 34 34 51 51 51 T F T <---
47512024-02-12T01:36:29.734Z 16 320-339 42 39 42 65 60 65 F F F <---
47522024-02-12T01:36:29.734Z 17 340-359 40 40 40 61 61 61 F F F
47532024-02-12T01:36:29.734Z 18 360-379 40 40 40 61 61 61 F F F
47542024-02-12T01:36:29.734Z 19 380-399 39 39 39 60 60 60 F F F
47552024-02-12T01:36:29.734Z 20 400-419 42 36 42 63 55 63 F F F <---
47562024-02-12T01:36:29.734Z 21 420-439 42 21 42 63 31 63 F F F <---
47572024-02-12T01:36:29.735Z 22 440-459 40 40 40 61 61 61 F F F
47582024-02-12T01:36:29.735Z 23 460-479 39 39 39 60 60 60 F F F
47592024-02-12T01:36:29.735Z 24 480-499 39 39 39 60 60 60 F F F
47602024-02-12T01:36:29.735Z 25 500-519 42 41 42 65 62 65 F F F <---
47612024-02-12T01:36:29.735Z 26 520-539 42 39 42 65 60 65 F F F <---
47622024-02-12T01:36:29.735Z 27 540-559 37 37 37 57 57 57 F F F
47632024-02-12T01:36:29.735Z 28 560-579 37 37 37 57 57 57 F F F
47642024-02-12T01:36:29.735Z 29 580-599 40 40 40 61 61 61 F F F
47652024-02-12T01:36:29.736ZMax gen: 42, Max flush: 65
47662024-02-12T01:36:29.736ZError: Difference in extent metadata found!
47672024-02-12T01:36:29.736ZA Difference in extent metadata is expected here
47682024-02-12T01:36:29.736ZOn loop 21
47692024-02-12T01:36:29.736Z
47702024-02-12T01:36:29.736ZVerifying data now
47712024-02-12T01:36:34.306ZNo change after verify
47722024-02-12T01:36:34.306ZLoop: 21 Downstairs dump after verify (and repair):
47732024-02-12T01:36:34.322ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47742024-02-12T01:36:34.322Z 0 000-019 42 42 42 64 64 64 F F F
47752024-02-12T01:36:34.322Z 1 020-039 42 42 42 63 63 63 F F F
47762024-02-12T01:36:34.322Z 2 040-059 41 41 41 62 62 62 F F F
47772024-02-12T01:36:34.323Z 3 060-079 41 41 41 62 62 62 F F F
47782024-02-12T01:36:34.323Z 4 080-099 41 41 41 62 62 62 F F F
47792024-02-12T01:36:34.323Z 5 100-119 40 40 40 61 61 61 F F F
47802024-02-12T01:36:34.323Z 6 120-139 39 39 39 60 60 60 F F F
47812024-02-12T01:36:34.323Z 7 140-159 41 41 41 62 62 62 F F F
47822024-02-12T01:36:34.323Z 8 160-179 42 42 42 65 65 65 F F F
47832024-02-12T01:36:34.323Z 9 180-199 42 42 42 65 65 65 F F F
47842024-02-12T01:36:34.323Z 10 200-219 41 41 41 62 62 62 F F F
47852024-02-12T01:36:34.323Z 11 220-239 43 43 43 66 66 66 F F F
47862024-02-12T01:36:34.324Z 12 240-259 32 32 32 49 49 49 F F F
47872024-02-12T01:36:34.324Z 13 260-279 40 40 40 61 61 61 F F F
47882024-02-12T01:36:34.324Z 14 280-299 43 43 43 66 66 66 F F F
47892024-02-12T01:36:34.324Z 15 300-319 43 43 43 66 66 66 F F F
47902024-02-12T01:36:34.324Z 16 320-339 42 42 42 65 65 65 F F F
47912024-02-12T01:36:34.324Z 17 340-359 40 40 40 61 61 61 F F F
47922024-02-12T01:36:34.324Z 18 360-379 40 40 40 61 61 61 F F F
47932024-02-12T01:36:34.324Z 19 380-399 39 39 39 60 60 60 F F F
47942024-02-12T01:36:34.325Z 20 400-419 42 42 42 63 63 63 F F F
47952024-02-12T01:36:34.325Z 21 420-439 42 42 42 63 63 63 F F F
47962024-02-12T01:36:34.325Z 22 440-459 40 40 40 61 61 61 F F F
47972024-02-12T01:36:34.325Z 23 460-479 39 39 39 60 60 60 F F F
47982024-02-12T01:36:34.325Z 24 480-499 39 39 39 60 60 60 F F F
47992024-02-12T01:36:34.325Z 25 500-519 42 42 42 65 65 65 F F F
48002024-02-12T01:36:34.325Z 26 520-539 42 42 42 65 65 65 F F F
48012024-02-12T01:36:34.326Z 27 540-559 37 37 37 57 57 57 F F F
48022024-02-12T01:36:34.326Z 28 560-579 37 37 37 57 57 57 F F F
48032024-02-12T01:36:34.326Z 29 580-599 40 40 40 61 61 61 F F F
48042024-02-12T01:36:34.326ZMax gen: 43, Max flush: 66
48052024-02-12T01:36:34.335Z
48062024-02-12T01:36:34.335ZBegin loop 22
48072024-02-12T01:36:34.335ZDownstairs to restart: 2
48082024-02-12T01:36:34.372Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240183846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
48092024-02-12T01:36:34.372Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
48102024-02-12T01:36:34.373Z,"time":"2024-02-12T01:36:33.240473303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
48112024-02-12T01:36:34.373Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240565738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
48122024-02-12T01:36:34.373Z{"msg":"Crucible d4b3342d-ed4c-464c-ba19-e0f9b221d387 has session id: 942f8fa0-f9cf-47c7-8e22-64c827a38301","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240607366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48132024-02-12T01:36:34.373Z{"msg":"Upstairs opts: Upstairs UUID: d4b3342d-ed4c-464c-ba19-e0f9b221d387, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240661374Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48142024-02-12T01:36:34.373Z{"msg":"Crucible stats registered with UUID: d4b3342d-ed4c-464c-ba19-e0f9b221d387","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240690342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48152024-02-12T01:36:34.374Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240704442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
48162024-02-12T01:36:34.374Z{"msg":"d4b3342d-ed4c-464c-ba19-e0f9b221d387 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240768169Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48172024-02-12T01:36:34.374Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240794817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48182024-02-12T01:36:34.374Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240810576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48192024-02-12T01:36:34.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240849385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48202024-02-12T01:36:34.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240874024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48212024-02-12T01:36:34.375Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240913952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48222024-02-12T01:36:34.375Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.24094128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48232024-02-12T01:36:34.375Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.240974549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48242024-02-12T01:36:34.376Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241049845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48252024-02-12T01:36:34.376Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241081074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48262024-02-12T01:36:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241108362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48272024-02-12T01:36:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241198758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48282024-02-12T01:36:34.376Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241240656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48292024-02-12T01:36:34.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241717963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48302024-02-12T01:36:34.377Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241766781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48312024-02-12T01:36:34.377Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241838727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48322024-02-12T01:36:34.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.241882536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48332024-02-12T01:36:34.377Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242328844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48342024-02-12T01:36:34.378Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242479207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48352024-02-12T01:36:34.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242815141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48362024-02-12T01:36:34.378Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242850279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48372024-02-12T01:36:34.378Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242865918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48382024-02-12T01:36:34.378Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242956174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48392024-02-12T01:36:34.379Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.242983743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48402024-02-12T01:36:34.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243002582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48412024-02-12T01:36:34.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.24304882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48422024-02-12T01:36:34.379Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243073689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48432024-02-12T01:36:34.379Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243537846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48442024-02-12T01:36:34.380Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243930348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48452024-02-12T01:36:34.380Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243962936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48462024-02-12T01:36:34.380Z{"msg":"[0]R flush_numbers[0..12]: [64, 63, 62, 62, 62, 61, 60, 62, 65, 65, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243978276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48472024-02-12T01:36:34.380Z{"msg":"[0]R generation[0..12]: [42, 42, 41, 41, 41, 40, 39, 41, 42, 42, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.243993465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48482024-02-12T01:36:34.381Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244015934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48492024-02-12T01:36:34.381Z{"msg":"[1]R flush_numbers[0..12]: [64, 63, 62, 62, 62, 61, 60, 62, 65, 65, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244030283Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48502024-02-12T01:36:34.381Z{"msg":"[1]R generation[0..12]: [42, 42, 41, 41, 41, 40, 39, 41, 42, 42, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244051252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48512024-02-12T01:36:34.381Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244066111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48522024-02-12T01:36:34.381Z{"msg":"[2]R flush_numbers[0..12]: [64, 63, 62, 62, 62, 61, 60, 62, 65, 65, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.24408647Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48532024-02-12T01:36:34.382Z{"msg":"[2]R generation[0..12]: [42, 42, 41, 41, 41, 40, 39, 41, 42, 42, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.24410046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48542024-02-12T01:36:34.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244120569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48552024-02-12T01:36:34.382Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
48562024-02-12T01:36:34.382Z2024-02-12T01:36:33.244133858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48572024-02-12T01:36:34.383Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244150737Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48582024-02-12T01:36:34.383Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244163797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48592024-02-12T01:36:34.383Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244183416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48602024-02-12T01:36:34.383Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244197065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48612024-02-12T01:36:34.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244216024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48622024-02-12T01:36:34.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244229304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48632024-02-12T01:36:34.384Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244242363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48642024-02-12T01:36:34.384Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244256152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48652024-02-12T01:36:34.384Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244270851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48662024-02-12T01:36:34.385Z{"msg":"d4b3342d-ed4c-464c-ba19-e0f9b221d387 is now active with session: 942f8fa0-f9cf-47c7-8e22-64c827a38301","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.24429379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48672024-02-12T01:36:34.385Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244328899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311,"session_id":"942f8fa0-f9cf-47c7-8e22-64c827a38301"}
48682024-02-12T01:36:34.385Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:33.244359507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
48692024-02-12T01:36:34.385ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48702024-02-12T01:36:34.385ZLoading write count information from file "/tmp/test_repair_verify.data"
48712024-02-12T01:36:34.386ZRun Repair workload
48722024-02-12T01:36:34.386Z01/30 Write block 104 len 8192 data: 5 5
48732024-02-12T01:36:34.396Z02/30 Write block 505 len 32768 data: 3 2 2 2 2 2 3 3
48742024-02-12T01:36:34.396Z03/30 Read block 169 len 4096
48752024-02-12T01:36:34.396Z04/30 Read block 370 len 16384
48762024-02-12T01:36:34.396Z05/30 Read block 203 len 16384
48772024-02-12T01:36:34.396Z06/30 Read block 482 len 12288
48782024-02-12T01:36:34.396Z07/30 Flush
48792024-02-12T01:36:34.396Z08/30 Write block 558 len 16384 data: 4 6 5 5
48802024-02-12T01:36:34.397Z09/30 Write block 160 len 28672 data: 4 4 4 4 4 6 6
48812024-02-12T01:36:34.397Z10/30 Read block 61 len 8192
48822024-02-12T01:36:34.397Z11/30 Read block 175 len 32768
48832024-02-12T01:36:34.397Z12/30 Read block 526 len 12288
48842024-02-12T01:36:34.397Z13/30 Write block 115 len 40960 data: 5 5 6 7 7 6 6 5 6 6
48852024-02-12T01:36:34.397Z14/30 Read block 100 len 32768
48862024-02-12T01:36:34.398Z15/30 Read block 508 len 40960
48872024-02-12T01:36:34.398Z16/30 Write block 343 len 36864 data: 4 4 5 5 5 4 4 4 4
48882024-02-12T01:36:34.398Z17/30 Write block 567 len 16384 data: 3 4 4 4
48892024-02-12T01:36:34.398Z18/30 Read block 406 len 8192
48902024-02-12T01:36:34.398Z19/30 Read block 67 len 4096
48912024-02-12T01:36:34.398Z20/30 Read block 234 len 4096
48922024-02-12T01:36:34.398Z21/30 Flush
48932024-02-12T01:36:34.399Z22/30 Write block 305 len 36864 data: 4 5 6 7 5 5 5 4 4
48942024-02-12T01:36:34.399Z23/30 Write block 253 len 12288 data: 3 3 3
48952024-02-12T01:36:34.399Z24/30 Read block 258 len 24576
48962024-02-12T01:36:34.399Z25/30 Flush
48972024-02-12T01:36:34.399Z26/30 Write block 39 len 24576 data: 4 3 4 3 6 6
48982024-02-12T01:36:34.399Z27/30 Write block 231 len 36864 data: 4 5 5 5 4 4 5 5 5
48992024-02-12T01:36:34.399Z28/30 Read block 197 len 32768
49002024-02-12T01:36:34.400Z29/30 Write block 506 len 32768 data: 3 3 3 3 3 4 4 3
49012024-02-12T01:36:34.400Z30/30 Read block 400 len 12288
49022024-02-12T01:36:34.400Z----------------------------------------------------------------
49032024-02-12T01:36:34.400Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:23
49042024-02-12T01:36:34.400ZStates: Active Active Active
49052024-02-12T01:36:34.400ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
49062024-02-12T01:36:34.401Z 8 Acked 1007 Write 4 Done Done Done false
49072024-02-12T01:36:34.401Z 9 Acked 1008 Write 7 Done Done Sent false
49082024-02-12T01:36:34.401Z 10 Acked 1009 Read 2 Done Done Sent false
49092024-02-12T01:36:34.401Z 11 Acked 1010 Read 8 Done Done Sent false
49102024-02-12T01:36:34.401Z 12 Acked 1011 Read 3 Done Done Sent false
49112024-02-12T01:36:34.401Z 13 Acked 1012 Write 10 Done Done Sent false
49122024-02-12T01:36:34.401Z 14 Acked 1013 Read 8 Done Done Sent false
49132024-02-12T01:36:34.402Z 15 Acked 1014 Read 10 Done Done Sent false
49142024-02-12T01:36:34.402Z 16 Acked 1015 Write 9 Done Done Sent false
49152024-02-12T01:36:34.402Z 17 Acked 1016 Write 4 Done Done Sent false
49162024-02-12T01:36:34.402Z 18 Acked 1017 Read 2 Done Done Sent false
49172024-02-12T01:36:34.402Z 19 Acked 1018 Read 1 Done Done Sent false
49182024-02-12T01:36:34.402Z 20 Acked 1019 Read 1 Done Done Sent false
49192024-02-12T01:36:34.402Z 21 Acked 1020 Flush 0 Done Done Sent false
49202024-02-12T01:36:34.402Z 22 Acked 1021 Write 9 Done Done Sent false
49212024-02-12T01:36:34.403Z 23 Acked 1022 Write 3 Done Done Sent false
49222024-02-12T01:36:34.403Z 24 Acked 1023 Read 6 Done Done Sent false
49232024-02-12T01:36:34.403Z 25 Acked 1024 Flush 0 Done Done Sent false
49242024-02-12T01:36:34.403Z 26 Acked 1025 Write 6 Done Done Sent false
49252024-02-12T01:36:34.403Z 27 Acked 1026 Write 9 Done Done Sent false
49262024-02-12T01:36:34.403Z 28 Acked 1027 Read 8 Done Done Sent false
49272024-02-12T01:36:34.403Z 29 Acked 1028 Write 8 Done Done Sent false
49282024-02-12T01:36:34.403Z 30 Acked 1029 Read 3 Sent Done Sent false
49292024-02-12T01:36:34.404Z STATES DS:0 DS:1 DS:2 TOTAL
49302024-02-12T01:36:34.404Z New 0 0 0 0
49312024-02-12T01:36:34.404Z Sent 1 0 22 23
49322024-02-12T01:36:34.404Z Done 22 23 1 46
49332024-02-12T01:36:34.404Z Skipped 0 0 0 0
49342024-02-12T01:36:34.404Z Error 0 0 0 0
49352024-02-12T01:36:34.405ZLast Flush: 1024 1024 1006
49362024-02-12T01:36:34.405ZDownstairs last five completed: 1006 1005 1004 1003 1002
49372024-02-12T01:36:34.405ZUpstairs last five completed: 30 29 28 27 26
49382024-02-12T01:36:34.405Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:33.275622156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1311}
49392024-02-12T01:36:34.405ZWrote out file "/tmp/test_repair_verify.data"
49402024-02-12T01:36:34.405Z
49412024-02-12T01:36:34.405ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
49422024-02-12T01:36:34.835ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49432024-02-12T01:36:34.835Z 0 000-019 42 42 42 64 64 64 F F F
49442024-02-12T01:36:34.835Z 1 020-039 42 42 42 63 63 63 T T F <---
49452024-02-12T01:36:34.835Z 2 040-059 41 41 41 62 62 62 T T F <---
49462024-02-12T01:36:34.835Z 3 060-079 41 41 41 62 62 62 F F F
49472024-02-12T01:36:34.835Z 4 080-099 41 41 41 62 62 62 F F F
49482024-02-12T01:36:34.835Z 5 100-119 44 44 44 68 68 67 F F F <---
49492024-02-12T01:36:34.836Z 6 120-139 44 44 39 68 68 60 F F F <---
49502024-02-12T01:36:34.836Z 7 140-159 41 41 41 62 62 62 F F F
49512024-02-12T01:36:34.836Z 8 160-179 44 44 42 68 68 65 F F F <---
49522024-02-12T01:36:34.836Z 9 180-199 42 42 42 65 65 65 F F F
49532024-02-12T01:36:34.836Z 10 200-219 41 41 41 62 62 62 F F F
49542024-02-12T01:36:34.836Z 11 220-239 43 43 43 66 66 66 T T F <---
49552024-02-12T01:36:34.836Z 12 240-259 44 44 32 69 69 49 F F F <---
49562024-02-12T01:36:34.836Z 13 260-279 40 40 40 61 61 61 F F F
49572024-02-12T01:36:34.836Z 14 280-299 43 43 43 66 66 66 F F F
49582024-02-12T01:36:34.836Z 15 300-319 44 44 43 69 69 66 F F F <---
49592024-02-12T01:36:34.837Z 16 320-339 42 42 42 65 65 65 F F F
49602024-02-12T01:36:34.837Z 17 340-359 44 44 40 68 68 61 F F F <---
49612024-02-12T01:36:34.837Z 18 360-379 40 40 40 61 61 61 F F F
49622024-02-12T01:36:34.837Z 19 380-399 39 39 39 60 60 60 F F F
49632024-02-12T01:36:34.837Z 20 400-419 42 42 42 63 63 63 F F F
49642024-02-12T01:36:34.837Z 21 420-439 42 42 42 63 63 63 F F F
49652024-02-12T01:36:34.837Z 22 440-459 40 40 40 61 61 61 F F F
49662024-02-12T01:36:34.837Z 23 460-479 39 39 39 60 60 60 F F F
49672024-02-12T01:36:34.838Z 24 480-499 39 39 39 60 60 60 F F F
49682024-02-12T01:36:34.838Z 25 500-519 44 44 44 67 67 67 T T F <---
49692024-02-12T01:36:34.838Z 26 520-539 42 42 42 65 65 65 F F F
49702024-02-12T01:36:34.838Z 27 540-559 44 44 37 68 68 57 F F T <---
49712024-02-12T01:36:34.838Z 28 560-579 44 44 37 68 68 57 F F T <---
49722024-02-12T01:36:34.838Z 29 580-599 40 40 40 61 61 61 F F F
49732024-02-12T01:36:34.838ZMax gen: 44, Max flush: 69
49742024-02-12T01:36:34.838ZError: Difference in extent metadata found!
49752024-02-12T01:36:34.838ZA Difference in extent metadata is expected here
49762024-02-12T01:36:34.839ZOn loop 22
49772024-02-12T01:36:34.839Z
49782024-02-12T01:36:34.839ZVerifying data now
49792024-02-12T01:36:39.988ZNo change after verify
49802024-02-12T01:36:39.988ZLoop: 22 Downstairs dump after verify (and repair):
49812024-02-12T01:36:40.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49822024-02-12T01:36:40.013Z 0 000-019 42 42 42 64 64 64 F F F
49832024-02-12T01:36:40.013Z 1 020-039 45 45 45 70 70 70 F F F
49842024-02-12T01:36:40.013Z 2 040-059 45 45 45 70 70 70 F F F
49852024-02-12T01:36:40.013Z 3 060-079 41 41 41 62 62 62 F F F
49862024-02-12T01:36:40.014Z 4 080-099 41 41 41 62 62 62 F F F
49872024-02-12T01:36:40.014Z 5 100-119 44 44 44 68 68 68 F F F
49882024-02-12T01:36:40.015Z 6 120-139 44 44 44 68 68 68 F F F
49892024-02-12T01:36:40.015Z 7 140-159 41 41 41 62 62 62 F F F
49902024-02-12T01:36:40.015Z 8 160-179 44 44 44 68 68 68 F F F
49912024-02-12T01:36:40.025Z 9 180-199 42 42 42 65 65 65 F F F
49922024-02-12T01:36:40.025Z 10 200-219 41 41 41 62 62 62 F F F
49932024-02-12T01:36:40.025Z 11 220-239 45 45 45 70 70 70 F F F
49942024-02-12T01:36:40.025Z 12 240-259 44 44 44 69 69 69 F F F
49952024-02-12T01:36:40.026Z 13 260-279 40 40 40 61 61 61 F F F
49962024-02-12T01:36:40.026Z 14 280-299 43 43 43 66 66 66 F F F
49972024-02-12T01:36:40.026Z 15 300-319 44 44 44 69 69 69 F F F
49982024-02-12T01:36:40.026Z 16 320-339 42 42 42 65 65 65 F F F
49992024-02-12T01:36:40.026Z 17 340-359 44 44 44 68 68 68 F F F
50002024-02-12T01:36:40.027Z 18 360-379 40 40 40 61 61 61 F F F
50012024-02-12T01:36:40.027Z 19 380-399 39 39 39 60 60 60 F F F
50022024-02-12T01:36:40.027Z 20 400-419 42 42 42 63 63 63 F F F
50032024-02-12T01:36:40.027Z 21 420-439 42 42 42 63 63 63 F F F
50042024-02-12T01:36:40.027Z 22 440-459 40 40 40 61 61 61 F F F
50052024-02-12T01:36:40.027Z 23 460-479 39 39 39 60 60 60 F F F
50062024-02-12T01:36:40.028Z 24 480-499 39 39 39 60 60 60 F F F
50072024-02-12T01:36:40.028Z 25 500-519 45 45 45 70 70 70 F F F
50082024-02-12T01:36:40.028Z 26 520-539 42 42 42 65 65 65 F F F
50092024-02-12T01:36:40.028Z 27 540-559 44 44 44 68 68 68 F F F
50102024-02-12T01:36:40.028Z 28 560-579 44 44 44 68 68 68 F F F
50112024-02-12T01:36:40.029Z 29 580-599 40 40 40 61 61 61 F F F
50122024-02-12T01:36:40.029ZMax gen: 45, Max flush: 70
50132024-02-12T01:36:40.029Z
50142024-02-12T01:36:40.029ZBegin loop 23
50152024-02-12T01:36:40.029ZDownstairs to restart: 0
50162024-02-12T01:36:40.046Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.926572414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319}
50172024-02-12T01:36:40.047Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.926868859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid"Crucible runtime is spawned
50182024-02-12T01:36:40.047Z:1319}
50192024-02-12T01:36:40.047Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.926915687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319}
50202024-02-12T01:36:40.047Z{"msg":"Crucible 66a11c2a-e0da-4d24-bbc4-f61986f9e746 has session id: 48c67ddb-cc3a-4469-ae4c-405a2db1a1fb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.926938566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50212024-02-12T01:36:40.048Z{"msg":"Upstairs opts: Upstairs UUID: 66a11c2a-e0da-4d24-bbc4-f61986f9e746, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.926983044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50222024-02-12T01:36:40.048Z{"msg":"Crucible stats registered with UUID: 66a11c2a-e0da-4d24-bbc4-f61986f9e746","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927008723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50232024-02-12T01:36:40.048Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927042211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319}
50242024-02-12T01:36:40.048Z{"msg":"66a11c2a-e0da-4d24-bbc4-f61986f9e746 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927092939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50252024-02-12T01:36:40.048Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927128227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50262024-02-12T01:36:40.048Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927145276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50272024-02-12T01:36:40.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927178845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50282024-02-12T01:36:40.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927204834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50292024-02-12T01:36:40.049Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927226282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50302024-02-12T01:36:40.049Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927240602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50312024-02-12T01:36:40.049Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927254261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50322024-02-12T01:36:40.050Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.92728147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50332024-02-12T01:36:40.050Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927325368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50342024-02-12T01:36:40.050Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927500349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50352024-02-12T01:36:40.051Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927669191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50362024-02-12T01:36:40.051Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:38.927760667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50372024-02-12T01:36:40.051Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:38.927807355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50382024-02-12T01:36:40.051Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:38.927846553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50392024-02-12T01:36:40.052Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927867972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50402024-02-12T01:36:40.052Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927890671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50412024-02-12T01:36:40.052Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.927920769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50422024-02-12T01:36:40.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928133999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50432024-02-12T01:36:40.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928174197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50442024-02-12T01:36:40.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928200566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50452024-02-12T01:36:40.054Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928237684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50462024-02-12T01:36:40.054Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928676453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50472024-02-12T01:36:40.054Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.928845775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50482024-02-12T01:36:40.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.929241406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50492024-02-12T01:36:40.055Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.929331072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50502024-02-12T01:36:40.055Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.9293675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50512024-02-12T01:36:40.055Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.929389909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50522024-02-12T01:36:40.056Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.929423208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50532024-02-12T01:36:40.056Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:38.929448747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50542024-02-12T01:36:41.028Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.929270436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50552024-02-12T01:36:41.048Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.929466766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50562024-02-12T01:36:41.049Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.930331335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50572024-02-12T01:36:41.049Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.930375823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50582024-02-12T01:36:41.049Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.93086519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50592024-02-12T01:36:41.049Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931409334Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50602024-02-12T01:36:41.049Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931461641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50612024-02-12T01:36:41.049Z{"msg":"[0]R flush_numbers[0..12]: [64, 70, 70, 62, 62, 68, 68, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.93148764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50622024-02-12T01:36:41.050Z{"msg":"[0]R generation[0..12]: [42, 45, 45, 41, 41, 44, 44, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931522258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50632024-02-12T01:36:41.050Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931548317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50642024-02-12T01:36:41.050Z{"msg":"[1]R flush_numbers[0..12]: [64, 70, 70, 62, 62, 68, 68, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931583605Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50652024-02-12T01:36:41.050Z{"msg":"[1]R generation[0..12]: [42, 45, 45, 41, 41, 44, 44, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931618524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50662024-02-12T01:36:41.050Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931664401Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
50672024-02-12T01:36:41.050Z","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50682024-02-12T01:36:41.051Z{"msg":"[2]R flush_numbers[0..12]: [64, 70, 70, 62, 62, 68, 68, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.93169731Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50692024-02-12T01:36:41.051Z{"msg":"[2]R generation[0..12]: [42, 45, 45, 41, 41, 44, 44, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931730878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50702024-02-12T01:36:41.051Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931776856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50712024-02-12T01:36:41.052Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931804695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50722024-02-12T01:36:41.052Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931840833Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50732024-02-12T01:36:41.052Z{"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931875111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50742024-02-12T01:36:41.052Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931891851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50752024-02-12T01:36:41.052Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931919989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50762024-02-12T01:36:41.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931952508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50772024-02-12T01:36:41.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.931988426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50782024-02-12T01:36:41.053Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.932009675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50792024-02-12T01:36:41.053Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50802024-02-12T01:36:41.053Ztime":"2024-02-12T01:36:39.932043843Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50812024-02-12T01:36:41.054Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.932084112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50822024-02-12T01:36:41.054Z{"msg":"66a11c2a-e0da-4d24-bbc4-f61986f9e746 is now active with session: 48c67ddb-cc3a-4469-ae4c-405a2db1a1fb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.932100851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50832024-02-12T01:36:41.054Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.9321208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319,"session_id":"48c67ddb-cc3a-4469-ae4c-405a2db1a1fb"}
50842024-02-12T01:36:41.054Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:39.932134049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319}
50852024-02-12T01:36:41.054ZLoading write count information from file "/tmp/test_repair_verify.data"
50862024-02-12T01:36:41.055ZRun Repair workload
50872024-02-12T01:36:41.055Z01/30 Write block 421 len 24576 data: 4 3 3 3 3 3
50882024-02-12T01:36:41.055Z02/30 Write block 87 len 40960 data: 6 6 6 5 5 4 3 2 3 3
50892024-02-12T01:36:41.055Z03/30 Read block 280 len 40960
50902024-02-12T01:36:41.055Z04/30 Read block 178 len 20480
50912024-02-12T01:36:41.055Z05/30 Read block 265 len 36864
50922024-02-12T01:36:41.055Z06/30 Read block 55 len 16384
50932024-02-12T01:36:41.084Z07/30 Write block 123 len 24576 data: 7 7 6 8 8 7
50942024-02-12T01:36:41.084Z08/30 Read block 488 len 16384
50952024-02-12T01:36:41.084Z09/30 Write block 351 len 12288 data: 5 7 5
50962024-02-12T01:36:41.084Z10/30 Read block 459 len 4096
50972024-02-12T01:36:41.084Z11/30 Read block 210 len 12288
50982024-02-12T01:36:41.085Z12/30 Write block 375 len 24576 data: 6 6 6 5 4 4
50992024-02-12T01:36:41.085Z13/30 Read block 168 len 32768
51002024-02-12T01:36:41.085Z14/30 Read block 234 len 20480
51012024-02-12T01:36:41.085Z15/30 Write block 113 len 20480 data: 5 7 6 6 7
51022024-02-12T01:36:41.085Z16/30 Read block 281 len 36864
51032024-02-12T01:36:41.085Z17/30 Flush
51042024-02-12T01:36:41.085Z18/30 Write block 478 len 16384 data: 4 6 7 5
51052024-02-12T01:36:41.086Z19/30 Write block 538 len 36864 data: 8 7 6 4 4 3 3 4 5
51062024-02-12T01:36:41.086Z20/30 Write block 498 len 8192 data: 3 4
51072024-02-12T01:36:41.086Z21/30 Read block 123 len 12288
51082024-02-12T01:36:41.086Z22/30 Write block 526 len 4096 data: 2
51092024-02-12T01:36:41.086Z23/30 Read block 80 len 16384
51102024-02-12T01:36:41.086Z24/30 Write block 85 len 36864 data: 5 5 7 7 7 6 6 5 4
51112024-02-12T01:36:41.086Z25/30 Read block 591 len 32768
51122024-02-12T01:36:41.087Z26/30 Flush
51132024-02-12T01:36:41.087Z27/30 Write block 119 len 20480 data: 8 7 7 6 8
51142024-02-12T01:36:41.087Z28/30 Write block 40 len 20480 data: 4 5 4 7 7
51152024-02-12T01:36:41.087Z29/30 Read block 136 len 4096
51162024-02-12T01:36:41.087Z30/30 Read block 196 len 4096
51172024-02-12T01:36:41.087Z----------------------------------------------------------------
51182024-02-12T01:36:41.087Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
51192024-02-12T01:36:41.088ZStates: Active Active Active
51202024-02-12T01:36:41.088ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
51212024-02-12T01:36:41.088Z 1 Acked 1000 Write 6 Done Done Done false
51222024-02-12T01:36:41.088Z 2 Acked 1001 Write 10 Sent Done Done false
51232024-02-12T01:36:41.088Z 3 Acked 1002 Read 10 Sent Done Done false
51242024-02-12T01:36:41.088Z 4 Acked 1003 Read 5 Sent Done Done false
51252024-02-12T01:36:41.089Z 5 Acked 1004 Read 9 Sent Done Done false
51262024-02-12T01:36:41.089Z 6 Acked 1005 Read 4 Sent Done Done false
51272024-02-12T01:36:41.089Z 7 Acked 1006 Write 6 Sent Done Done false
51282024-02-12T01:36:41.089Z 8 Acked 1007 Read 4 Sent Done Done false
51292024-02-12T01:36:41.089Z 9 Acked 1008 Write 3 Sent Done Done false
51302024-02-12T01:36:41.089Z 10 Acked 1009 Read 1 Sent Done Done false
51312024-02-12T01:36:41.089Z 11 Acked 1010 Read 3 Sent Done Done false
51322024-02-12T01:36:41.090Z 12 Acked 1011 Write 6 Sent Done Done false
51332024-02-12T01:36:41.090Z 13 Acked 1012 Read 8 Sent Done Done false
51342024-02-12T01:36:41.090Z 14 Acked 1013 Read 5 Sent Done Done false
51352024-02-12T01:36:41.090Z 15 Acked 1014 Write 5 Sent Done Done false
51362024-02-12T01:36:41.090Z 16 Acked 1015 Read 9 Sent Done Done false
51372024-02-12T01:36:41.090Z 17 Acked 1016 Flush 0 Sent Done Done false
51382024-02-12T01:36:41.091Z 18 Acked 1017 Write 4 Sent Done Done false
51392024-02-12T01:36:41.091Z 19 Acked 1018 Write 9 Sent Done Done false
51402024-02-12T01:36:41.091Z 20 Acked 1019 Write 2 Sent Done Done false
51412024-02-12T01:36:41.091Z 21 Acked 1020 Read 3 Sent Done Done false
51422024-02-12T01:36:41.091Z 22 Acked 1021 Write 1 Sent Done Done false
51432024-02-12T01:36:41.091Z 23 Acked 1022 Read 4 Sent Done Done false
51442024-02-12T01:36:41.092Z 24 Acked 1023 Write 9 Sent Done Done false
51452024-02-12T01:36:41.092Z 25 Acked 1024 Read 8 Sent Done Done false
51462024-02-12T01:36:41.092Z 26 Acked 1025 Flush 0 Sent Done Done false
51472024-02-12T01:36:41.092Z 27 Acked 1026 Write 5 Sent Done Done false
51482024-02-12T01:36:41.092Z 28 Acked 1027 Write 5 Sent Done Done false
51492024-02-12T01:36:41.092Z 29 Acked 1028 Read 1 Sent Done Done false
51502024-02-12T01:36:41.092Z 30 Acked 1029 Read 1 Sent Sent Done false
51512024-02-12T01:36:41.093Z STATES DS:0 DS:1 DS:2 TOTAL
51522024-02-12T01:36:41.093Z New 0 0 0 0
51532024-02-12T01:36:41.093Z Sent 29 1 0 30
51542024-02-12T01:36:41.093Z Done 1 29 30 60
51552024-02-12T01:36:41.093Z Skipped 0 0 0 0
51562024-02-12T01:36:41.093Z Error 0 0 0 0
51572024-02-12T01:36:41.094ZLast Flush: 0 1025 1025
51582024-02-12T01:36:41.094ZDownstairs last five completed:
51592024-02-12T01:36:41.094ZUpstairs last five completed: 30 29 28 27 26
51602024-02-12T01:36:41.094Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:39.962990467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1319}
51612024-02-12T01:36:41.094ZWrote out file "/tmp/test_repair_verify.data"
51622024-02-12T01:36:41.094Z
51632024-02-12T01:36:41.094ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51642024-02-12T01:36:41.114ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51652024-02-12T01:36:41.114Z 0 000-019 42 42 42 64 64 64 F F F
51662024-02-12T01:36:41.114Z 1 020-039 45 45 45 70 70 70 F F F
51672024-02-12T01:36:41.114Z 2 040-059 45 45 45 70 70 70 F T T <---
51682024-02-12T01:36:41.114Z 3 060-079 41 41 41 62 62 62 F F F
51692024-02-12T01:36:41.114Z 4 080-099 41 46 46 62 72 72 F F F <---
51702024-02-12T01:36:41.114Z 5 100-119 44 46 46 68 71 71 F T T <---
51712024-02-12T01:36:41.115Z 6 120-139 44 46 46 68 71 71 F T T <---
51722024-02-12T01:36:41.115Z 7 140-159 41 41 41 62 62 62 F F F
51732024-02-12T01:36:41.115Z 8 160-179 44 44 44 68 68 68 F F F
51742024-02-12T01:36:41.115Z 9 180-199 42 42 42 65 65 65 F F F
51752024-02-12T01:36:41.115Z 10 200-219 41 41 41 62 62 62 F F F
51762024-02-12T01:36:41.115Z 11 220-239 45 45 45 70 70 70 F F F
51772024-02-12T01:36:41.115Z 12 240-259 44 44 44 69 69 69 F F F
51782024-02-12T01:36:41.115Z 13 260-279 40 40 40 61 61 61 F F F
51792024-02-12T01:36:41.116Z 14 280-299 43 43 43 66 66 66 F F F
51802024-02-12T01:36:41.116Z 15 300-319 44 44 44 69 69 69 F F F
51812024-02-12T01:36:41.116Z 16 320-339 42 42 42 65 65 65 F F F
51822024-02-12T01:36:41.116Z 17 340-359 44 46 46 68 71 71 F F F <---
51832024-02-12T01:36:41.116Z 18 360-379 40 46 46 61 71 71 F F F <---
51842024-02-12T01:36:41.116Z 19 380-399 39 46 46 60 71 71 F F F <---
51852024-02-12T01:36:41.116Z 20 400-419 42 42 42 63 63 63 F F F
51862024-02-12T01:36:41.116Z 21 420-439 42 46 46 63 71 71 T F F <---
51872024-02-12T01:36:41.116Z 22 440-459 40 40 40 61 61 61 F F F
51882024-02-12T01:36:41.116Z 23 460-479 39 46 46 60 72 72 F F F <---
51892024-02-12T01:36:41.117Z 24 480-499 39 46 46 60 72 72 F F F <---
51902024-02-12T01:36:41.117Z 25 500-519 45 45 45 70 70 70 F F F
51912024-02-12T01:36:41.117Z 26 520-539 42 46 46 65 72 72 F F F <---
51922024-02-12T01:36:41.117Z 27 540-559 44 46 46 68 72 72 F F F <---
51932024-02-12T01:36:41.117Z 28 560-579 44 44 44 68 68 68 F F F
51942024-02-12T01:36:41.117Z 29 580-599 40 40 40 61 61 61 F F F
51952024-02-12T01:36:41.117ZMax gen: 46, Max flush: 72
51962024-02-12T01:36:41.117ZError: Difference in extent metadata found!
51972024-02-12T01:36:41.117ZA Difference in extent metadata is expected here
51982024-02-12T01:36:41.118ZOn loop 23
51992024-02-12T01:36:41.118Z
52002024-02-12T01:36:41.118ZVerifying data now
52012024-02-12T01:36:46.676ZNo change after verify
52022024-02-12T01:36:46.676ZLoop: 23 Downstairs dump after verify (and repair):
52032024-02-12T01:36:46.691ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
52042024-02-12T01:36:46.691Z 0 000-019 42 42 42 64 64 64 F F F
52052024-02-12T01:36:46.691Z 1 020-039 45 45 45 70 70 70 F F F
52062024-02-12T01:36:46.691Z 2 040-059 47 47 47 73 73 73 F F F
52072024-02-12T01:36:46.691Z 3 060-079 41 41 41 62 62 62 F F F
52082024-02-12T01:36:46.691Z 4 080-099 46 46 46 72 72 72 F F F
52092024-02-12T01:36:46.691Z 5 100-119 47 47 47 73 73 73 F F F
52102024-02-12T01:36:46.691Z 6 120-139 47 47 47 73 73 73 F F F
52112024-02-12T01:36:46.697Z 7 140-159 41 41 41 62 62 62 F F F
52122024-02-12T01:36:46.697Z 8 160-179 44 44 44 68 68 68 F F F
52132024-02-12T01:36:46.697Z 9 180-199 42 42 42 65 65 65 F F F
52142024-02-12T01:36:46.697Z 10 200-219 41 41 41 62 62 62 F F F
52152024-02-12T01:36:46.697Z 11 220-239 45 45 45 70 70 70 F F F
52162024-02-12T01:36:46.698Z 12 240-259 44 44 44 69 69 69 F F F
52172024-02-12T01:36:46.698Z 13 260-279 40 40 40 61 61 61 F F F
52182024-02-12T01:36:46.698Z 14 280-299 43 43 43 66 66 66 F F F
52192024-02-12T01:36:46.698Z 15 300-319 44 44 44 69 69 69 F F F
52202024-02-12T01:36:46.698Z 16 320-339 42 42 42 65 65 65 F F F
52212024-02-12T01:36:46.698Z 17 340-359 46 46 46 71 71 71 F F F
52222024-02-12T01:36:46.698Z 18 360-379 46 46 46 71 71 71 F F F
52232024-02-12T01:36:46.698Z 19 380-399 46 46 46 71 71 71 F F F
52242024-02-12T01:36:46.698Z 20 400-419 42 42 42 63 63 63 F F F
52252024-02-12T01:36:46.698Z 21 420-439 46 46 46 71 71 71 F F F
52262024-02-12T01:36:46.698Z 22 440-459 40 40 40 61 61 61 F F F
52272024-02-12T01:36:46.698Z 23 460-479 46 46 46 72 72 72 F F F
52282024-02-12T01:36:46.698Z 24 480-499 46 46 46 72 72 72 F F F
52292024-02-12T01:36:46.698Z 25 500-519 45 45 45 70 70 70 F F F
52302024-02-12T01:36:46.698Z 26 520-539 46 46 46 72 72 72 F F F
52312024-02-12T01:36:46.698Z 27 540-559 46 46 46 72 72 72 F F F
52322024-02-12T01:36:46.698Z 28 560-579 44 44 44 68 68 68 F F F
52332024-02-12T01:36:46.698Z 29 580-599 40 40 40 61 61 61 F F F
52342024-02-12T01:36:46.698ZMax gen: 47, Max flush: 73
52352024-02-12T01:36:46.698Z
52362024-02-12T01:36:46.698ZBegin loop 24
52372024-02-12T01:36:46.698ZDownstairs to restart: 0
52382024-02-12T01:36:46.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618227741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
52392024-02-12T01:36:46.713Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618559875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
52402024-02-12T01:36:46.713ZCrucible runtime is spawned
52412024-02-12T01:36:46.717Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618609713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
52422024-02-12T01:36:46.718Z{"msg":"Crucible 46ff692f-d800-4039-8ed9-5ce3eb067254 has session id: 4b8f105f-2ad6-48fb-af85-7859f7f70559","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.6186569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52432024-02-12T01:36:46.718Z{"msg":"Upstairs opts: Upstairs UUID: 46ff692f-d800-4039-8ed9-5ce3eb067254, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618696079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52442024-02-12T01:36:46.718Z{"msg":"Crucible stats registered with UUID: 46ff692f-d800-4039-8ed9-5ce3eb067254","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618739716Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52452024-02-12T01:36:46.718Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618758396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
52462024-02-12T01:36:46.718Z{"msg":"46ff692f-d800-4039-8ed9-5ce3eb067254 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618793094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52472024-02-12T01:36:46.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618817913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52482024-02-12T01:36:46.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618854721Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52492024-02-12T01:36:46.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.61887519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52502024-02-12T01:36:46.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618924408Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52512024-02-12T01:36:46.718Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.618970045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52522024-02-12T01:36:46.718Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619002344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52532024-02-12T01:36:46.718Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.61909075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52542024-02-12T01:36:46.718Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619148967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52552024-02-12T01:36:46.718Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619186885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52562024-02-12T01:36:46.718Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619213074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52572024-02-12T01:36:46.718Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619239333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52582024-02-12T01:36:46.718Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619263462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52592024-02-12T01:36:46.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619891561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52602024-02-12T01:36:46.718Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619933379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52612024-02-12T01:36:46.718Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619961488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52622024-02-12T01:36:46.718Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.619996996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52632024-02-12T01:36:46.718Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.620509852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52642024-02-12T01:36:46.718Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.62055312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52652024-02-12T01:36:46.719Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621082055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52662024-02-12T01:36:46.719Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621123693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52672024-02-12T01:36:46.719Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621150072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52682024-02-12T01:36:46.719Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.62117873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52692024-02-12T01:36:46.719Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621204359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52702024-02-12T01:36:46.719Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621231518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52712024-02-12T01:36:46.719Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621283775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52722024-02-12T01:36:46.719Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621323973Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52732024-02-12T01:36:46.723Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.621801221Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52742024-02-12T01:36:46.723Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.62222177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52752024-02-12T01:36:46.723Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622255749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52762024-02-12T01:36:46.723Z{"msg":"[0]R flush_numbers[0..12]: [64, 70, 73, 62, 72, 73, 73, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622272858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52772024-02-12T01:36:46.723Z{"msg":"[0]R generation[0..12]: [42, 45, 47, 41, 46, 47, 47, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622293827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52782024-02-12T01:36:46.724Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622308576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52792024-02-12T01:36:46.724Z{"msg":"[1]R flush_numbers[0..12]: [64, 70, 73, 62, 72, 73, 73, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622329035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52802024-02-12T01:36:46.724Z{"msg":"[1]R generation[0..12]: [42, 45, 47, 41, 46, 47, 47, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622343524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52812024-02-12T01:36:46.724Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622367774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52822024-02-12T01:36:46.724Z{"msg":"[2]R flush_numbers[0..12]: [64, 70, 73, 62, 72, 73, 73, 62, 68, 65, 62, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622416601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52832024-02-12T01:36:46.724Z{"msg":"[2]R generation[0..12]: [42, 45, 47, 41, 46, 47, 47, 41, 44, 42, 41, 45]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.62243185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52842024-02-12T01:36:46.724Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622454069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52852024-02-12T01:36:46.724Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
52862024-02-12T01:36:46.724Z,"time":"2024-02-12T01:36:45.622468249Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52872024-02-12T01:36:46.724Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622482078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52882024-02-12T01:36:46.724Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622508057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52892024-02-12T01:36:46.724Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622523466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52902024-02-12T01:36:46.724Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622543635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52912024-02-12T01:36:46.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622557314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52922024-02-12T01:36:46.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622589573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52932024-02-12T01:36:46.724Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622615682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52942024-02-12T01:36:46.724Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.62264989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52952024-02-12T01:36:46.724Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622665239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52962024-02-12T01:36:46.724Z{"msg":"46ff692f-d800-4039-8ed9-5ce3eb067254 is now active with session: 4b8f105f-2ad6-48fb-af85-7859f7f70559","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622697038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52972024-02-12T01:36:46.724Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622718597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327,"session_id":"4b8f105f-2ad6-48fb-af85-7859f7f70559"}
52982024-02-12T01:36:46.724Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:45.622751645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
52992024-02-12T01:36:46.724ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
53002024-02-12T01:36:46.724ZLoading write count information from file "/tmp/test_repair_verify.data"
53012024-02-12T01:36:46.724ZRun Repair workload
53022024-02-12T01:36:46.725Z01/30 Write block 216 len 28672 data: 5 2 2 3 3 2 3
53032024-02-12T01:36:46.725Z02/30 Read block 415 len 4096
53042024-02-12T01:36:46.725Z03/30 Write block 551 len 16384 data: 6 6 5 5
53052024-02-12T01:36:46.725Z04/30 Write block 495 len 32768 data: 4 4 3 4 5 4 5 5
53062024-02-12T01:36:46.728Z05/30 Read block 569 len 16384
53072024-02-12T01:36:46.728Z06/30 Flush
53082024-02-12T01:36:46.728Z07/30 Write block 324 len 16384 data: 3 2 2 2
53092024-02-12T01:36:46.728Z08/30 Read block 469 len 16384
53102024-02-12T01:36:46.728Z09/30 Write block 30 len 36864 data: 9 8 10 9 9 5 6 5 5
53112024-02-12T01:36:46.731Z10/30 Read block 555 len 36864
53122024-02-12T01:36:46.731Z11/30 Read block 456 len 12288
53132024-02-12T01:36:46.731Z12/30 Read block 553 len 20480
53142024-02-12T01:36:46.734Z13/30 Read block 297 len 8192
53152024-02-12T01:36:46.734Z14/30 Read block 404 len 32768
53162024-02-12T01:36:46.734Z15/30 Read block 481 len 12288
53172024-02-12T01:36:46.734Z16/30 Read block 379 len 40960
53182024-02-12T01:36:46.737Z17/30 Write block 84 len 8192 data: 6 6
53192024-02-12T01:36:46.737Z18/30 Write block 287 len 16384 data: 5 5 4 4
53202024-02-12T01:36:46.737Z19/30 Flush
53212024-02-12T01:36:46.740Z20/30 Write block 358 len 32768 data: 3 3 3 3 3 4 6 8
53222024-02-12T01:36:46.740Z21/30 Read block 486 len 40960
53232024-02-12T01:36:46.742Z22/30 Flush
53242024-02-12T01:36:46.743Z23/30 Write block 569 len 12288 data: 5 5 4
53252024-02-12T01:36:46.743Z24/30 Read block 34 len 8192
53262024-02-12T01:36:46.746Z25/30 Write block 583 len 16384 data: 5 4 5 5
53272024-02-12T01:36:46.746Z26/30 Read block 15 len 4096
53282024-02-12T01:36:46.746Z27/30 Write block 529 len 20480 data: 4 4 4 6 6
53292024-02-12T01:36:46.746Z28/30 Write block 179 len 4096 data: 7
53302024-02-12T01:36:46.746Z29/30 Read block 120 len 8192
53312024-02-12T01:36:46.749Z30/30 Read block 24 len 20480
53322024-02-12T01:36:46.749Z----------------------------------------------------------------
53332024-02-12T01:36:46.749Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
53342024-02-12T01:36:46.749ZStates: Active Active Active
53352024-02-12T01:36:46.749ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53362024-02-12T01:36:46.749Z 1 Acked 1000 Write 7 Done Done Done false
53372024-02-12T01:36:46.750Z 2 Acked 1001 Read 1 Done Done Done false
53382024-02-12T01:36:46.750Z 3 Acked 1002 Write 4 Sent Done Done false
53392024-02-12T01:36:46.750Z 4 Acked 1003 Write 8 Sent Done Done false
53402024-02-12T01:36:46.750Z 5 Acked 1004 Read 4 Sent Done Done false
53412024-02-12T01:36:46.750Z 6 Acked 1005 Flush 0 Sent Done Done false
53422024-02-12T01:36:46.750Z 7 Acked 1006 Write 4 Sent Done Done false
53432024-02-12T01:36:46.750Z 8 Acked 1007 Read 4 Sent Done Done false
53442024-02-12T01:36:46.750Z 9 Acked 1008 Write 9 Sent Done Done false
53452024-02-12T01:36:46.750Z 10 Acked 1009 Read 9 Sent Done Done false
53462024-02-12T01:36:46.750Z 11 Acked 1010 Read 3 Sent Done Done false
53472024-02-12T01:36:46.750Z 12 Acked 1011 Read 5 Sent Done Done false
53482024-02-12T01:36:46.750Z 13 Acked 1012 Read 2 Sent Done Done false
53492024-02-12T01:36:46.750Z 14 Acked 1013 Read 8 Sent Done Done false
53502024-02-12T01:36:46.750Z 15 Acked 1014 Read 3 Sent Done Done false
53512024-02-12T01:36:46.750Z 16 Acked 1015 Read 10 Sent Done Done false
53522024-02-12T01:36:46.750Z 17 Acked 1016 Write 2 Sent Done Done false
53532024-02-12T01:36:46.750Z 18 Acked 1017 Write 4 Sent Done Done false
53542024-02-12T01:36:46.750Z 19 Acked 1018 Flush 0 Sent Done Done false
53552024-02-12T01:36:46.750Z 20 Acked 1019 Write 8 Sent Done Done false
53562024-02-12T01:36:46.750Z 21 Acked 1020 Read 10 Sent Done Done false
53572024-02-12T01:36:46.750Z 22 Acked 1021 Flush 0 Sent Done Done false
53582024-02-12T01:36:46.750Z 23 Acked 1022 Write 3 Sent Done Done false
53592024-02-12T01:36:46.750Z 24 Acked 1023 Read 2 Sent Done Done false
53602024-02-12T01:36:46.750Z 25 Acked 1024 Write 4 Sent Done Done false
53612024-02-12T01:36:46.750Z 26 Acked 1025 Read 1 Sent Done Done false
53622024-02-12T01:36:46.751Z 27 Acked 1026 Write 5 Sent Done Done false
53632024-02-12T01:36:46.751Z 28 Acked 1027 Write 1 Sent Done Done false
53642024-02-12T01:36:46.751Z 29 Acked 1028 Read 2 Sent Done Done false
53652024-02-12T01:36:46.751Z 30 Acked 1029 Read 5 Sent Sent Done false
53662024-02-12T01:36:46.751Z STATES DS:0 DS:1 DS:2 TOTAL
53672024-02-12T01:36:46.751Z New 0 0 0 0
53682024-02-12T01:36:46.751Z Sent 28 1 0 29
53692024-02-12T01:36:46.751Z Done 2 29 30 61
53702024-02-12T01:36:46.751Z Skipped 0 0 0 0
53712024-02-12T01:36:46.751Z Error 0 0 0 0
53722024-02-12T01:36:46.751ZLast Flush: 0 1021 1021
53732024-02-12T01:36:46.751ZDownstairs last five completed:
53742024-02-12T01:36:46.751ZUpstairs last five completed: 30 29 28 27 26
53752024-02-12T01:36:46.751Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:45.652893107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1327}
53762024-02-12T01:36:46.751ZWrote out file "/tmp/test_repair_verify.data"
53772024-02-12T01:36:46.754Z
53782024-02-12T01:36:46.757ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53792024-02-12T01:36:46.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53802024-02-12T01:36:46.774Z 0 000-019 42 42 42 64 64 64 F F F
53812024-02-12T01:36:46.774Z 1 020-039 45 48 48 70 75 75 F F F <---
53822024-02-12T01:36:46.775Z 2 040-059 47 47 47 73 73 73 F F F
53832024-02-12T01:36:46.775Z 3 060-079 41 41 41 62 62 62 F F F
53842024-02-12T01:36:46.775Z 4 080-099 46 48 48 72 75 75 F F F <---
53852024-02-12T01:36:46.775Z 5 100-119 47 47 47 73 73 73 F F F
53862024-02-12T01:36:46.775Z 6 120-139 47 47 47 73 73 73 F F F
53872024-02-12T01:36:46.775Z 7 140-159 41 41 41 62 62 62 F F F
53882024-02-12T01:36:46.775Z 8 160-179 44 44 44 68 68 68 F T T <---
53892024-02-12T01:36:46.778Z 9 180-199 42 42 42 65 65 65 F F F
53902024-02-12T01:36:46.778Z 10 200-219 41 48 48 62 74 74 T F F <---
53912024-02-12T01:36:46.778Z 11 220-239 45 48 48 70 74 74 T F F <---
53922024-02-12T01:36:46.778Z 12 240-259 44 44 44 69 69 69 F F F
53932024-02-12T01:36:46.778Z 13 260-279 40 40 40 61 61 61 F F F
53942024-02-12T01:36:46.778Z 14 280-299 43 48 48 66 75 75 F F F <---
53952024-02-12T01:36:46.778Z 15 300-319 44 44 44 69 69 69 F F F
53962024-02-12T01:36:46.778Z 16 320-339 42 48 48 65 75 75 F F F <---
53972024-02-12T01:36:46.778Z 17 340-359 46 48 48 71 76 76 F F F <---
53982024-02-12T01:36:46.778Z 18 360-379 46 48 48 71 76 76 F F F <---
53992024-02-12T01:36:46.778Z 19 380-399 46 46 46 71 71 71 F F F
54002024-02-12T01:36:46.778Z 20 400-419 42 42 42 63 63 63 F F F
54012024-02-12T01:36:46.779Z 21 420-439 46 46 46 71 71 71 F F F
54022024-02-12T01:36:46.779Z 22 440-459 40 40 40 61 61 61 F F F
54032024-02-12T01:36:46.779Z 23 460-479 46 46 46 72 72 72 F F F
54042024-02-12T01:36:46.779Z 24 480-499 46 48 48 72 74 74 F F F <---
54052024-02-12T01:36:46.779Z 25 500-519 45 48 48 70 74 74 F F F <---
54062024-02-12T01:36:46.779Z 26 520-539 46 46 46 72 72 72 F T T <---
54072024-02-12T01:36:46.779Z 27 540-559 46 48 48 72 74 74 F F F <---
54082024-02-12T01:36:46.779Z 28 560-579 44 44 44 68 68 68 F T T <---
54092024-02-12T01:36:46.779Z 29 580-599 40 40 40 61 61 61 F T T <---
54102024-02-12T01:36:46.779ZMax gen: 48, Max flush: 76
54112024-02-12T01:36:46.779ZError: Difference in extent metadata found!
54122024-02-12T01:36:46.779ZA Difference in extent metadata is expected here
54132024-02-12T01:36:46.779ZOn loop 24
54142024-02-12T01:36:46.779Z
54152024-02-12T01:36:46.779ZVerifying data now
54162024-02-12T01:36:52.477ZNo change after verify
54172024-02-12T01:36:52.480ZLoop: 24 Downstairs dump after verify (and repair):
54182024-02-12T01:36:52.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54192024-02-12T01:36:52.492Z 0 000-019 42 42 42 64 64 64 F F F
54202024-02-12T01:36:52.492Z 1 020-039 48 48 48 75 75 75 F F F
54212024-02-12T01:36:52.492Z 2 040-059 47 47 47 73 73 73 F F F
54222024-02-12T01:36:52.492Z 3 060-079 41 41 41 62 62 62 F F F
54232024-02-12T01:36:52.492Z 4 080-099 48 48 48 75 75 75 F F F
54242024-02-12T01:36:52.492Z 5 100-119 47 47 47 73 73 73 F F F
54252024-02-12T01:36:52.492Z 6 120-139 47 47 47 73 73 73 F F F
54262024-02-12T01:36:52.502Z 7 140-159 41 41 41 62 62 62 F F F
54272024-02-12T01:36:52.502Z 8 160-179 49 49 49 77 77 77 F F F
54282024-02-12T01:36:52.502Z 9 180-199 42 42 42 65 65 65 F F F
54292024-02-12T01:36:52.503Z 10 200-219 48 48 48 74 74 74 F F F
54302024-02-12T01:36:52.503Z 11 220-239 48 48 48 74 74 74 F F F
54312024-02-12T01:36:52.503Z 12 240-259 44 44 44 69 69 69 F F F
54322024-02-12T01:36:52.503Z 13 260-279 40 40 40 61 61 61 F F F
54332024-02-12T01:36:52.503Z 14 280-299 48 48 48 75 75 75 F F F
54342024-02-12T01:36:52.503Z 15 300-319 44 44 44 69 69 69 F F F
54352024-02-12T01:36:52.503Z 16 320-339 48 48 48 75 75 75 F F F
54362024-02-12T01:36:52.503Z 17 340-359 48 48 48 76 76 76 F F F
54372024-02-12T01:36:52.503Z 18 360-379 48 48 48 76 76 76 F F F
54382024-02-12T01:36:52.503Z 19 380-399 46 46 46 71 71 71 F F F
54392024-02-12T01:36:52.503Z 20 400-419 42 42 42 63 63 63 F F F
54402024-02-12T01:36:52.503Z 21 420-439 46 46 46 71 71 71 F F F
54412024-02-12T01:36:52.503Z 22 440-459 40 40 40 61 61 61 F F F
54422024-02-12T01:36:52.503Z 23 460-479 46 46 46 72 72 72 F F F
54432024-02-12T01:36:52.503Z 24 480-499 48 48 48 74 74 74 F F F
54442024-02-12T01:36:52.503Z 25 500-519 48 48 48 74 74 74 F F F
54452024-02-12T01:36:52.503Z 26 520-539 49 49 49 77 77 77 F F F
54462024-02-12T01:36:52.503Z 27 540-559 48 48 48 74 74 74 F F F
54472024-02-12T01:36:52.503Z 28 560-579 49 49 49 77 77 77 F F F
54482024-02-12T01:36:52.503Z 29 580-599 49 49 49 77 77 77 F F F
54492024-02-12T01:36:52.503ZMax gen: 49, Max flush: 77
54502024-02-12T01:36:52.503Z
54512024-02-12T01:36:52.503ZBegin loop 25
54522024-02-12T01:36:52.503ZDownstairs to restart: 2
54532024-02-12T01:36:52.513Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.41735532Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
54542024-02-12T01:36:52.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
54552024-02-12T01:36:52.518Z,"time":"2024-02-12T01:36:51.417647026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
54562024-02-12T01:36:52.518Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417703344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
54572024-02-12T01:36:52.518Z{"msg":"Crucible e4b89316-eee3-426d-b345-1cdc1faddbfb has session id: 3828a210-c1db-4115-9067-cdaf0a0af0b3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417734202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54582024-02-12T01:36:52.518Z{"msg":"Upstairs opts: Upstairs UUID: e4b89316-eee3-426d-b345-1cdc1faddbfb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417790049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54592024-02-12T01:36:52.518Z{"msg":"Crucible stats registered with UUID: e4b89316-eee3-426d-b345-1cdc1faddbfb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417834147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54602024-02-12T01:36:52.518Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417862646Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
54612024-02-12T01:36:52.518Z{"msg":"e4b89316-eee3-426d-b345-1cdc1faddbfb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417920753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54622024-02-12T01:36:52.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.417966741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54632024-02-12T01:36:52.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418010809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54642024-02-12T01:36:52.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418057047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54652024-02-12T01:36:52.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418132023Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54662024-02-12T01:36:52.518Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.41819254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54672024-02-12T01:36:52.518Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418246327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54682024-02-12T01:36:52.518Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418277176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54692024-02-12T01:36:52.518Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418303025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54702024-02-12T01:36:52.518Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418337643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54712024-02-12T01:36:52.518Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418354522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54722024-02-12T01:36:52.518Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:51.418389651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54732024-02-12T01:36:52.518Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:51.418425359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54742024-02-12T01:36:52.519Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418458608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54752024-02-12T01:36:52.519Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:51.418493676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54762024-02-12T01:36:52.519Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418519434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54772024-02-12T01:36:52.519Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418552953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54782024-02-12T01:36:52.519Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418570692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54792024-02-12T01:36:52.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418902736Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54802024-02-12T01:36:52.519Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418940574Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54812024-02-12T01:36:52.519Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418964823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54822024-02-12T01:36:52.519Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.418989932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54832024-02-12T01:36:52.519Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.419442581Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54842024-02-12T01:36:52.519Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.419543626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54852024-02-12T01:36:52.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.419978295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54862024-02-12T01:36:52.519Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.420015763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54872024-02-12T01:36:52.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.420033352Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54882024-02-12T01:36:52.519Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.420061581Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54892024-02-12T01:36:52.519Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.42007714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54902024-02-12T01:36:52.519Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:51.420099279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54912024-02-12T01:36:53.515Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.41883679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54922024-02-12T01:36:53.515Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.419075709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54932024-02-12T01:36:53.522Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.420182866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54942024-02-12T01:36:53.522Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.420219335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54952024-02-12T01:36:53.522Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.420820366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54962024-02-12T01:36:53.522Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.42136604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54972024-02-12T01:36:53.522Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421433766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54982024-02-12T01:36:53.522Z{"msg":"[0]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 73, 73, 62, 77, 65, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421472535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
54992024-02-12T01:36:53.522Z{"msg":"[0]R generation[0..12]: [42, 48, 47, 41, 48, 47, 47, 41, 49, 42, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421505093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55002024-02-12T01:36:53.522Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421547331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55012024-02-12T01:36:53.522Z{"msg":"[1]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 73, 73, 62, 77, 65, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421588889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55022024-02-12T01:36:53.522ZWait for a query_work_queue command to finish before sending IO
55032024-02-12T01:36:53.522Z{"msg":"[1]R generation[0..12]: [42, 48, 47, 41, 48, 47, 47, 41, 49, 42, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421635937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55042024-02-12T01:36:53.522Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421677425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55052024-02-12T01:36:53.522Z{"msg":"[2]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 73, 73, 62, 77, 65, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421727383Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55062024-02-12T01:36:53.522Z{"msg":"[2]R generation[0..12]: [42, 48, 47, 41, 48, 47, 47, 41, 49, 42, 48, 48]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.42177155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55072024-02-12T01:36:53.522Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421805709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55082024-02-12T01:36:53.522Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421835187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55092024-02-12T01:36:53.522Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421872316Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55102024-02-12T01:36:53.522Z{"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421901064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55112024-02-12T01:36:53.523Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421931473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55122024-02-12T01:36:53.523Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.421954062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55132024-02-12T01:36:53.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.42198305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55142024-02-12T01:36:53.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.422007569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55152024-02-12T01:36:53.523Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
55162024-02-12T01:36:53.523Z,"time":"2024-02-12T01:36:52.422053787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55172024-02-12T01:36:53.523Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.422081916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55182024-02-12T01:36:53.523Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.422113154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55192024-02-12T01:36:53.523Z{"msg":"e4b89316-eee3-426d-b345-1cdc1faddbfb is now active with session: 3828a210-c1db-4115-9067-cdaf0a0af0b3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.422144613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55202024-02-12T01:36:53.523Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.422175951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335,"session_id":"3828a210-c1db-4115-9067-cdaf0a0af0b3"}
55212024-02-12T01:36:53.523Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:52.42220658Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
55222024-02-12T01:36:53.523ZLoading write count information from file "/tmp/test_repair_verify.data"
55232024-02-12T01:36:53.523ZRun Repair workload
55242024-02-12T01:36:53.526Z01/30 Flush
55252024-02-12T01:36:53.526Z02/30 Write block 524 len 16384 data: 4 3 3 2
55262024-02-12T01:36:53.526Z03/30 Flush
55272024-02-12T01:36:53.526Z04/30 Write block 236 len 20480 data: 5 6 6 6 4
55282024-02-12T01:36:53.526Z05/30 Write block 324 len 24576 data: 4 3 3 3 3 3
55292024-02-12T01:36:53.526Z06/30 Read block 179 len 16384
55302024-02-12T01:36:53.526Z07/30 Read block 465 len 36864
55312024-02-12T01:36:53.526Z08/30 Write block 293 len 12288 data: 3 3 4
55322024-02-12T01:36:53.529Z09/30 Read block 537 len 16384
55332024-02-12T01:36:53.529Z10/30 Write block 522 len 20480 data: 4 4 5 4 4
55342024-02-12T01:36:53.529Z11/30 Write block 535 len 8192 data: 8 7
55352024-02-12T01:36:53.529Z12/30 Read block 96 len 12288
55362024-02-12T01:36:53.529Z13/30 Read block 416 len 40960
55372024-02-12T01:36:53.532Z14/30 Write block 187 len 36864 data: 7 8 8 6 5 5 4 3 2
55382024-02-12T01:36:53.532Z15/30 Read block 183 len 36864
55392024-02-12T01:36:53.535Z16/30 Read block 108 len 12288
55402024-02-12T01:36:53.535Z17/30 Write block 206 len 12288 data: 5 7 7
55412024-02-12T01:36:53.535Z18/30 Read block 569 len 12288
55422024-02-12T01:36:53.535Z19/30 Read block 515 len 40960
55432024-02-12T01:36:53.538Z20/30 Read block 165 len 40960
55442024-02-12T01:36:53.538Z21/30 Write block 246 len 32768 data: 3 4 4 4 3 3 3 4
55452024-02-12T01:36:53.538Z22/30 Read block 183 len 4096
55462024-02-12T01:36:53.541Z23/30 Flush
55472024-02-12T01:36:53.543Z24/30 Flush
55482024-02-12T01:36:53.543Z25/30 Write block 116 len 40960 data: 7 8 8 9 8 8 7 9 8 7
55492024-02-12T01:36:53.543Z26/30 Flush
55502024-02-12T01:36:53.546Z27/30 Write block 543 len 32768 data: 4 4 5 6 6 6 5 7
55512024-02-12T01:36:53.549Z28/30 Write block 546 len 32768 data: 7 7 7 6 8 7 7 6
55522024-02-12T01:36:53.549Z29/30 Write block 490 len 24576 data: 6 4 3 3 3 5
55532024-02-12T01:36:53.550Z30/30 Write block 228 len 8192 data: 4 4
55542024-02-12T01:36:53.550Z----------------------------------------------------------------
55552024-02-12T01:36:53.550Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:29
55562024-02-12T01:36:53.550ZStates: Active Active Active
55572024-02-12T01:36:53.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55582024-02-12T01:36:53.550Z 2 Acked 1001 Write 4 Done Done Sent false
55592024-02-12T01:36:53.550Z 3 Acked 1002 Flush 0 Done Done Sent false
55602024-02-12T01:36:53.550Z 4 Acked 1003 Write 5 Done Done Sent false
55612024-02-12T01:36:53.550Z 5 Acked 1004 Write 6 Done Done Sent false
55622024-02-12T01:36:53.550Z 6 Acked 1005 Read 4 Done Done Sent false
55632024-02-12T01:36:53.550Z 7 Acked 1006 Read 9 Done Done Sent false
55642024-02-12T01:36:53.550Z 8 Acked 1007 Write 3 Done Done Sent false
55652024-02-12T01:36:53.550Z 9 Acked 1008 Read 4 Done Done Sent false
55662024-02-12T01:36:53.550Z 10 Acked 1009 Write 5 Done Done Sent false
55672024-02-12T01:36:53.550Z 11 Acked 1010 Write 2 Done Done Sent false
55682024-02-12T01:36:53.550Z 12 Acked 1011 Read 3 Done Done Sent false
55692024-02-12T01:36:53.550Z 13 Acked 1012 Read 10 Done Done Sent false
55702024-02-12T01:36:53.550Z 14 Acked 1013 Write 9 Done Done Sent false
55712024-02-12T01:36:53.550Z 15 Acked 1014 Read 9 Done Done Sent false
55722024-02-12T01:36:53.550Z 16 Acked 1015 Read 3 Done Done Sent false
55732024-02-12T01:36:53.550Z 17 Acked 1016 Write 3 Done Done Sent false
55742024-02-12T01:36:53.550Z 18 Acked 1017 Read 3 Done Done Sent false
55752024-02-12T01:36:53.550Z 19 Acked 1018 Read 10 Done Done Sent false
55762024-02-12T01:36:53.551Z 20 Acked 1019 Read 10 Done Done Sent false
55772024-02-12T01:36:53.551Z 21 Acked 1020 Write 8 Done Done Sent false
55782024-02-12T01:36:53.551Z 22 Acked 1021 Read 1 Done Done Sent false
55792024-02-12T01:36:53.551Z 23 Acked 1022 Flush 0 Done Done Sent false
55802024-02-12T01:36:53.551Z 24 Acked 1023 Flush 0 Done Done Sent false
55812024-02-12T01:36:53.551Z 25 Acked 1024 Write 10 Done Done Sent false
55822024-02-12T01:36:53.551Z 26 Acked 1025 Flush 0 Done Done Sent false
55832024-02-12T01:36:53.551Z 27 Acked 1026 Write 8 Done Done Sent false
55842024-02-12T01:36:53.551Z 28 Acked 1027 Write 8 Done Done Sent false
55852024-02-12T01:36:53.551Z 29 Acked 1028 Write 6 Sent Sent Sent false
55862024-02-12T01:36:53.551Z 30 Acked 1029 Write 2 Sent Sent Sent false
55872024-02-12T01:36:53.551Z STATES DS:0 DS:1 DS:2 TOTAL
55882024-02-12T01:36:53.551Z New 0 0 0 0
55892024-02-12T01:36:53.551Z Sent 2 2 29 33
55902024-02-12T01:36:53.551Z Done 27 27 0 54
55912024-02-12T01:36:53.551Z Skipped 0 0 0 0
55922024-02-12T01:36:53.551Z Error 0 0 0 0
55932024-02-12T01:36:53.551ZLast Flush: 1025 1025 1000
55942024-02-12T01:36:53.551ZDownstairs last five completed: 1000
55952024-02-12T01:36:53.551ZUpstairs last five completed: 30 29 28 27 26
55962024-02-12T01:36:53.551Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:52.453026669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1335}
55972024-02-12T01:36:53.551ZWrote out file "/tmp/test_repair_verify.data"
55982024-02-12T01:36:53.554Z
55992024-02-12T01:36:53.557ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
56002024-02-12T01:36:53.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56012024-02-12T01:36:53.573Z 0 000-019 42 42 42 64 64 64 F F F
56022024-02-12T01:36:53.573Z 1 020-039 48 48 48 75 75 75 F F F
56032024-02-12T01:36:53.573Z 2 040-059 47 47 47 73 73 73 F F F
56042024-02-12T01:36:53.573Z 3 060-079 41 41 41 62 62 62 F F F
56052024-02-12T01:36:53.573Z 4 080-099 48 48 48 75 75 75 F F F
56062024-02-12T01:36:53.573Z 5 100-119 50 50 47 82 82 73 F F F <---
56072024-02-12T01:36:53.573Z 6 120-139 50 50 47 82 82 73 F F F <---
56082024-02-12T01:36:53.573Z 7 140-159 41 41 41 62 62 62 F F F
56092024-02-12T01:36:53.573Z 8 160-179 49 49 49 77 77 77 F F F
56102024-02-12T01:36:53.573Z 9 180-199 50 50 42 80 80 65 F F F <---
56112024-02-12T01:36:53.574Z 10 200-219 50 50 48 80 80 74 F F F <---
56122024-02-12T01:36:53.574Z 11 220-239 50 50 48 80 80 74 T T F <---
56132024-02-12T01:36:53.574Z 12 240-259 50 50 44 80 80 69 F F F <---
56142024-02-12T01:36:53.577Z 13 260-279 40 40 40 61 61 61 F F F
56152024-02-12T01:36:53.577Z 14 280-299 50 50 48 80 80 75 F F F <---
56162024-02-12T01:36:53.577Z 15 300-319 44 44 44 69 69 69 F F F
56172024-02-12T01:36:53.577Z 16 320-339 50 50 48 80 80 75 F F F <---
56182024-02-12T01:36:53.577Z 17 340-359 48 48 48 76 76 76 F F F
56192024-02-12T01:36:53.577Z 18 360-379 48 48 48 76 76 76 F F F
56202024-02-12T01:36:53.577Z 19 380-399 46 46 46 71 71 71 F F F
56212024-02-12T01:36:53.577Z 20 400-419 42 42 42 63 63 63 F F F
56222024-02-12T01:36:53.577Z 21 420-439 46 46 46 71 71 71 F F F
56232024-02-12T01:36:53.577Z 22 440-459 40 40 40 61 61 61 F F F
56242024-02-12T01:36:53.577Z 23 460-479 46 46 46 72 72 72 F F F
56252024-02-12T01:36:53.577Z 24 480-499 48 48 48 74 74 74 T T F <---
56262024-02-12T01:36:53.577Z 25 500-519 48 48 48 74 74 74 F F F
56272024-02-12T01:36:53.577Z 26 520-539 50 50 49 80 80 77 F F F <---
56282024-02-12T01:36:53.578Z 27 540-559 48 48 48 74 74 74 T T F <---
56292024-02-12T01:36:53.578Z 28 560-579 49 49 49 77 77 77 F F F
56302024-02-12T01:36:53.578Z 29 580-599 49 49 49 77 77 77 F F F
56312024-02-12T01:36:53.578ZMax gen: 50, Max flush: 82
56322024-02-12T01:36:53.578ZError: Difference in extent metadata found!
56332024-02-12T01:36:53.578ZA Difference in extent metadata is expected here
56342024-02-12T01:36:53.578ZOn loop 25
56352024-02-12T01:36:53.578Z
56362024-02-12T01:36:53.583ZVerifying data now
56372024-02-12T01:36:58.124ZNo change after verify
56382024-02-12T01:36:58.124ZLoop: 25 Downstairs dump after verify (and repair):
56392024-02-12T01:36:58.139ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56402024-02-12T01:36:58.139Z 0 000-019 42 42 42 64 64 64 F F F
56412024-02-12T01:36:58.139Z 1 020-039 48 48 48 75 75 75 F F F
56422024-02-12T01:36:58.139Z 2 040-059 47 47 47 73 73 73 F F F
56432024-02-12T01:36:58.139Z 3 060-079 41 41 41 62 62 62 F F F
56442024-02-12T01:36:58.139Z 4 080-099 48 48 48 75 75 75 F F F
56452024-02-12T01:36:58.140Z 5 100-119 50 50 50 82 82 82 F F F
56462024-02-12T01:36:58.140Z 6 120-139 50 50 50 82 82 82 F F F
56472024-02-12T01:36:58.140Z 7 140-159 41 41 41 62 62 62 F F F
56482024-02-12T01:36:58.140Z 8 160-179 49 49 49 77 77 77 F F F
56492024-02-12T01:36:58.140Z 9 180-199 50 50 50 80 80 80 F F F
56502024-02-12T01:36:58.140Z 10 200-219 50 50 50 80 80 80 F F F
56512024-02-12T01:36:58.140Z 11 220-239 51 51 51 83 83 83 F F F
56522024-02-12T01:36:58.140Z 12 240-259 50 50 50 80 80 80 F F F
56532024-02-12T01:36:58.140Z 13 260-279 40 40 40 61 61 61 F F F
56542024-02-12T01:36:58.140Z 14 280-299 50 50 50 80 80 80 F F F
56552024-02-12T01:36:58.143Z 15 300-319 44 44 44 69 69 69 F F F
56562024-02-12T01:36:58.144Z 16 320-339 50 50 50 80 80 80 F F F
56572024-02-12T01:36:58.144Z 17 340-359 48 48 48 76 76 76 F F F
56582024-02-12T01:36:58.144Z 18 360-379 48 48 48 76 76 76 F F F
56592024-02-12T01:36:58.144Z 19 380-399 46 46 46 71 71 71 F F F
56602024-02-12T01:36:58.144Z 20 400-419 42 42 42 63 63 63 F F F
56612024-02-12T01:36:58.144Z 21 420-439 46 46 46 71 71 71 F F F
56622024-02-12T01:36:58.144Z 22 440-459 40 40 40 61 61 61 F F F
56632024-02-12T01:36:58.144Z 23 460-479 46 46 46 72 72 72 F F F
56642024-02-12T01:36:58.144Z 24 480-499 51 51 51 83 83 83 F F F
56652024-02-12T01:36:58.144Z 25 500-519 48 48 48 74 74 74 F F F
56662024-02-12T01:36:58.144Z 26 520-539 50 50 50 80 80 80 F F F
56672024-02-12T01:36:58.144Z 27 540-559 51 51 51 83 83 83 F F F
56682024-02-12T01:36:58.145Z 28 560-579 49 49 49 77 77 77 F F F
56692024-02-12T01:36:58.145Z 29 580-599 49 49 49 77 77 77 F F F
56702024-02-12T01:36:58.145ZMax gen: 51, Max flush: 83
56712024-02-12T01:36:58.145Z
56722024-02-12T01:36:58.145ZBegin loop 26
56732024-02-12T01:36:58.145ZDownstairs to restart: 2
56742024-02-12T01:36:58.161Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.063672698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
56752024-02-12T01:36:58.161Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.063903327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
56762024-02-12T01:36:58.161Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.063923676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
56772024-02-12T01:36:58.168Z{"msg":"Crucible 5192921b-efe9-4f16-ac04-39ba4d0dd0aa has session id: 0247c593-05d5-48d2-91fe-98c79a629ec6","v":0,"name":"crucible","level":30Crucible runtime is spawned
56782024-02-12T01:36:58.168Z,"time":"2024-02-12T01:36:57.063957824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56792024-02-12T01:36:58.168Z{"msg":"Upstairs opts: Upstairs UUID: 5192921b-efe9-4f16-ac04-39ba4d0dd0aa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064024141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56802024-02-12T01:36:58.168Z{"msg":"Crucible stats registered with UUID: 5192921b-efe9-4f16-ac04-39ba4d0dd0aa","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.06404629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56812024-02-12T01:36:58.168Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064077908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
56822024-02-12T01:36:58.168Z{"msg":"5192921b-efe9-4f16-ac04-39ba4d0dd0aa active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064153135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56832024-02-12T01:36:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064207232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56842024-02-12T01:36:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064235551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56852024-02-12T01:36:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.06425936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56862024-02-12T01:36:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064278609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56872024-02-12T01:36:58.168Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064299688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56882024-02-12T01:36:58.168Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064314167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56892024-02-12T01:36:58.169Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064337346Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56902024-02-12T01:36:58.169Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064357635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56912024-02-12T01:36:58.169Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064381174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56922024-02-12T01:36:58.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064524817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56932024-02-12T01:36:58.169Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064712358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56942024-02-12T01:36:58.169Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:57.064805154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56952024-02-12T01:36:58.169Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:57.064843412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56962024-02-12T01:36:58.169Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:57.064909729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56972024-02-12T01:36:58.169Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064949307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56982024-02-12T01:36:58.169Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.064977845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
56992024-02-12T01:36:58.169Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065061772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57002024-02-12T01:36:58.169Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065259592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57012024-02-12T01:36:58.169Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.06529904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57022024-02-12T01:36:58.169Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065432324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57032024-02-12T01:36:58.169Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065452913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57042024-02-12T01:36:58.169Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065847304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57052024-02-12T01:36:58.169Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.065897082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57062024-02-12T01:36:58.169Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066438886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57072024-02-12T01:36:58.169Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066478954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57082024-02-12T01:36:58.169Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066495103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57092024-02-12T01:36:58.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066516522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57102024-02-12T01:36:58.170Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066537831Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57112024-02-12T01:36:58.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:57.066552261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57122024-02-12T01:36:59.163Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.066426538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57132024-02-12T01:36:59.163Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.066546542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57142024-02-12T01:36:59.167Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.067593842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57152024-02-12T01:36:59.167Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.0676385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57162024-02-12T01:36:59.167Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068237761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57172024-02-12T01:36:59.167Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068770226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57182024-02-12T01:36:59.168Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068822233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57192024-02-12T01:36:59.168Z{"msg":"[0]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 82, 82, 62, 77, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068841302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57202024-02-12T01:36:59.168Z{"msg":"[0]R generation[0..12]: [42, 48, 47, 41, 48, 50, 50, 41, 49, 50, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068863961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57212024-02-12T01:36:59.168Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.06887996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57222024-02-12T01:36:59.168Z{"msg":"[1]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 82, 82, 62, 77, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068901369Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57232024-02-12T01:36:59.168Z{"msg":"[1]R generation[0..12]: [42, 48, 47, 41, 48, 50, 50, 41, 49, 50, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068917348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57242024-02-12T01:36:59.168Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068932428Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57252024-02-12T01:36:59.168Z{"msg":"[2]R flush_numbers[0..12]: [64, 75, 73, 62, 75, 82, 82, 62, 77, 80, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068954787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57262024-02-12T01:36:59.168Z{"msg":"[2]R generation[0..12]: [42, 48, 47, 41, 48, 50, 50, 41, 49, 50, 50, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068969056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57272024-02-12T01:36:59.168Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.068989555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}Wait for a query_work_queue command to finish before sending IO
57282024-02-12T01:36:59.168Z
57292024-02-12T01:36:59.168Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069013794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57302024-02-12T01:36:59.168Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069031813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57312024-02-12T01:36:59.168Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069049732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57322024-02-12T01:36:59.168Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069065732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57332024-02-12T01:36:59.168Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069083871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57342024-02-12T01:36:59.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.06909687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57352024-02-12T01:36:59.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069139208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57362024-02-12T01:36:59.168Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069182536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57372024-02-12T01:36:59.168Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069201825Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57382024-02-12T01:36:59.168Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069227364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57392024-02-12T01:36:59.168Z{"msg":"5192921b-efe9-4f16-ac04-39ba4d0dd0aa is now active with session: 0247c593-05d5-48d2-91fe-98c79a629ec6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069245773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57402024-02-12T01:36:59.168Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.069278701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343,"session_id":"0247c593-05d5-48d2-91fe-98c79a629ec6"}
57412024-02-12T01:36:59.168Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:36:58.06930958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
57422024-02-12T01:36:59.169ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57432024-02-12T01:36:59.169ZLoading write count information from file "/tmp/test_repair_verify.data"
57442024-02-12T01:36:59.169ZRun Repair workload
57452024-02-12T01:36:59.176Z01/30 Write block 319 len 20480 data: 4 4 5 6 4
57462024-02-12T01:36:59.176Z02/30 Read block 451 len 4096
57472024-02-12T01:36:59.176Z03/30 Flush
57482024-02-12T01:36:59.176Z04/30 Write block 100 len 8192 data: 2 4
57492024-02-12T01:36:59.176Z05/30 Write block 352 len 8192 data: 8 6
57502024-02-12T01:36:59.176Z06/30 Flush
57512024-02-12T01:36:59.179Z07/30 Write block 94 len 4096 data: 3
57522024-02-12T01:36:59.179Z08/30 Flush
57532024-02-12T01:36:59.179Z09/30 Write block 520 len 16384 data: 4 4 5 5
57542024-02-12T01:36:59.179Z10/30 Flush
57552024-02-12T01:36:59.179Z11/30 Write block 5 len 20480 data: 5 5 5 5 5
57562024-02-12T01:36:59.179Z12/30 Flush
57572024-02-12T01:36:59.182Z13/30 Write block 218 len 12288 data: 3 4 4
57582024-02-12T01:36:59.182Z14/30 Write block 178 len 4096 data: 7
57592024-02-12T01:36:59.182Z15/30 Flush
57602024-02-12T01:36:59.185Z16/30 Write block 451 len 40960 data: 3 4 4 6 6 5 5 6 5 4
57612024-02-12T01:36:59.185Z17/30 Flush
57622024-02-12T01:36:59.188Z18/30 Write block 150 len 12288 data: 5 5 5
57632024-02-12T01:36:59.188Z19/30 Flush
57642024-02-12T01:36:59.191Z20/30 Write block 38 len 12288 data: 6 5 5
57652024-02-12T01:36:59.191Z21/30 Write block 579 len 12288 data: 4 4 5
57662024-02-12T01:36:59.191Z22/30 Write block 273 len 16384 data: 3 4 5 5
57672024-02-12T01:36:59.191Z23/30 Read block 307 len 40960
57682024-02-12T01:36:59.194Z24/30 Flush
57692024-02-12T01:36:59.194Z25/30 Write block 317 len 4096 data: 3
57702024-02-12T01:36:59.194Z26/30 Read block 35 len 8192
57712024-02-12T01:36:59.197Z27/30 Read block 335 len 20480
57722024-02-12T01:36:59.197Z28/30 Read block 142 len 24576
57732024-02-12T01:36:59.197Z29/30 Read block 169 len 28672
57742024-02-12T01:36:59.200Z30/30 Write block 495 len 32768 data: 6 5 4 5 6 5 6 6
57752024-02-12T01:36:59.200Z----------------------------------------------------------------
57762024-02-12T01:36:59.200Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:22
57772024-02-12T01:36:59.200ZStates: Active Active Active
57782024-02-12T01:36:59.200ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57792024-02-12T01:36:59.200Z 9 Acked 1008 Write 4 Done Done Sent false
57802024-02-12T01:36:59.200Z 10 Acked 1009 Flush 0 Done Done Sent false
57812024-02-12T01:36:59.200Z 11 Acked 1010 Write 5 Done Done Sent false
57822024-02-12T01:36:59.200Z 12 Acked 1011 Flush 0 Done Done Sent false
57832024-02-12T01:36:59.200Z 13 Acked 1012 Write 3 Done Done Sent false
57842024-02-12T01:36:59.200Z 14 Acked 1013 Write 1 Done Done Sent false
57852024-02-12T01:36:59.201Z 15 Acked 1014 Flush 0 Done Done Sent false
57862024-02-12T01:36:59.201Z 16 Acked 1015 Write 10 Done Done Sent false
57872024-02-12T01:36:59.201Z 17 Acked 1016 Flush 0 Done Done Sent false
57882024-02-12T01:36:59.201Z 18 Acked 1017 Write 3 Done Done Sent false
57892024-02-12T01:36:59.201Z 19 Acked 1018 Flush 0 Done Done Sent false
57902024-02-12T01:36:59.201Z 20 Acked 1019 Write 3 Done Done Sent false
57912024-02-12T01:36:59.201Z 21 Acked 1020 Write 3 Done Done Sent false
57922024-02-12T01:36:59.201Z 22 Acked 1021 Write 4 Done Done Sent false
57932024-02-12T01:36:59.201Z 23 Acked 1022 Read 10 Done Done Sent false
57942024-02-12T01:36:59.201Z 24 Acked 1023 Flush 0 Done Done Sent false
57952024-02-12T01:36:59.201Z 25 Acked 1024 Write 1 Done Done Sent false
57962024-02-12T01:36:59.201Z 26 Acked 1025 Read 2 Done Done Sent false
57972024-02-12T01:36:59.201Z 27 Acked 1026 Read 5 Done Done Sent false
57982024-02-12T01:36:59.201Z 28 Acked 1027 Read 6 Done Done Sent false
57992024-02-12T01:36:59.201Z 29 Acked 1028 Read 7 Done Done Sent false
58002024-02-12T01:36:59.201Z 30 Acked 1029 Write 8 Sent Sent Sent false
58012024-02-12T01:36:59.201Z STATES DS:0 DS:1 DS:2 TOTAL
58022024-02-12T01:36:59.201Z New 0 0 0 0
58032024-02-12T01:36:59.201Z Sent 1 1 22 24
58042024-02-12T01:36:59.201Z Done 21 21 0 42
58052024-02-12T01:36:59.201Z Skipped 0 0 0 0
58062024-02-12T01:36:59.201Z Error 0 0 0 0
58072024-02-12T01:36:59.202ZLast Flush: 1023 1023 1007
58082024-02-12T01:36:59.202ZDownstairs last five completed: 1007 1006 1005 1004 1003
58092024-02-12T01:36:59.202ZUpstairs last five completed: 30 29 28 27 26
58102024-02-12T01:36:59.202Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:36:58.102466438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1343}
58112024-02-12T01:36:59.202ZWrote out file "/tmp/test_repair_verify.data"
58122024-02-12T01:36:59.205Z
58132024-02-12T01:36:59.209ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
58142024-02-12T01:36:59.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58152024-02-12T01:36:59.224Z 0 000-019 52 52 42 88 88 64 F F F <---
58162024-02-12T01:36:59.224Z 1 020-039 52 52 48 92 92 75 F F F <---
58172024-02-12T01:36:59.224Z 2 040-059 52 52 47 92 92 73 F F F <---
58182024-02-12T01:36:59.224Z 3 060-079 41 41 41 62 62 62 F F F
58192024-02-12T01:36:59.224Z 4 080-099 52 52 52 86 86 86 F F F
58202024-02-12T01:36:59.224Z 5 100-119 52 52 52 85 85 85 F F F
58212024-02-12T01:36:59.224Z 6 120-139 50 50 50 82 82 82 F F F
58222024-02-12T01:36:59.225Z 7 140-159 52 52 41 91 91 62 F F F <---
58232024-02-12T01:36:59.225Z 8 160-179 52 52 49 89 89 77 F F F <---
58242024-02-12T01:36:59.225Z 9 180-199 50 50 50 80 80 80 F F F
58252024-02-12T01:36:59.228Z 10 200-219 52 52 50 89 89 80 F F F <---
58262024-02-12T01:36:59.228Z 11 220-239 52 52 51 89 89 83 F F F <---
58272024-02-12T01:36:59.228Z 12 240-259 50 50 50 80 80 80 F F F
58282024-02-12T01:36:59.228Z 13 260-279 52 52 40 92 92 61 F F F <---
58292024-02-12T01:36:59.228Z 14 280-299 50 50 50 80 80 80 F F F
58302024-02-12T01:36:59.228Z 15 300-319 52 52 52 84 84 84 T T F <---
58312024-02-12T01:36:59.228Z 16 320-339 52 52 52 84 84 84 F F F
58322024-02-12T01:36:59.228Z 17 340-359 52 52 52 85 85 85 F F F
58332024-02-12T01:36:59.228Z 18 360-379 48 48 48 76 76 76 F F F
58342024-02-12T01:36:59.228Z 19 380-399 46 46 46 71 71 71 F F F
58352024-02-12T01:36:59.228Z 20 400-419 42 42 42 63 63 63 F F F
58362024-02-12T01:36:59.228Z 21 420-439 46 46 46 71 71 71 F F F
58372024-02-12T01:36:59.228Z 22 440-459 52 52 40 90 90 61 F F F <---
58382024-02-12T01:36:59.228Z 23 460-479 52 52 46 90 90 72 F F F <---
58392024-02-12T01:36:59.228Z 24 480-499 51 51 51 83 83 83 T T F <---
58402024-02-12T01:36:59.228Z 25 500-519 48 48 48 74 74 74 T T F <---
58412024-02-12T01:36:59.228Z 26 520-539 52 52 50 87 87 80 F F F <---
58422024-02-12T01:36:59.228Z 27 540-559 51 51 51 83 83 83 F F F
58432024-02-12T01:36:59.228Z 28 560-579 52 52 49 92 92 77 F F F <---
58442024-02-12T01:36:59.228Z 29 580-599 52 52 49 92 92 77 F F F <---
58452024-02-12T01:36:59.229ZMax gen: 52, Max flush: 92
58462024-02-12T01:36:59.229ZError: Difference in extent metadata found!
58472024-02-12T01:36:59.229ZA Difference in extent metadata is expected here
58482024-02-12T01:36:59.229ZOn loop 26
58492024-02-12T01:36:59.229Z
58502024-02-12T01:36:59.232ZVerifying data now
58512024-02-12T01:37:03.965ZNo change after verify
58522024-02-12T01:37:03.965ZLoop: 26 Downstairs dump after verify (and repair):
58532024-02-12T01:37:03.980ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58542024-02-12T01:37:03.980Z 0 000-019 52 52 52 88 88 88 F F F
58552024-02-12T01:37:03.980Z 1 020-039 52 52 52 92 92 92 F F F
58562024-02-12T01:37:03.980Z 2 040-059 52 52 52 92 92 92 F F F
58572024-02-12T01:37:03.980Z 3 060-079 41 41 41 62 62 62 F F F
58582024-02-12T01:37:03.980Z 4 080-099 52 52 52 86 86 86 F F F
58592024-02-12T01:37:03.980Z 5 100-119 52 52 52 85 85 85 F F F
58602024-02-12T01:37:03.980Z 6 120-139 50 50 50 82 82 82 F F F
58612024-02-12T01:37:03.980Z 7 140-159 52 52 52 91 91 91 F F F
58622024-02-12T01:37:03.980Z 8 160-179 52 52 52 89 89 89 F F F
58632024-02-12T01:37:03.980Z 9 180-199 50 50 50 80 80 80 F F F
58642024-02-12T01:37:03.985Z 10 200-219 52 52 52 89 89 89 F F F
58652024-02-12T01:37:03.985Z 11 220-239 52 52 52 89 89 89 F F F
58662024-02-12T01:37:03.985Z 12 240-259 50 50 50 80 80 80 F F F
58672024-02-12T01:37:03.985Z 13 260-279 52 52 52 92 92 92 F F F
58682024-02-12T01:37:03.985Z 14 280-299 50 50 50 80 80 80 F F F
58692024-02-12T01:37:03.985Z 15 300-319 53 53 53 93 93 93 F F F
58702024-02-12T01:37:03.985Z 16 320-339 52 52 52 84 84 84 F F F
58712024-02-12T01:37:03.985Z 17 340-359 52 52 52 85 85 85 F F F
58722024-02-12T01:37:03.985Z 18 360-379 48 48 48 76 76 76 F F F
58732024-02-12T01:37:03.985Z 19 380-399 46 46 46 71 71 71 F F F
58742024-02-12T01:37:03.985Z 20 400-419 42 42 42 63 63 63 F F F
58752024-02-12T01:37:03.985Z 21 420-439 46 46 46 71 71 71 F F F
58762024-02-12T01:37:03.985Z 22 440-459 52 52 52 90 90 90 F F F
58772024-02-12T01:37:03.985Z 23 460-479 52 52 52 90 90 90 F F F
58782024-02-12T01:37:03.985Z 24 480-499 53 53 53 93 93 93 F F F
58792024-02-12T01:37:03.985Z 25 500-519 53 53 53 93 93 93 F F F
58802024-02-12T01:37:03.985Z 26 520-539 52 52 52 87 87 87 F F F
58812024-02-12T01:37:03.985Z 27 540-559 51 51 51 83 83 83 F F F
58822024-02-12T01:37:03.985Z 28 560-579 52 52 52 92 92 92 F F F
58832024-02-12T01:37:03.985Z 29 580-599 52 52 52 92 92 92 F F F
58842024-02-12T01:37:03.986ZMax gen: 53, Max flush: 93
58852024-02-12T01:37:03.986Z
58862024-02-12T01:37:03.986ZBegin loop 27
58872024-02-12T01:37:03.986ZDownstairs to restart: 0
58882024-02-12T01:37:04.002Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905025702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
58892024-02-12T01:37:04.003Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\"Crucible runtime is spawned
58902024-02-12T01:37:04.003Z,\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905396925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
58912024-02-12T01:37:04.003Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905450792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
58922024-02-12T01:37:04.008Z{"msg":"Crucible 324026a6-6274-4324-8b14-709078157b20 has session id: 4d7b8fb2-5a1e-4488-bd32-14fb45f02e10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905482501Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58932024-02-12T01:37:04.008Z{"msg":"Upstairs opts: Upstairs UUID: 324026a6-6274-4324-8b14-709078157b20, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905537708Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58942024-02-12T01:37:04.008Z{"msg":"Crucible stats registered with UUID: 324026a6-6274-4324-8b14-709078157b20","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905555897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58952024-02-12T01:37:04.008Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905577856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
58962024-02-12T01:37:04.008Z{"msg":"324026a6-6274-4324-8b14-709078157b20 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905647563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58972024-02-12T01:37:04.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905678631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58982024-02-12T01:37:04.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.90570607Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
58992024-02-12T01:37:04.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905743568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59002024-02-12T01:37:04.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905772037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59012024-02-12T01:37:04.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905818745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59022024-02-12T01:37:04.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905850963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59032024-02-12T01:37:04.008Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905892771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59042024-02-12T01:37:04.008Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905939469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59052024-02-12T01:37:04.008Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.905966948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59062024-02-12T01:37:04.008Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:02.906040294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59072024-02-12T01:37:04.008Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:02.906208876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59082024-02-12T01:37:04.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906278943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59092024-02-12T01:37:04.008Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:02.906351589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59102024-02-12T01:37:04.008Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906381468Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59112024-02-12T01:37:04.009Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906419506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59122024-02-12T01:37:04.009Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906461384Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59132024-02-12T01:37:04.009Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906488883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59142024-02-12T01:37:04.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906834036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59152024-02-12T01:37:04.009Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.906870665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59162024-02-12T01:37:04.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907001328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59172024-02-12T01:37:04.009Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907035966Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59182024-02-12T01:37:04.009Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907399929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59192024-02-12T01:37:04.009Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907531283Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59202024-02-12T01:37:04.009Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907856607Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59212024-02-12T01:37:04.009Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907904075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59222024-02-12T01:37:04.009Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907948153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59232024-02-12T01:37:04.009Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.907974802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59242024-02-12T01:37:04.009Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.90800493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59252024-02-12T01:37:04.009Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:02.90801976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59262024-02-12T01:37:05.004Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.907625519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59272024-02-12T01:37:05.005Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.907707585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59282024-02-12T01:37:05.012Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.908603422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59292024-02-12T01:37:05.012Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.908645851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59302024-02-12T01:37:05.012Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909125828Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59312024-02-12T01:37:05.012Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909652623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59322024-02-12T01:37:05.013Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.90970471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59332024-02-12T01:37:05.013Z{"msg":"[0]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 82, 91, 89, 80, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909731189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59342024-02-12T01:37:05.013Z{"msg":"[0]R generation[0..12]: [52, 52, 52, 41, 52, 52, 50, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909763857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59352024-02-12T01:37:05.013Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909792566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59362024-02-12T01:37:05.013Z{"msg":"[1]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 82, 91, 89, 80, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909807445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59372024-02-12T01:37:05.013Z{"msg":"[1]R generation[0..12]: [52, 52, 52, 41, 52, 52, 50, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909821904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59382024-02-12T01:37:05.013Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909836314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59392024-02-12T01:37:05.013Z{"msg":"[2]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 82, 91, 89, 80, 89, 89]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909857803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59402024-02-12T01:37:05.013Z{"msg":"[2]R generation[0..12]: [52, 52, 52, 41, 52, 52, 50, 52, 52, 50, 52, 52]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909872092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59412024-02-12T01:37:05.013ZWait for a query_work_queue command to finish before sending IO
59422024-02-12T01:37:05.013Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909898251Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59432024-02-12T01:37:05.013Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.90991327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59442024-02-12T01:37:05.013Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909927699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59452024-02-12T01:37:05.013Z{"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909947298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59462024-02-12T01:37:05.013Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909961338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59472024-02-12T01:37:05.013Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909975007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59482024-02-12T01:37:05.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.909996096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59492024-02-12T01:37:05.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910009815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59502024-02-12T01:37:05.013Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910023805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59512024-02-12T01:37:05.013Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910036984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59522024-02-12T01:37:05.013Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910049394Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59532024-02-12T01:37:05.013Z{"msg":"324026a6-6274-4324-8b14-709078157b20 is now active with session: 4d7b8fb2-5a1e-4488-bd32-14fb45f02e10","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910070243Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59542024-02-12T01:37:05.013Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910083102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351,"session_id":"4d7b8fb2-5a1e-4488-bd32-14fb45f02e10"}
59552024-02-12T01:37:05.013Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:03.910096521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
59562024-02-12T01:37:05.014ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59572024-02-12T01:37:05.014ZLoading write count information from file "/tmp/test_repair_verify.data"
59582024-02-12T01:37:05.014ZRun Repair workload
59592024-02-12T01:37:05.014Z01/30 Write block 584 len 12288 data: 5 6 6
59602024-02-12T01:37:05.017Z02/30 Read block 93 len 40960
59612024-02-12T01:37:05.017Z03/30 Write block 479 len 40960 data: 7 8 6 6 7 7 7 7 8 8
59622024-02-12T01:37:05.017Z04/30 Read block 341 len 32768
59632024-02-12T01:37:05.017Z05/30 Read block 280 len 40960
59642024-02-12T01:37:05.017Z06/30 Read block 65 len 28672
59652024-02-12T01:37:05.017Z07/30 Read block 485 len 12288
59662024-02-12T01:37:05.017Z08/30 Flush
59672024-02-12T01:37:05.020Z09/30 Flush
59682024-02-12T01:37:05.020Z10/30 Write block 129 len 16384 data: 6 6 7 7
59692024-02-12T01:37:05.020Z11/30 Flush
59702024-02-12T01:37:05.024Z12/30 Flush
59712024-02-12T01:37:05.024Z13/30 Write block 151 len 20480 data: 6 6 4 4 3
59722024-02-12T01:37:05.024Z14/30 Read block 391 len 20480
59732024-02-12T01:37:05.029Z15/30 Read block 7 len 8192
59742024-02-12T01:37:05.029Z16/30 Write block 162 len 12288 data: 5 5 5
59752024-02-12T01:37:05.029Z17/30 Write block 238 len 4096 data: 7
59762024-02-12T01:37:05.029Z18/30 Read block 495 len 8192
59772024-02-12T01:37:05.029Z19/30 Write block 451 len 4096 data: 4
59782024-02-12T01:37:05.033Z20/30 Read block 295 len 20480
59792024-02-12T01:37:05.033Z21/30 Read block 498 len 20480
59802024-02-12T01:37:05.033Z22/30 Write block 304 len 24576 data: 5 5 6 7 8 6
59812024-02-12T01:37:05.033Z23/30 Read block 125 len 8192
59822024-02-12T01:37:05.033Z24/30 Read block 263 len 4096
59832024-02-12T01:37:05.033Z25/30 Read block 492 len 36864
59842024-02-12T01:37:05.033Z26/30 Read block 522 len 20480
59852024-02-12T01:37:05.036Z27/30 Read block 1 len 36864
59862024-02-12T01:37:05.036Z28/30 Write block 306 len 12288 data: 7 8 9
59872024-02-12T01:37:05.036Z29/30 Read block 348 len 32768
59882024-02-12T01:37:05.036Z30/30 Read block 586 len 8192
59892024-02-12T01:37:05.039Z----------------------------------------------------------------
59902024-02-12T01:37:05.040Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59912024-02-12T01:37:05.040ZStates: Active Active Active
59922024-02-12T01:37:05.040ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59932024-02-12T01:37:05.040Z 1 Acked 1000 Write 3 Done Done Done false
59942024-02-12T01:37:05.040Z 2 Acked 1001 Read 10 Done Done Done false
59952024-02-12T01:37:05.040Z 3 Acked 1002 Write 10 Done Done Done false
59962024-02-12T01:37:05.040Z 4 Acked 1003 Read 8 Done Done Done false
59972024-02-12T01:37:05.040Z 5 Acked 1004 Read 10 Done Done Done false
59982024-02-12T01:37:05.040Z 6 Acked 1005 Read 7 Sent Done Done false
59992024-02-12T01:37:05.040Z 7 Acked 1006 Read 3 Sent Done Done false
60002024-02-12T01:37:05.040Z 8 Acked 1007 Flush 0 Sent Done Done false
60012024-02-12T01:37:05.040Z 9 Acked 1008 Flush 0 Sent Done Done false
60022024-02-12T01:37:05.040Z 10 Acked 1009 Write 4 Sent Done Done false
60032024-02-12T01:37:05.040Z 11 Acked 1010 Flush 0 Sent Done Done false
60042024-02-12T01:37:05.040Z 12 Acked 1011 Flush 0 Sent Done Done false
60052024-02-12T01:37:05.040Z 13 Acked 1012 Write 5 Sent Done Done false
60062024-02-12T01:37:05.040Z 14 Acked 1013 Read 5 Sent Done Done false
60072024-02-12T01:37:05.040Z 15 Acked 1014 Read 2 Sent Done Done false
60082024-02-12T01:37:05.040Z 16 Acked 1015 Write 3 Sent Done Done false
60092024-02-12T01:37:05.040Z 17 Acked 1016 Write 1 Sent Done Done false
60102024-02-12T01:37:05.040Z 18 Acked 1017 Read 2 Sent Done Done false
60112024-02-12T01:37:05.040Z 19 Acked 1018 Write 1 Sent Done Done false
60122024-02-12T01:37:05.040Z 20 Acked 1019 Read 5 Sent Done Done false
60132024-02-12T01:37:05.040Z 21 Acked 1020 Read 5 Sent Done Done false
60142024-02-12T01:37:05.041Z 22 Acked 1021 Write 6 Sent Done Done false
60152024-02-12T01:37:05.041Z 23 Acked 1022 Read 2 Sent Done Done false
60162024-02-12T01:37:05.041Z 24 Acked 1023 Read 1 Sent Done Done false
60172024-02-12T01:37:05.041Z 25 Acked 1024 Read 9 Sent Done Done false
60182024-02-12T01:37:05.041Z 26 Acked 1025 Read 5 Sent Done Done false
60192024-02-12T01:37:05.041Z 27 Acked 1026 Read 9 Sent Done Done false
60202024-02-12T01:37:05.041Z 28 Acked 1027 Write 3 Sent Done Done false
60212024-02-12T01:37:05.041Z 29 Acked 1028 Read 8 Sent Done Done false
60222024-02-12T01:37:05.041Z 30 Acked 1029 Read 2 Sent Done Done false
60232024-02-12T01:37:05.041Z STATES DS:0 DS:1 DS:2 TOTAL
60242024-02-12T01:37:05.041Z New 0 0 0 0
60252024-02-12T01:37:05.041Z Sent 25 0 0 25
60262024-02-12T01:37:05.041Z Done 5 30 30 65
60272024-02-12T01:37:05.041Z Skipped 0 0 0 0
60282024-02-12T01:37:05.041Z Error 0 0 0 0
60292024-02-12T01:37:05.041ZLast Flush: 0 1011 1011
60302024-02-12T01:37:05.041ZDownstairs last five completed:
60312024-02-12T01:37:05.041ZUpstairs last five completed: 30 29 28 27 26
60322024-02-12T01:37:05.041Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:03.940265842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1351}
60332024-02-12T01:37:05.041ZWrote out file "/tmp/test_repair_verify.data"
60342024-02-12T01:37:05.044Z
60352024-02-12T01:37:05.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
60362024-02-12T01:37:05.064ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60372024-02-12T01:37:05.064Z 0 000-019 52 52 52 88 88 88 F F F
60382024-02-12T01:37:05.064Z 1 020-039 52 52 52 92 92 92 F F F
60392024-02-12T01:37:05.064Z 2 040-059 52 52 52 92 92 92 F F F
60402024-02-12T01:37:05.064Z 3 060-079 41 41 41 62 62 62 F F F
60412024-02-12T01:37:05.064Z 4 080-099 52 52 52 86 86 86 F F F
60422024-02-12T01:37:05.065Z 5 100-119 52 52 52 85 85 85 F F F
60432024-02-12T01:37:05.065Z 6 120-139 50 54 54 82 96 96 F F F <---
60442024-02-12T01:37:05.065Z 7 140-159 52 52 52 91 91 91 F T T <---
60452024-02-12T01:37:05.065Z 8 160-179 52 52 52 89 89 89 F T T <---
60462024-02-12T01:37:05.068Z 9 180-199 50 50 50 80 80 80 F F F
60472024-02-12T01:37:05.068Z 10 200-219 52 52 52 89 89 89 F F F
60482024-02-12T01:37:05.068Z 11 220-239 52 52 52 89 89 89 F T T <---
60492024-02-12T01:37:05.068Z 12 240-259 50 50 50 80 80 80 F F F
60502024-02-12T01:37:05.068Z 13 260-279 52 52 52 92 92 92 F F F
60512024-02-12T01:37:05.068Z 14 280-299 50 50 50 80 80 80 F F F
60522024-02-12T01:37:05.068Z 15 300-319 53 53 53 93 93 93 F T T <---
60532024-02-12T01:37:05.069Z 16 320-339 52 52 52 84 84 84 F F F
60542024-02-12T01:37:05.069Z 17 340-359 52 52 52 85 85 85 F F F
60552024-02-12T01:37:05.069Z 18 360-379 48 48 48 76 76 76 F F F
60562024-02-12T01:37:05.069Z 19 380-399 46 46 46 71 71 71 F F F
60572024-02-12T01:37:05.069Z 20 400-419 42 42 42 63 63 63 F F F
60582024-02-12T01:37:05.069Z 21 420-439 46 46 46 71 71 71 F F F
60592024-02-12T01:37:05.069Z 22 440-459 52 52 52 90 90 90 F T T <---
60602024-02-12T01:37:05.069Z 23 460-479 52 54 54 90 94 94 T F F <---
60612024-02-12T01:37:05.069Z 24 480-499 53 54 54 93 94 94 T F F <---
60622024-02-12T01:37:05.069Z 25 500-519 53 53 53 93 93 93 F F F
60632024-02-12T01:37:05.069Z 26 520-539 52 52 52 87 87 87 F F F
60642024-02-12T01:37:05.069Z 27 540-559 51 51 51 83 83 83 F F F
60652024-02-12T01:37:05.069Z 28 560-579 52 52 52 92 92 92 F F F
60662024-02-12T01:37:05.069Z 29 580-599 52 54 54 92 94 94 T F F <---
60672024-02-12T01:37:05.069ZMax gen: 54, Max flush: 96
60682024-02-12T01:37:05.069ZError: Difference in extent metadata found!
60692024-02-12T01:37:05.069ZA Difference in extent metadata is expected here
60702024-02-12T01:37:05.069ZOn loop 27
60712024-02-12T01:37:05.069Z
60722024-02-12T01:37:05.073ZVerifying data now
60732024-02-12T01:37:10.540ZNo change after verify
60742024-02-12T01:37:10.540ZLoop: 27 Downstairs dump after verify (and repair):
60752024-02-12T01:37:10.558ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60762024-02-12T01:37:10.558Z 0 000-019 52 52 52 88 88 88 F F F
60772024-02-12T01:37:10.558Z 1 020-039 52 52 52 92 92 92 F F F
60782024-02-12T01:37:10.558Z 2 040-059 52 52 52 92 92 92 F F F
60792024-02-12T01:37:10.558Z 3 060-079 41 41 41 62 62 62 F F F
60802024-02-12T01:37:10.558Z 4 080-099 52 52 52 86 86 86 F F F
60812024-02-12T01:37:10.558Z 5 100-119 52 52 52 85 85 85 F F F
60822024-02-12T01:37:10.559Z 6 120-139 54 54 54 96 96 96 F F F
60832024-02-12T01:37:10.559Z 7 140-159 55 55 55 97 97 97 F F F
60842024-02-12T01:37:10.559Z 8 160-179 55 55 55 97 97 97 F F F
60852024-02-12T01:37:10.559Z 9 180-199 50 50 50 80 80 80 F F F
60862024-02-12T01:37:10.559Z 10 200-219 52 52 52 89 89 89 F F F
60872024-02-12T01:37:10.559Z 11 220-239 55 55 55 97 97 97 F F F
60882024-02-12T01:37:10.559Z 12 240-259 50 50 50 80 80 80 F F F
60892024-02-12T01:37:10.559Z 13 260-279 52 52 52 92 92 92 F F F
60902024-02-12T01:37:10.559Z 14 280-299 50 50 50 80 80 80 F F F
60912024-02-12T01:37:10.559Z 15 300-319 55 55 55 97 97 97 F F F
60922024-02-12T01:37:10.559Z 16 320-339 52 52 52 84 84 84 F F F
60932024-02-12T01:37:10.559Z 17 340-359 52 52 52 85 85 85 F F F
60942024-02-12T01:37:10.559Z 18 360-379 48 48 48 76 76 76 F F F
60952024-02-12T01:37:10.562Z 19 380-399 46 46 46 71 71 71 F F F
60962024-02-12T01:37:10.562Z 20 400-419 42 42 42 63 63 63 F F F
60972024-02-12T01:37:10.562Z 21 420-439 46 46 46 71 71 71 F F F
60982024-02-12T01:37:10.562Z 22 440-459 55 55 55 97 97 97 F F F
60992024-02-12T01:37:10.562Z 23 460-479 54 54 54 94 94 94 F F F
61002024-02-12T01:37:10.562Z 24 480-499 54 54 54 94 94 94 F F F
61012024-02-12T01:37:10.562Z 25 500-519 53 53 53 93 93 93 F F F
61022024-02-12T01:37:10.562Z 26 520-539 52 52 52 87 87 87 F F F
61032024-02-12T01:37:10.562Z 27 540-559 51 51 51 83 83 83 F F F
61042024-02-12T01:37:10.562Z 28 560-579 52 52 52 92 92 92 F F F
61052024-02-12T01:37:10.562Z 29 580-599 54 54 54 94 94 94 F F F
61062024-02-12T01:37:10.562ZMax gen: 55, Max flush: 97
61072024-02-12T01:37:10.562Z
61082024-02-12T01:37:10.562ZBegin loop 28
61092024-02-12T01:37:10.563ZDownstairs to restart: 2
61102024-02-12T01:37:10.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.483979787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
61112024-02-12T01:37:10.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484256594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
61122024-02-12T01:37:10.582Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484286992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
61132024-02-12T01:37:10.582Z{"msg":"Crucible b3fd08fa-c0a6-4ab6-b430-4b83dbce5623 has session id: 2728664e-b528-4661-b6b5-dcc0bddad3db","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484315661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61142024-02-12T01:37:10.587Z{"msg":"Upstairs opts: Upstairs UUID: b3fd08fa-c0a6-4ab6-b430-4b83dbce5623, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible"Crucible runtime is spawned
61152024-02-12T01:37:10.587Z,"level":30,"time":"2024-02-12T01:37:09.484445485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61162024-02-12T01:37:10.587Z{"msg":"Crucible stats registered with UUID: b3fd08fa-c0a6-4ab6-b430-4b83dbce5623","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484500262Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61172024-02-12T01:37:10.587Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.4845436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
61182024-02-12T01:37:10.587Z{"msg":"b3fd08fa-c0a6-4ab6-b430-4b83dbce5623 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484580138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61192024-02-12T01:37:10.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484619787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61202024-02-12T01:37:10.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484652295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61212024-02-12T01:37:10.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484683713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61222024-02-12T01:37:10.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484731511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61232024-02-12T01:37:10.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.48476271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61242024-02-12T01:37:10.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484790348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61252024-02-12T01:37:10.587Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484831017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61262024-02-12T01:37:10.587Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484877034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61272024-02-12T01:37:10.587Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.484920932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61282024-02-12T01:37:10.587Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:09.484958611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61292024-02-12T01:37:10.587Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:09.485004838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61302024-02-12T01:37:10.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485036317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61312024-02-12T01:37:10.587Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:09.485067235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61322024-02-12T01:37:10.587Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485150571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61332024-02-12T01:37:10.587Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485186429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61342024-02-12T01:37:10.588Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485214798Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61352024-02-12T01:37:10.588Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485252077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61362024-02-12T01:37:10.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485561162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61372024-02-12T01:37:10.588Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.48560317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61382024-02-12T01:37:10.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485621259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61392024-02-12T01:37:10.588Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.485661817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61402024-02-12T01:37:10.588Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486191942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61412024-02-12T01:37:10.588Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.4862278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61422024-02-12T01:37:10.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486692818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61432024-02-12T01:37:10.588Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486736856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61442024-02-12T01:37:10.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486756745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61452024-02-12T01:37:10.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486786503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61462024-02-12T01:37:10.588Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486807292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61472024-02-12T01:37:10.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:09.486835751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61482024-02-12T01:37:11.585Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.486804253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61492024-02-12T01:37:11.585Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.487018543Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61502024-02-12T01:37:11.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.487940409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61512024-02-12T01:37:11.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.487997926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61522024-02-12T01:37:11.590Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.488487223Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61532024-02-12T01:37:11.590Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.48897959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61542024-02-12T01:37:11.590Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489024957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61552024-02-12T01:37:11.590Z{"msg":"[0]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 96, 97, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489051196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61562024-02-12T01:37:11.590Z{"msg":"[0]R generation[0..12]: [52, 52, 52, 41, 52, 52, 54, 55, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489085905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61572024-02-12T01:37:11.590Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489122713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61582024-02-12T01:37:11.590Z{"msg":"[1]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 96, 97, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489157471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61592024-02-12T01:37:11.590Z{"msg":"[1]R generation[0..12]: [52, 52, 52, 41, 52, 52, 54, 55, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.48918234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61602024-02-12T01:37:11.590Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489217578Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
61612024-02-12T01:37:11.590Z,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61622024-02-12T01:37:11.590Z{"msg":"[2]R flush_numbers[0..12]: [88, 92, 92, 62, 86, 85, 96, 97, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489260096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61632024-02-12T01:37:11.590Z{"msg":"[2]R generation[0..12]: [52, 52, 52, 41, 52, 52, 54, 55, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489299014Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61642024-02-12T01:37:11.590Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489338062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61652024-02-12T01:37:11.590Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489373311Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61662024-02-12T01:37:11.590Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489412429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61672024-02-12T01:37:11.591Z{"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489447627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61682024-02-12T01:37:11.591Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489483335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61692024-02-12T01:37:11.591Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489521914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61702024-02-12T01:37:11.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489546443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61712024-02-12T01:37:11.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489580921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61722024-02-12T01:37:11.591Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489620469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61732024-02-12T01:37:11.591Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489658197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61742024-02-12T01:37:11.591Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61752024-02-12T01:37:11.591Z,"time":"2024-02-12T01:37:10.489682836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61762024-02-12T01:37:11.591Z{"msg":"b3fd08fa-c0a6-4ab6-b430-4b83dbce5623 is now active with session: 2728664e-b528-4661-b6b5-dcc0bddad3db","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489715695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61772024-02-12T01:37:11.591Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489740473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359,"session_id":"2728664e-b528-4661-b6b5-dcc0bddad3db"}
61782024-02-12T01:37:11.591Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:10.489754503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
61792024-02-12T01:37:11.591ZLoading write count information from file "/tmp/test_repair_verify.data"
61802024-02-12T01:37:11.591ZRun Repair workload
61812024-02-12T01:37:11.591Z01/30 Write block 421 len 16384 data: 5 4 4 4
61822024-02-12T01:37:11.595Z02/30 Read block 36 len 40960
61832024-02-12T01:37:11.596Z03/30 Write block 270 len 20480 data: 3 4 3 4 5
61842024-02-12T01:37:11.596Z04/30 Write block 456 len 36864 data: 6 6 7 6 5 4 4 4 4
61852024-02-12T01:37:11.596Z05/30 Read block 538 len 4096
61862024-02-12T01:37:11.596Z06/30 Write block 282 len 36864 data: 7 7 7 7 6 6 6 5 5
61872024-02-12T01:37:11.599Z07/30 Write block 55 len 24576 data: 3 3 4 8 6 6
61882024-02-12T01:37:11.599Z08/30 Write block 65 len 28672 data: 6 4 3 3 4 3 5
61892024-02-12T01:37:11.599Z09/30 Write block 109 len 24576 data: 4 4 4 4 6 8
61902024-02-12T01:37:11.599Z10/30 Write block 141 len 12288 data: 6 7 7
61912024-02-12T01:37:11.599Z11/30 Write block 260 len 4096 data: 5
61922024-02-12T01:37:11.599Z12/30 Read block 446 len 28672
61932024-02-12T01:37:11.604Z13/30 Read block 513 len 28672
61942024-02-12T01:37:11.604Z14/30 Write block 277 len 24576 data: 4 6 7 7 6 8
61952024-02-12T01:37:11.604Z15/30 Read block 469 len 24576
61962024-02-12T01:37:11.604Z16/30 Read block 24 len 32768
61972024-02-12T01:37:11.604Z17/30 Read block 129 len 28672
61982024-02-12T01:37:11.609Z18/30 Read block 100 len 28672
61992024-02-12T01:37:11.610Z19/30 Read block 33 len 4096
62002024-02-12T01:37:11.610Z20/30 Read block 79 len 16384
62012024-02-12T01:37:11.610Z21/30 Read block 77 len 16384
62022024-02-12T01:37:11.610Z22/30 Write block 440 len 20480 data: 4 4 3 3 3
62032024-02-12T01:37:11.610Z23/30 Flush
62042024-02-12T01:37:11.614Z24/30 Write block 67 len 20480 data: 4 4 5 4 6
62052024-02-12T01:37:11.614Z25/30 Write block 515 len 40960 data: 5 4 4 4 4 5 5 6 6 6
62062024-02-12T01:37:11.614Z26/30 Flush
62072024-02-12T01:37:11.617Z27/30 Write block 275 len 12288 data: 6 6 5
62082024-02-12T01:37:11.617Z28/30 Write block 392 len 36864 data: 6 5 4 4 3 3 3 3 3
62092024-02-12T01:37:11.620Z29/30 Read block 530 len 28672
62102024-02-12T01:37:11.620Z30/30 Read block 98 len 4096
62112024-02-12T01:37:11.620Z----------------------------------------------------------------
62122024-02-12T01:37:11.620Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
62132024-02-12T01:37:11.621ZStates: Active Active Active
62142024-02-12T01:37:11.621ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
62152024-02-12T01:37:11.621Z 1 Acked 1000 Write 4 Done Done Done false
62162024-02-12T01:37:11.621Z 2 Acked 1001 Read 10 Done Done Done false
62172024-02-12T01:37:11.621Z 3 Acked 1002 Write 5 Done Done Sent false
62182024-02-12T01:37:11.621Z 4 Acked 1003 Write 9 Done Done Sent false
62192024-02-12T01:37:11.621Z 5 Acked 1004 Read 1 Done Done Sent false
62202024-02-12T01:37:11.621Z 6 Acked 1005 Write 9 Done Done Sent false
62212024-02-12T01:37:11.621Z 7 Acked 1006 Write 6 Done Done Sent false
62222024-02-12T01:37:11.621Z 8 Acked 1007 Write 7 Done Done Sent false
62232024-02-12T01:37:11.621Z 9 Acked 1008 Write 6 Done Done Sent false
62242024-02-12T01:37:11.621Z 10 Acked 1009 Write 3 Done Done Sent false
62252024-02-12T01:37:11.621Z 11 Acked 1010 Write 1 Done Done Sent false
62262024-02-12T01:37:11.621Z 12 Acked 1011 Read 7 Done Done Sent false
62272024-02-12T01:37:11.621Z 13 Acked 1012 Read 7 Done Done Sent false
62282024-02-12T01:37:11.621Z 14 Acked 1013 Write 6 Done Done Sent false
62292024-02-12T01:37:11.621Z 15 Acked 1014 Read 6 Done Done Sent false
62302024-02-12T01:37:11.621Z 16 Acked 1015 Read 8 Done Done Sent false
62312024-02-12T01:37:11.621Z 17 Acked 1016 Read 7 Done Done Sent false
62322024-02-12T01:37:11.621Z 18 Acked 1017 Read 7 Done Done Sent false
62332024-02-12T01:37:11.621Z 19 Acked 1018 Read 1 Done Done Sent false
62342024-02-12T01:37:11.621Z 20 Acked 1019 Read 4 Done Done Sent false
62352024-02-12T01:37:11.621Z 21 Acked 1020 Read 4 Done Done Sent false
62362024-02-12T01:37:11.622Z 22 Acked 1021 Write 5 Done Done Sent false
62372024-02-12T01:37:11.622Z 23 Acked 1022 Flush 0 Done Done Sent false
62382024-02-12T01:37:11.622Z 24 Acked 1023 Write 5 Done Done Sent false
62392024-02-12T01:37:11.622Z 25 Acked 1024 Write 10 Done Done Sent false
62402024-02-12T01:37:11.622Z 26 Acked 1025 Flush 0 Done Done Sent false
62412024-02-12T01:37:11.622Z 27 Acked 1026 Write 3 Done Done Sent false
62422024-02-12T01:37:11.622Z 28 Acked 1027 Write 9 Done Done Sent false
62432024-02-12T01:37:11.622Z 29 Acked 1028 Read 7 Done Done Sent false
62442024-02-12T01:37:11.622Z 30 Acked 1029 Read 1 Done Sent Sent false
62452024-02-12T01:37:11.622Z STATES DS:0 DS:1 DS:2 TOTAL
62462024-02-12T01:37:11.622Z New 0 0 0 0
62472024-02-12T01:37:11.622Z Sent 0 1 28 29
62482024-02-12T01:37:11.622Z Done 30 29 2 61
62492024-02-12T01:37:11.622Z Skipped 0 0 0 0
62502024-02-12T01:37:11.622Z Error 0 0 0 0
62512024-02-12T01:37:11.622ZLast Flush: 1025 1025 0
62522024-02-12T01:37:11.622ZDownstairs last five completed:
62532024-02-12T01:37:11.622ZUpstairs last five completed: 30 29 28 27 26
62542024-02-12T01:37:11.622Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:10.521934557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1359}
62552024-02-12T01:37:11.622ZWrote out file "/tmp/test_repair_verify.data"
62562024-02-12T01:37:11.626Z
62572024-02-12T01:37:11.629ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62582024-02-12T01:37:11.645ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62592024-02-12T01:37:11.645Z 0 000-019 52 52 52 88 88 88 F F F
62602024-02-12T01:37:11.645Z 1 020-039 52 52 52 92 92 92 F F F
62612024-02-12T01:37:11.645Z 2 040-059 56 56 52 98 98 92 F F F <---
62622024-02-12T01:37:11.645Z 3 060-079 56 56 41 99 99 62 F F F <---
62632024-02-12T01:37:11.645Z 4 080-099 52 52 52 86 86 86 F F F
62642024-02-12T01:37:11.645Z 5 100-119 56 56 52 98 98 85 F F F <---
62652024-02-12T01:37:11.645Z 6 120-139 54 54 54 96 96 96 F F F
62662024-02-12T01:37:11.645Z 7 140-159 56 56 55 98 98 97 F F F <---
62672024-02-12T01:37:11.645Z 8 160-179 55 55 55 97 97 97 F F F
62682024-02-12T01:37:11.645Z 9 180-199 50 50 50 80 80 80 F F F
62692024-02-12T01:37:11.645Z 10 200-219 52 52 52 89 89 89 F F F
62702024-02-12T01:37:11.645Z 11 220-239 55 55 55 97 97 97 F F F
62712024-02-12T01:37:11.645Z 12 240-259 50 50 50 80 80 80 F F F
62722024-02-12T01:37:11.645Z 13 260-279 56 56 52 98 98 92 T T F <---
62732024-02-12T01:37:11.645Z 14 280-299 56 56 50 98 98 80 F F F <---
62742024-02-12T01:37:11.646Z 15 300-319 55 55 55 97 97 97 F F F
62752024-02-12T01:37:11.646Z 16 320-339 52 52 52 84 84 84 F F F
62762024-02-12T01:37:11.649Z 17 340-359 52 52 52 85 85 85 F F F
62772024-02-12T01:37:11.649Z 18 360-379 48 48 48 76 76 76 F F F
62782024-02-12T01:37:11.649Z 19 380-399 46 46 46 71 71 71 T T F <---
62792024-02-12T01:37:11.649Z 20 400-419 42 42 42 63 63 63 T T F <---
62802024-02-12T01:37:11.649Z 21 420-439 56 56 46 98 98 71 F F T <---
62812024-02-12T01:37:11.649Z 22 440-459 56 56 55 98 98 97 F F F <---
62822024-02-12T01:37:11.649Z 23 460-479 56 56 54 98 98 94 F F F <---
62832024-02-12T01:37:11.649Z 24 480-499 54 54 54 94 94 94 F F F
62842024-02-12T01:37:11.649Z 25 500-519 56 56 53 99 99 93 F F F <---
62852024-02-12T01:37:11.649Z 26 520-539 56 56 52 99 99 87 F F F <---
62862024-02-12T01:37:11.649Z 27 540-559 51 51 51 83 83 83 F F F
62872024-02-12T01:37:11.649Z 28 560-579 52 52 52 92 92 92 F F F
62882024-02-12T01:37:11.649Z 29 580-599 54 54 54 94 94 94 F F F
62892024-02-12T01:37:11.649ZMax gen: 56, Max flush: 99
62902024-02-12T01:37:11.649ZError: Difference in extent metadata found!
62912024-02-12T01:37:11.649ZA Difference in extent metadata is expected here
62922024-02-12T01:37:11.649ZOn loop 28
62932024-02-12T01:37:11.649Z
62942024-02-12T01:37:11.655ZVerifying data now
62952024-02-12T01:37:16.284ZNo change after verify
62962024-02-12T01:37:16.284ZLoop: 28 Downstairs dump after verify (and repair):
62972024-02-12T01:37:16.300ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62982024-02-12T01:37:16.300Z 0 000-019 52 52 52 88 88 88 F F F
62992024-02-12T01:37:16.300Z 1 020-039 52 52 52 92 92 92 F F F
63002024-02-12T01:37:16.300Z 2 040-059 56 56 56 98 98 98 F F F
63012024-02-12T01:37:16.300Z 3 060-079 56 56 56 99 99 99 F F F
63022024-02-12T01:37:16.300Z 4 080-099 52 52 52 86 86 86 F F F
63032024-02-12T01:37:16.300Z 5 100-119 56 56 56 98 98 98 F F F
63042024-02-12T01:37:16.300Z 6 120-139 54 54 54 96 96 96 F F F
63052024-02-12T01:37:16.300Z 7 140-159 56 56 56 98 98 98 F F F
63062024-02-12T01:37:16.300Z 8 160-179 55 55 55 97 97 97 F F F
63072024-02-12T01:37:16.300Z 9 180-199 50 50 50 80 80 80 F F F
63082024-02-12T01:37:16.300Z 10 200-219 52 52 52 89 89 89 F F F
63092024-02-12T01:37:16.300Z 11 220-239 55 55 55 97 97 97 F F F
63102024-02-12T01:37:16.300Z 12 240-259 50 50 50 80 80 80 F F F
63112024-02-12T01:37:16.300Z 13 260-279 57 57 57 100 100 100 F F F
63122024-02-12T01:37:16.301Z 14 280-299 56 56 56 98 98 98 F F F
63132024-02-12T01:37:16.301Z 15 300-319 55 55 55 97 97 97 F F F
63142024-02-12T01:37:16.301Z 16 320-339 52 52 52 84 84 84 F F F
63152024-02-12T01:37:16.301Z 17 340-359 52 52 52 85 85 85 F F F
63162024-02-12T01:37:16.301Z 18 360-379 48 48 48 76 76 76 F F F
63172024-02-12T01:37:16.301Z 19 380-399 57 57 57 100 100 100 F F F
63182024-02-12T01:37:16.301Z 20 400-419 57 57 57 100 100 100 F F F
63192024-02-12T01:37:16.301Z 21 420-439 56 56 56 98 98 98 F F F
63202024-02-12T01:37:16.301Z 22 440-459 56 56 56 98 98 98 F F F
63212024-02-12T01:37:16.304Z 23 460-479 56 56 56 98 98 98 F F F
63222024-02-12T01:37:16.304Z 24 480-499 54 54 54 94 94 94 F F F
63232024-02-12T01:37:16.304Z 25 500-519 56 56 56 99 99 99 F F F
63242024-02-12T01:37:16.304Z 26 520-539 56 56 56 99 99 99 F F F
63252024-02-12T01:37:16.304Z 27 540-559 51 51 51 83 83 83 F F F
63262024-02-12T01:37:16.304Z 28 560-579 52 52 52 92 92 92 F F F
63272024-02-12T01:37:16.305Z 29 580-599 54 54 54 94 94 94 F F F
63282024-02-12T01:37:16.305ZMax gen: 57, Max flush: 100
63292024-02-12T01:37:16.305Z
63302024-02-12T01:37:16.305ZBegin loop 29
63312024-02-12T01:37:16.305ZDownstairs to restart: 1
63322024-02-12T01:37:16.325Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227192214Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
63332024-02-12T01:37:16.326Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227454362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
63342024-02-12T01:37:16.326Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.22748058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
63352024-02-12T01:37:16.326Z{"msg":"Crucible 9007aa36-410e-45d8-b8f1-465f1e3aa892 has session id: 89eb9657-1982-4f13-9cf7-c04e00060623","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227506259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63362024-02-12T01:37:16.334Z{"msg":"Upstairs opts: Upstairs UUID: 9007aa36-410e-45d8-b8f1-465f1e3aa892, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
63372024-02-12T01:37:16.335Z,"time":"2024-02-12T01:37:15.227550967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63382024-02-12T01:37:16.335Z{"msg":"Crucible stats registered with UUID: 9007aa36-410e-45d8-b8f1-465f1e3aa892","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227592935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63392024-02-12T01:37:16.335Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227643973Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
63402024-02-12T01:37:16.335Z{"msg":"9007aa36-410e-45d8-b8f1-465f1e3aa892 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227753097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63412024-02-12T01:37:16.335Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227793995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63422024-02-12T01:37:16.335Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227850623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63432024-02-12T01:37:16.335Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227923629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63442024-02-12T01:37:16.335Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227951078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63452024-02-12T01:37:16.335Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.227976337Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63462024-02-12T01:37:16.335Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228013605Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63472024-02-12T01:37:16.335Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228050503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63482024-02-12T01:37:16.335Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228094021Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63492024-02-12T01:37:16.335Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228238354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63502024-02-12T01:37:16.335Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:15.228298211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63512024-02-12T01:37:16.335Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:15.22833224Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63522024-02-12T01:37:16.335Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228357788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63532024-02-12T01:37:16.335Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228383977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63542024-02-12T01:37:16.335Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:15.228419176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63552024-02-12T01:37:16.336Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228434425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63562024-02-12T01:37:16.336Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.228457244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63572024-02-12T01:37:16.336Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.22854685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63582024-02-12T01:37:16.336Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229007368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63592024-02-12T01:37:16.336Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229054015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63602024-02-12T01:37:16.336Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229073085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63612024-02-12T01:37:16.336Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229102373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63622024-02-12T01:37:16.336Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229634028Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63632024-02-12T01:37:16.336Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.229684965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63642024-02-12T01:37:16.336Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.230186821Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63652024-02-12T01:37:16.336Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.23022289Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63662024-02-12T01:37:16.336Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.230238669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63672024-02-12T01:37:16.336Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.230251348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63682024-02-12T01:37:16.336Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.230272067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63692024-02-12T01:37:16.336Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:15.230286076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63702024-02-12T01:37:17.329Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.229983962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63712024-02-12T01:37:17.329Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.230111286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63722024-02-12T01:37:17.333Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.231028302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63732024-02-12T01:37:17.333Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.23108128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63742024-02-12T01:37:17.333Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.231548937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63752024-02-12T01:37:17.333Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232140019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63762024-02-12T01:37:17.333Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232173277Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63772024-02-12T01:37:17.333Z{"msg":"[0]R flush_numbers[0..12]: [88, 92, 98, 99, 86, 98, 96, 98, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232189687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63782024-02-12T01:37:17.333Z{"msg":"[0]R generation[0..12]: [52, 52, 56, 56, 52, 56, 54, 56, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232205356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63792024-02-12T01:37:17.333Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232227685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63802024-02-12T01:37:17.333Z{"msg":"[1]R flush_numbers[0..12]: [88, 92, 98, 99, 86, 98, 96, 98, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232242484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63812024-02-12T01:37:17.333Z{"msg":"[1]R generation[0..12]: [52, 52, 56, 56, 52, 56, 54, 56, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232256333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63822024-02-12T01:37:17.333Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232277923Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63832024-02-12T01:37:17.333Z{"msg":"[2]R flush_numbers[0..12]: [88, 92, 98, 99, 86, 98, 96, 98, 97, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232292262Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63842024-02-12T01:37:17.334Z{"msg":"[2]R generation[0..12]: [52, 52, 56, 56, 52, 56, 54, 56, 55, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.23233066Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63852024-02-12T01:37:17.334Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232345199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63862024-02-12T01:37:17.334Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232358879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63872024-02-12T01:37:17.334Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232374438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63882024-02-12T01:37:17.334Z{"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
63892024-02-12T01:37:17.334Ztime":"2024-02-12T01:37:16.232387657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63902024-02-12T01:37:17.334Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232405517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63912024-02-12T01:37:17.334Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232419206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63922024-02-12T01:37:17.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232431995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63932024-02-12T01:37:17.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232446175Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63942024-02-12T01:37:17.334Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232460554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63952024-02-12T01:37:17.334Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232482003Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63962024-02-12T01:37:17.334Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232495242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63972024-02-12T01:37:17.334Z{"msg":"9007aa36-410e-45d8-b8f1-465f1e3aa892 is now active with session: 89eb9657-1982-4f13-9cf7-c04e00060623","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232509241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63982024-02-12T01:37:17.334Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.232521811Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367,"session_id":"89eb9657-1982-4f13-9cf7-c04e00060623"}
63992024-02-12T01:37:17.334Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:16.23254086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
64002024-02-12T01:37:17.334ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64012024-02-12T01:37:17.334ZLoading write count information from file "/tmp/test_repair_verify.data"
64022024-02-12T01:37:17.334ZRun Repair workload
64032024-02-12T01:37:17.337Z01/30 Write block 62 len 40960 data: 7 6 6 7 5 5 5 6 5 7
64042024-02-12T01:37:17.338Z02/30 Read block 580 len 20480
64052024-02-12T01:37:17.338Z03/30 Write block 579 len 16384 data: 5 5 6 5
64062024-02-12T01:37:17.338Z04/30 Read block 351 len 40960
64072024-02-12T01:37:17.338Z05/30 Read block 488 len 16384
64082024-02-12T01:37:17.338Z06/30 Read block 434 len 36864
64092024-02-12T01:37:17.341Z07/30 Read block 113 len 32768
64102024-02-12T01:37:17.341Z08/30 Read block 579 len 36864
64112024-02-12T01:37:17.341Z09/30 Read block 336 len 20480
64122024-02-12T01:37:17.344Z10/30 Read block 143 len 8192
64132024-02-12T01:37:17.344Z11/30 Read block 511 len 40960
64142024-02-12T01:37:17.349Z12/30 Write block 43 len 36864 data: 8 8 5 4 6 5 5 5 5
64152024-02-12T01:37:17.349Z13/30 Write block 175 len 12288 data: 6 7 7
64162024-02-12T01:37:17.349Z14/30 Read block 103 len 28672
64172024-02-12T01:37:17.349Z15/30 Read block 0 len 20480
64182024-02-12T01:37:17.352Z16/30 Read block 37 len 12288
64192024-02-12T01:37:17.352Z17/30 Read block 188 len 4096
64202024-02-12T01:37:17.352Z18/30 Write block 531 len 28672 data: 5 7 7 6 9 8 8
64212024-02-12T01:37:17.353Z19/30 Write block 330 len 16384 data: 3 3 3 3
64222024-02-12T01:37:17.353Z20/30 Read block 316 len 16384
64232024-02-12T01:37:17.353Z21/30 Read block 275 len 8192
64242024-02-12T01:37:17.353Z22/30 Read block 140 len 20480
64252024-02-12T01:37:17.353Z23/30 Read block 37 len 4096
64262024-02-12T01:37:17.356Z24/30 Read block 504 len 28672
64272024-02-12T01:37:17.356Z25/30 Flush
64282024-02-12T01:37:17.358Z26/30 Write block 22 len 20480 data: 3 3 4 4 4
64292024-02-12T01:37:17.358Z27/30 Read block 584 len 4096
64302024-02-12T01:37:17.359Z28/30 Read block 496 len 16384
64312024-02-12T01:37:17.362Z29/30 Write block 242 len 40960 data: 4 5 3 3 4 5 5 5 4 4
64322024-02-12T01:37:17.362Z30/30 Write block 119 len 8192 data: 10 9
64332024-02-12T01:37:17.362Z----------------------------------------------------------------
64342024-02-12T01:37:17.362Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
64352024-02-12T01:37:17.362ZStates: Active Active Active
64362024-02-12T01:37:17.362ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
64372024-02-12T01:37:17.362Z 1 Acked 1000 Write 10 Done Done Done false
64382024-02-12T01:37:17.362Z 2 Acked 1001 Read 5 Done Done Done false
64392024-02-12T01:37:17.362Z 3 Acked 1002 Write 4 Done Done Done false
64402024-02-12T01:37:17.362Z 4 Acked 1003 Read 10 Done Done Done false
64412024-02-12T01:37:17.362Z 5 Acked 1004 Read 4 Done Done Done false
64422024-02-12T01:37:17.362Z 6 Acked 1005 Read 9 Done Done Done false
64432024-02-12T01:37:17.362Z 7 Acked 1006 Read 8 Done Sent Done false
64442024-02-12T01:37:17.363Z 8 Acked 1007 Read 9 Done Sent Done false
64452024-02-12T01:37:17.363Z 9 Acked 1008 Read 5 Done Sent Done false
64462024-02-12T01:37:17.363Z 10 Acked 1009 Read 2 Done Sent Done false
64472024-02-12T01:37:17.363Z 11 Acked 1010 Read 10 Done Sent Done false
64482024-02-12T01:37:17.363Z 12 Acked 1011 Write 9 Done Sent Done false
64492024-02-12T01:37:17.363Z 13 Acked 1012 Write 3 Done Sent Done false
64502024-02-12T01:37:17.363Z 14 Acked 1013 Read 7 Done Sent Done false
64512024-02-12T01:37:17.363Z 15 Acked 1014 Read 5 Done Sent Done false
64522024-02-12T01:37:17.363Z 16 Acked 1015 Read 3 Done Sent Done false
64532024-02-12T01:37:17.363Z 17 Acked 1016 Read 1 Done Sent Done false
64542024-02-12T01:37:17.363Z 18 Acked 1017 Write 7 Done Sent Done false
64552024-02-12T01:37:17.363Z 19 Acked 1018 Write 4 Done Sent Done false
64562024-02-12T01:37:17.363Z 20 Acked 1019 Read 4 Done Sent Done false
64572024-02-12T01:37:17.363Z 21 Acked 1020 Read 2 Done Sent Done false
64582024-02-12T01:37:17.363Z 22 Acked 1021 Read 5 Done Sent Done false
64592024-02-12T01:37:17.363Z 23 Acked 1022 Read 1 Done Sent Done false
64602024-02-12T01:37:17.363Z 24 Acked 1023 Read 7 Done Sent Done false
64612024-02-12T01:37:17.363Z 25 Acked 1024 Flush 0 Done Sent Done false
64622024-02-12T01:37:17.363Z 26 Acked 1025 Write 5 Done Sent Done false
64632024-02-12T01:37:17.363Z 27 Acked 1026 Read 1 Done Sent Done false
64642024-02-12T01:37:17.363Z 28 Acked 1027 Read 4 Done Sent Done false
64652024-02-12T01:37:17.363Z 29 Acked 1028 Write 10 Sent Sent Sent false
64662024-02-12T01:37:17.363Z 30 Acked 1029 Write 2 Sent Sent Sent false
64672024-02-12T01:37:17.363Z STATES DS:0 DS:1 DS:2 TOTAL
64682024-02-12T01:37:17.363Z New 0 0 0 0
64692024-02-12T01:37:17.364Z Sent 2 24 2 28
64702024-02-12T01:37:17.364Z Done 28 6 28 62
64712024-02-12T01:37:17.364Z Skipped 0 0 0 0
64722024-02-12T01:37:17.364Z Error 0 0 0 0
64732024-02-12T01:37:17.364ZLast Flush: 1024 0 1024
64742024-02-12T01:37:17.364ZDownstairs last five completed:
64752024-02-12T01:37:17.364ZUpstairs last five completed: 30 29 28 27 26
64762024-02-12T01:37:17.364Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:16.262990847Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1367}
64772024-02-12T01:37:17.364ZWrote out file "/tmp/test_repair_verify.data"
64782024-02-12T01:37:17.367Z
64792024-02-12T01:37:17.370ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64802024-02-12T01:37:17.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64812024-02-12T01:37:17.390Z 0 000-019 52 52 52 88 88 88 F F F
64822024-02-12T01:37:17.390Z 1 020-039 52 52 52 92 92 92 T F T <---
64832024-02-12T01:37:17.390Z 2 040-059 58 56 58 101 98 101 F F F <---
64842024-02-12T01:37:17.390Z 3 060-079 58 56 58 101 99 101 F T F <---
64852024-02-12T01:37:17.390Z 4 080-099 52 52 52 86 86 86 F F F
64862024-02-12T01:37:17.390Z 5 100-119 56 56 56 98 98 98 T F T <---
64872024-02-12T01:37:17.390Z 6 120-139 54 54 54 96 96 96 T F T <---
64882024-02-12T01:37:17.390Z 7 140-159 56 56 56 98 98 98 F F F
64892024-02-12T01:37:17.390Z 8 160-179 58 55 58 101 97 101 F F F <---
64902024-02-12T01:37:17.393Z 9 180-199 50 50 50 80 80 80 F F F
64912024-02-12T01:37:17.393Z 10 200-219 52 52 52 89 89 89 F F F
64922024-02-12T01:37:17.393Z 11 220-239 55 55 55 97 97 97 F F F
64932024-02-12T01:37:17.393Z 12 240-259 50 50 50 80 80 80 T F T <---
64942024-02-12T01:37:17.393Z 13 260-279 57 57 57 100 100 100 F F F
64952024-02-12T01:37:17.394Z 14 280-299 56 56 56 98 98 98 F F F
64962024-02-12T01:37:17.394Z 15 300-319 55 55 55 97 97 97 F F F
64972024-02-12T01:37:17.394Z 16 320-339 58 52 58 101 84 101 F F F <---
64982024-02-12T01:37:17.394Z 17 340-359 52 52 52 85 85 85 F F F
64992024-02-12T01:37:17.394Z 18 360-379 48 48 48 76 76 76 F F F
65002024-02-12T01:37:17.394Z 19 380-399 57 57 57 100 100 100 F F F
65012024-02-12T01:37:17.394Z 20 400-419 57 57 57 100 100 100 F F F
65022024-02-12T01:37:17.394Z 21 420-439 56 56 56 98 98 98 F F F
65032024-02-12T01:37:17.394Z 22 440-459 56 56 56 98 98 98 F F F
65042024-02-12T01:37:17.394Z 23 460-479 56 56 56 98 98 98 F F F
65052024-02-12T01:37:17.394Z 24 480-499 54 54 54 94 94 94 F F F
65062024-02-12T01:37:17.394Z 25 500-519 56 56 56 99 99 99 F F F
65072024-02-12T01:37:17.394Z 26 520-539 58 56 58 101 99 101 F F F <---
65082024-02-12T01:37:17.394Z 27 540-559 51 51 51 83 83 83 F F F
65092024-02-12T01:37:17.394Z 28 560-579 58 52 58 101 92 101 F T F <---
65102024-02-12T01:37:17.394Z 29 580-599 58 54 58 101 94 101 F T F <---
65112024-02-12T01:37:17.394ZMax gen: 58, Max flush: 101
65122024-02-12T01:37:17.394ZError: Difference in extent metadata found!
65132024-02-12T01:37:17.394ZA Difference in extent metadata is expected here
65142024-02-12T01:37:17.394ZOn loop 29
65152024-02-12T01:37:17.394Z
65162024-02-12T01:37:17.397ZVerifying data now
65172024-02-12T01:37:22.965ZNo change after verify
65182024-02-12T01:37:22.965ZLoop: 29 Downstairs dump after verify (and repair):
65192024-02-12T01:37:22.980ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65202024-02-12T01:37:22.980Z 0 000-019 52 52 52 88 88 88 F F F
65212024-02-12T01:37:22.980Z 1 020-039 59 59 59 102 102 102 F F F
65222024-02-12T01:37:22.980Z 2 040-059 58 58 58 101 101 101 F F F
65232024-02-12T01:37:22.980Z 3 060-079 58 58 58 101 101 101 F F F
65242024-02-12T01:37:22.980Z 4 080-099 52 52 52 86 86 86 F F F
65252024-02-12T01:37:22.980Z 5 100-119 59 59 59 102 102 102 F F F
65262024-02-12T01:37:22.980Z 6 120-139 59 59 59 102 102 102 F F F
65272024-02-12T01:37:22.980Z 7 140-159 56 56 56 98 98 98 F F F
65282024-02-12T01:37:22.980Z 8 160-179 58 58 58 101 101 101 F F F
65292024-02-12T01:37:22.980Z 9 180-199 50 50 50 80 80 80 F F F
65302024-02-12T01:37:22.980Z 10 200-219 52 52 52 89 89 89 F F F
65312024-02-12T01:37:22.980Z 11 220-239 55 55 55 97 97 97 F F F
65322024-02-12T01:37:22.981Z 12 240-259 59 59 59 102 102 102 F F F
65332024-02-12T01:37:22.984Z 13 260-279 57 57 57 100 100 100 F F F
65342024-02-12T01:37:22.984Z 14 280-299 56 56 56 98 98 98 F F F
65352024-02-12T01:37:22.984Z 15 300-319 55 55 55 97 97 97 F F F
65362024-02-12T01:37:22.984Z 16 320-339 58 58 58 101 101 101 F F F
65372024-02-12T01:37:22.984Z 17 340-359 52 52 52 85 85 85 F F F
65382024-02-12T01:37:22.984Z 18 360-379 48 48 48 76 76 76 F F F
65392024-02-12T01:37:22.984Z 19 380-399 57 57 57 100 100 100 F F F
65402024-02-12T01:37:22.984Z 20 400-419 57 57 57 100 100 100 F F F
65412024-02-12T01:37:22.984Z 21 420-439 56 56 56 98 98 98 F F F
65422024-02-12T01:37:22.984Z 22 440-459 56 56 56 98 98 98 F F F
65432024-02-12T01:37:22.984Z 23 460-479 56 56 56 98 98 98 F F F
65442024-02-12T01:37:22.984Z 24 480-499 54 54 54 94 94 94 F F F
65452024-02-12T01:37:22.984Z 25 500-519 56 56 56 99 99 99 F F F
65462024-02-12T01:37:22.984Z 26 520-539 58 58 58 101 101 101 F F F
65472024-02-12T01:37:22.984Z 27 540-559 51 51 51 83 83 83 F F F
65482024-02-12T01:37:22.984Z 28 560-579 58 58 58 101 101 101 F F F
65492024-02-12T01:37:22.984Z 29 580-599 58 58 58 101 101 101 F F F
65502024-02-12T01:37:22.984ZMax gen: 59, Max flush: 102
65512024-02-12T01:37:22.984Z
65522024-02-12T01:37:22.984ZBegin loop 30
65532024-02-12T01:37:22.984ZDownstairs to restart: 2
65542024-02-12T01:37:23.002Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.903413608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
65552024-02-12T01:37:23.003Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.903776241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
65562024-02-12T01:37:23.003Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
65572024-02-12T01:37:23.003Z,"time":"2024-02-12T01:37:21.9038069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
65582024-02-12T01:37:23.007Z{"msg":"Crucible 1811a26e-3312-4ca0-b7ec-2d419d6e46de has session id: 5c9a61d7-4059-44fa-a9f1-fbba506d0895","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.903850098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65592024-02-12T01:37:23.007Z{"msg":"Upstairs opts: Upstairs UUID: 1811a26e-3312-4ca0-b7ec-2d419d6e46de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.903888766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65602024-02-12T01:37:23.007Z{"msg":"Crucible stats registered with UUID: 1811a26e-3312-4ca0-b7ec-2d419d6e46de","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.903914604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65612024-02-12T01:37:23.007Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904038958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
65622024-02-12T01:37:23.007Z{"msg":"1811a26e-3312-4ca0-b7ec-2d419d6e46de active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904082467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65632024-02-12T01:37:23.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904099636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65642024-02-12T01:37:23.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904115065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65652024-02-12T01:37:23.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904129414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65662024-02-12T01:37:23.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904145033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65672024-02-12T01:37:23.008Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904180242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65682024-02-12T01:37:23.008Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904196601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65692024-02-12T01:37:23.008Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.90421083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65702024-02-12T01:37:23.008Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904252948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65712024-02-12T01:37:23.008Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904318535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65722024-02-12T01:37:23.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904492807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65732024-02-12T01:37:23.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904566903Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65742024-02-12T01:37:23.008Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.904758674Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65752024-02-12T01:37:23.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.905315398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65762024-02-12T01:37:23.008Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.905372935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65772024-02-12T01:37:23.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.905425022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65782024-02-12T01:37:23.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.90547076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65792024-02-12T01:37:23.008Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.9058955Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65802024-02-12T01:37:23.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906046703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65812024-02-12T01:37:23.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906383227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65822024-02-12T01:37:23.008Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906426265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65832024-02-12T01:37:23.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906447054Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65842024-02-12T01:37:23.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906549499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65852024-02-12T01:37:23.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906598736Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65862024-02-12T01:37:23.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906635725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65872024-02-12T01:37:23.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.906874913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65882024-02-12T01:37:23.008Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.90693454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65892024-02-12T01:37:23.014Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907405978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65902024-02-12T01:37:23.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907871326Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65912024-02-12T01:37:23.014Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907905834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65922024-02-12T01:37:23.014Z{"msg":"[0]R flush_numbers[0..12]: [88, 102, 101, 101, 86, 102, 102, 98, 101, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907925733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65932024-02-12T01:37:23.014Z{"msg":"[0]R generation[0..12]: [52, 59, 58, 58, 52, 59, 59, 56, 58, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907943372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65942024-02-12T01:37:23.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.907970621Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65952024-02-12T01:37:23.014Z{"msg":"[1]R flush_numbers[0..12]: [88, 102, 101, 101, 86, 102, 102, 98, 101, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.90798908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65962024-02-12T01:37:23.014Z{"msg":"[1]R generation[0..12]: [52, 59, 58, 58, 52, 59, 59, 56, 58, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908032288Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65972024-02-12T01:37:23.015Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908051887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
65982024-02-12T01:37:23.015ZWait for a query_work_queue command to finish before sending IO
65992024-02-12T01:37:23.015Z{"msg":"[2]R flush_numbers[0..12]: [88, 102, 101, 101, 86, 102, 102, 98, 101, 80, 89, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908082936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66002024-02-12T01:37:23.015Z{"msg":"[2]R generation[0..12]: [52, 59, 58, 58, 52, 59, 59, 56, 58, 50, 52, 55]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908102765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66012024-02-12T01:37:23.015Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908121104Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66022024-02-12T01:37:23.015Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908138193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66032024-02-12T01:37:23.015Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908155462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66042024-02-12T01:37:23.015Z{"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.90819328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66052024-02-12T01:37:23.015Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908235258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66062024-02-12T01:37:23.015Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908277166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66072024-02-12T01:37:23.015Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908316705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66082024-02-12T01:37:23.015Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908357152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66092024-02-12T01:37:23.015Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908398601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66102024-02-12T01:37:23.015ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
66112024-02-12T01:37:23.015Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908444978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66122024-02-12T01:37:23.015Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908478327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66132024-02-12T01:37:23.015Z{"msg":"1811a26e-3312-4ca0-b7ec-2d419d6e46de is now active with session: 5c9a61d7-4059-44fa-a9f1-fbba506d0895","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908497096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66142024-02-12T01:37:23.015Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908520655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375,"session_id":"5c9a61d7-4059-44fa-a9f1-fbba506d0895"}
66152024-02-12T01:37:23.015Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:21.908536284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
66162024-02-12T01:37:23.015ZLoading write count information from file "/tmp/test_repair_verify.data"
66172024-02-12T01:37:23.015ZRun Repair workload
66182024-02-12T01:37:23.015Z01/30 Write block 265 len 20480 data: 5 5 5 4 3
66192024-02-12T01:37:23.016Z02/30 Read block 34 len 32768
66202024-02-12T01:37:23.016Z03/30 Write block 566 len 12288 data: 4 4 5
66212024-02-12T01:37:23.016Z04/30 Write block 370 len 28672 data: 6 5 5 6 5 7 7
66222024-02-12T01:37:23.019Z05/30 Read block 21 len 20480
66232024-02-12T01:37:23.019Z06/30 Read block 117 len 16384
66242024-02-12T01:37:23.019Z07/30 Read block 350 len 20480
66252024-02-12T01:37:23.019Z08/30 Read block 132 len 12288
66262024-02-12T01:37:23.019Z09/30 Write block 224 len 20480 data: 4 4 4 4 5
66272024-02-12T01:37:23.019Z10/30 Read block 367 len 36864
66282024-02-12T01:37:23.019Z11/30 Flush
66292024-02-12T01:37:23.034Z12/30 Write block 216 len 16384 data: 6 3 4 5
66302024-02-12T01:37:23.034Z13/30 Read block 287 len 16384
66312024-02-12T01:37:23.034Z14/30 Write block 573 len 12288 data: 7 6 8
66322024-02-12T01:37:23.034Z15/30 Read block 446 len 40960
66332024-02-12T01:37:23.034Z16/30 Read block 515 len 40960
66342024-02-12T01:37:23.034Z17/30 Write block 372 len 32768 data: 6 7 6 8 8 7 6 5
66352024-02-12T01:37:23.034Z18/30 Write block 424 len 8192 data: 5 4
66362024-02-12T01:37:23.034Z19/30 Write block 567 len 32768 data: 5 6 6 6 5 5 8 7
66372024-02-12T01:37:23.034Z20/30 Write block 152 len 24576 data: 7 5 5 4 3 4
66382024-02-12T01:37:23.034Z21/30 Read block 67 len 8192
66392024-02-12T01:37:23.034Z22/30 Read block 187 len 8192
66402024-02-12T01:37:23.035Z23/30 Write block 13 len 24576 data: 5 6 6 5 5 5
66412024-02-12T01:37:23.035Z24/30 Write block 161 len 4096 data: 5
66422024-02-12T01:37:23.035Z25/30 Write block 530 len 16384 data: 5 6 8 8
66432024-02-12T01:37:23.035Z26/30 Read block 371 len 12288
66442024-02-12T01:37:23.035Z27/30 Write block 216 len 32768 data: 7 4 5 6 5 3 4 4
66452024-02-12T01:37:23.035Z28/30 Read block 289 len 4096
66462024-02-12T01:37:23.039Z29/30 Read block 248 len 4096
66472024-02-12T01:37:23.039Z30/30 Read block 385 len 20480
66482024-02-12T01:37:23.039Z----------------------------------------------------------------
66492024-02-12T01:37:23.039Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:19
66502024-02-12T01:37:23.039ZStates: Active Active Active
66512024-02-12T01:37:23.039ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
66522024-02-12T01:37:23.039Z 12 Acked 1011 Write 4 Done Done Done false
66532024-02-12T01:37:23.040Z 13 Acked 1012 Read 4 Done Done Done false
66542024-02-12T01:37:23.040Z 14 Acked 1013 Write 3 Done Done Done false
66552024-02-12T01:37:23.040Z 15 Acked 1014 Read 10 Done Done Done false
66562024-02-12T01:37:23.040Z 16 Acked 1015 Read 10 Done Done Sent false
66572024-02-12T01:37:23.040Z 17 Acked 1016 Write 8 Done Done Sent false
66582024-02-12T01:37:23.040Z 18 Acked 1017 Write 2 Done Done Sent false
66592024-02-12T01:37:23.040Z 19 Acked 1018 Write 8 Done Done Sent false
66602024-02-12T01:37:23.040Z 20 Acked 1019 Write 6 Done Done Sent false
66612024-02-12T01:37:23.040Z 21 Acked 1020 Read 2 Done Done Sent false
66622024-02-12T01:37:23.040Z 22 Acked 1021 Read 2 Done Done Sent false
66632024-02-12T01:37:23.040Z 23 Acked 1022 Write 6 Done Done Sent false
66642024-02-12T01:37:23.040Z 24 Acked 1023 Write 1 Done Done Sent false
66652024-02-12T01:37:23.040Z 25 Acked 1024 Write 4 Done Done Sent false
66662024-02-12T01:37:23.040Z 26 Acked 1025 Read 3 Done Done Sent false
66672024-02-12T01:37:23.040Z 27 Acked 1026 Write 8 Done Done Sent false
66682024-02-12T01:37:23.040Z 28 Acked 1027 Read 1 Done Done Sent false
66692024-02-12T01:37:23.040Z 29 Acked 1028 Read 1 Done Done Sent false
66702024-02-12T01:37:23.040Z 30 Acked 1029 Read 5 Done Sent Sent false
66712024-02-12T01:37:23.040Z STATES DS:0 DS:1 DS:2 TOTAL
66722024-02-12T01:37:23.040Z New 0 0 0 0
66732024-02-12T01:37:23.040Z Sent 0 1 15 16
66742024-02-12T01:37:23.040Z Done 19 18 4 41
66752024-02-12T01:37:23.040Z Skipped 0 0 0 0
66762024-02-12T01:37:23.040Z Error 0 0 0 0
66772024-02-12T01:37:23.040ZLast Flush: 1010 1010 1010
66782024-02-12T01:37:23.041ZDownstairs last five completed: 1010 1009 1008 1007 1006
66792024-02-12T01:37:23.041ZUpstairs last five completed: 30 29 28 27 26
66802024-02-12T01:37:23.041Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:21.937458004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1375}
66812024-02-12T01:37:23.041ZWrote out file "/tmp/test_repair_verify.data"
66822024-02-12T01:37:23.044Z
66832024-02-12T01:37:23.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66842024-02-12T01:37:23.063ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66852024-02-12T01:37:23.063Z 0 000-019 52 52 52 88 88 88 T T F <---
66862024-02-12T01:37:23.063Z 1 020-039 59 59 59 102 102 102 F F F
66872024-02-12T01:37:23.063Z 2 040-059 58 58 58 101 101 101 F F F
66882024-02-12T01:37:23.063Z 3 060-079 58 58 58 101 101 101 F F F
66892024-02-12T01:37:23.063Z 4 080-099 52 52 52 86 86 86 F F F
66902024-02-12T01:37:23.063Z 5 100-119 59 59 59 102 102 102 F F F
66912024-02-12T01:37:23.063Z 6 120-139 59 59 59 102 102 102 F F F
66922024-02-12T01:37:23.063Z 7 140-159 56 56 56 98 98 98 T T F <---
66932024-02-12T01:37:23.063Z 8 160-179 58 58 58 101 101 101 T T F <---
66942024-02-12T01:37:23.063Z 9 180-199 50 50 50 80 80 80 F F F
66952024-02-12T01:37:23.063Z 10 200-219 52 52 52 89 89 89 T T T
66962024-02-12T01:37:23.064Z 11 220-239 60 60 60 103 103 103 T T F <---
66972024-02-12T01:37:23.067Z 12 240-259 59 59 59 102 102 102 F F F
66982024-02-12T01:37:23.067Z 13 260-279 60 60 60 103 103 103 F F F
66992024-02-12T01:37:23.067Z 14 280-299 56 56 56 98 98 98 F F F
67002024-02-12T01:37:23.067Z 15 300-319 55 55 55 97 97 97 F F F
67012024-02-12T01:37:23.067Z 16 320-339 58 58 58 101 101 101 F F F
67022024-02-12T01:37:23.067Z 17 340-359 52 52 52 85 85 85 F F F
67032024-02-12T01:37:23.067Z 18 360-379 60 60 60 103 103 103 T T F <---
67042024-02-12T01:37:23.067Z 19 380-399 57 57 57 100 100 100 F F F
67052024-02-12T01:37:23.067Z 20 400-419 57 57 57 100 100 100 F F F
67062024-02-12T01:37:23.067Z 21 420-439 56 56 56 98 98 98 T T F <---
67072024-02-12T01:37:23.067Z 22 440-459 56 56 56 98 98 98 F F F
67082024-02-12T01:37:23.067Z 23 460-479 56 56 56 98 98 98 F F F
67092024-02-12T01:37:23.067Z 24 480-499 54 54 54 94 94 94 F F F
67102024-02-12T01:37:23.067Z 25 500-519 56 56 56 99 99 99 F F F
67112024-02-12T01:37:23.067Z 26 520-539 58 58 58 101 101 101 T T F <---
67122024-02-12T01:37:23.067Z 27 540-559 51 51 51 83 83 83 F F F
67132024-02-12T01:37:23.067Z 28 560-579 60 60 60 103 103 103 T T T
67142024-02-12T01:37:23.067Z 29 580-599 58 58 58 101 101 101 F F F
67152024-02-12T01:37:23.067ZMax gen: 60, Max flush: 103
67162024-02-12T01:37:23.068ZError: Difference in extent metadata found!
67172024-02-12T01:37:23.068ZA Difference in extent metadata is expected here
67182024-02-12T01:37:23.068ZOn loop 30
67192024-02-12T01:37:23.068Z
67202024-02-12T01:37:23.071ZVerifying data now
67212024-02-12T01:37:28.551ZNo change after verify
67222024-02-12T01:37:28.552ZLoop: 30 Downstairs dump after verify (and repair):
67232024-02-12T01:37:28.567ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67242024-02-12T01:37:28.567Z 0 000-019 61 61 61 104 104 104 F F F
67252024-02-12T01:37:28.567Z 1 020-039 59 59 59 102 102 102 F F F
67262024-02-12T01:37:28.567Z 2 040-059 58 58 58 101 101 101 F F F
67272024-02-12T01:37:28.567Z 3 060-079 58 58 58 101 101 101 F F F
67282024-02-12T01:37:28.567Z 4 080-099 52 52 52 86 86 86 F F F
67292024-02-12T01:37:28.567Z 5 100-119 59 59 59 102 102 102 F F F
67302024-02-12T01:37:28.567Z 6 120-139 59 59 59 102 102 102 F F F
67312024-02-12T01:37:28.567Z 7 140-159 61 61 61 104 104 104 F F F
67322024-02-12T01:37:28.570Z 8 160-179 61 61 61 104 104 104 F F F
67332024-02-12T01:37:28.570Z 9 180-199 50 50 50 80 80 80 F F F
67342024-02-12T01:37:28.571Z 10 200-219 61 61 61 104 104 104 F F F
67352024-02-12T01:37:28.571Z 11 220-239 61 61 61 104 104 104 F F F
67362024-02-12T01:37:28.571Z 12 240-259 59 59 59 102 102 102 F F F
67372024-02-12T01:37:28.571Z 13 260-279 60 60 60 103 103 103 F F F
67382024-02-12T01:37:28.571Z 14 280-299 56 56 56 98 98 98 F F F
67392024-02-12T01:37:28.571Z 15 300-319 55 55 55 97 97 97 F F F
67402024-02-12T01:37:28.571Z 16 320-339 58 58 58 101 101 101 F F F
67412024-02-12T01:37:28.571Z 17 340-359 52 52 52 85 85 85 F F F
67422024-02-12T01:37:28.571Z 18 360-379 61 61 61 104 104 104 F F F
67432024-02-12T01:37:28.571Z 19 380-399 57 57 57 100 100 100 F F F
67442024-02-12T01:37:28.571Z 20 400-419 57 57 57 100 100 100 F F F
67452024-02-12T01:37:28.571Z 21 420-439 61 61 61 104 104 104 F F F
67462024-02-12T01:37:28.571Z 22 440-459 56 56 56 98 98 98 F F F
67472024-02-12T01:37:28.571Z 23 460-479 56 56 56 98 98 98 F F F
67482024-02-12T01:37:28.571Z 24 480-499 54 54 54 94 94 94 F F F
67492024-02-12T01:37:28.571Z 25 500-519 56 56 56 99 99 99 F F F
67502024-02-12T01:37:28.571Z 26 520-539 61 61 61 104 104 104 F F F
67512024-02-12T01:37:28.571Z 27 540-559 51 51 51 83 83 83 F F F
67522024-02-12T01:37:28.571Z 28 560-579 61 61 61 104 104 104 F F F
67532024-02-12T01:37:28.571Z 29 580-599 58 58 58 101 101 101 F F F
67542024-02-12T01:37:28.571ZMax gen: 61, Max flush: 104
67552024-02-12T01:37:28.571Z
67562024-02-12T01:37:28.571ZBegin loop 31
67572024-02-12T01:37:28.571ZDownstairs to restart: 2
67582024-02-12T01:37:28.589Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.489639125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
67592024-02-12T01:37:28.589Z{"msg":"Crucible Version: BuildInfo {\nCrucible runtime is spawned
67602024-02-12T01:37:28.589Z version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.48994466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
67612024-02-12T01:37:28.589Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.489991758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
67622024-02-12T01:37:28.594Z{"msg":"Crucible dfb188d4-3ce5-4f5c-b039-faf4daef3566 has session id: 4d84ba4f-948a-40cd-8b70-9c0dbd415c71","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490021447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67632024-02-12T01:37:28.594Z{"msg":"Upstairs opts: Upstairs UUID: dfb188d4-3ce5-4f5c-b039-faf4daef3566, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490077444Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67642024-02-12T01:37:28.594Z{"msg":"Crucible stats registered with UUID: dfb188d4-3ce5-4f5c-b039-faf4daef3566","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490109503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67652024-02-12T01:37:28.594Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490151461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
67662024-02-12T01:37:28.594Z{"msg":"dfb188d4-3ce5-4f5c-b039-faf4daef3566 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490230437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67672024-02-12T01:37:28.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490274465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67682024-02-12T01:37:28.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490326162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67692024-02-12T01:37:28.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490442137Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67702024-02-12T01:37:28.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490489164Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67712024-02-12T01:37:28.594Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490517673Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67722024-02-12T01:37:28.594Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490543092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67732024-02-12T01:37:28.595Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490566891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67742024-02-12T01:37:28.595Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490591909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67752024-02-12T01:37:28.595Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490615748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67762024-02-12T01:37:28.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490640247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67772024-02-12T01:37:28.595Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:27.490683395Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67782024-02-12T01:37:28.595Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:27.490714514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67792024-02-12T01:37:28.595Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:27.490758972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67802024-02-12T01:37:28.595Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.49078466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67812024-02-12T01:37:28.595Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490820998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67822024-02-12T01:37:28.595Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490845807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67832024-02-12T01:37:28.595Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.490869736Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67842024-02-12T01:37:28.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.491266667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67852024-02-12T01:37:28.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.491303615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67862024-02-12T01:37:28.595Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.491353863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67872024-02-12T01:37:28.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.491380852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67882024-02-12T01:37:28.595Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.49184682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67892024-02-12T01:37:28.595Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492097308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67902024-02-12T01:37:28.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492359705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67912024-02-12T01:37:28.595Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.49246539Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67922024-02-12T01:37:28.595Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492524977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67932024-02-12T01:37:28.595Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492615603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67942024-02-12T01:37:28.595Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492645642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67952024-02-12T01:37:28.595Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:27.492660891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67962024-02-12T01:37:29.591Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.491146774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67972024-02-12T01:37:29.591Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.49123692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67982024-02-12T01:37:29.596Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.492108298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
67992024-02-12T01:37:29.596Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.492145306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68002024-02-12T01:37:29.596Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.492639903Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68012024-02-12T01:37:29.596Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493195336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68022024-02-12T01:37:29.596Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493232895Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68032024-02-12T01:37:29.596Z{"msg":"[0]R flush_numbers[0..12]: [104, 102, 101, 101, 86, 102, 102, 104, 104, 80, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493251514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68042024-02-12T01:37:29.596Z{"msg":"[0]R generation[0..12]: [61, 59, 58, 58, 52, 59, 59, 61, 61, 50, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493277052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68052024-02-12T01:37:29.596Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493295312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68062024-02-12T01:37:29.596Z{"msg":"[1]R flush_numbers[0..12]: [104, 102, 101, 101, 86, 102, 102, 104, 104, 80, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493313461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68072024-02-12T01:37:29.596Z{"msg":"[1]R generation[0..12]: [61, 59, 58, 58, 52, 59, 59, 61, 61, 50, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493339129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68082024-02-12T01:37:29.596Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493357059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68092024-02-12T01:37:29.596Z{"msg":"[2]R flush_numbers[0..12]: [104, 102, 101, 101, 86, 102, 102, 104, 104, 80, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493382318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
68102024-02-12T01:37:29.596Z,"pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68112024-02-12T01:37:29.596Z{"msg":"[2]R generation[0..12]: [61, 59, 58, 58, 52, 59, 59, 61, 61, 50, 61, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493404517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68122024-02-12T01:37:29.596Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493430245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68132024-02-12T01:37:29.596Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493447094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68142024-02-12T01:37:29.597Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493472213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68152024-02-12T01:37:29.597Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493488632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68162024-02-12T01:37:29.597Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493506092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68172024-02-12T01:37:29.597Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493522721Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68182024-02-12T01:37:29.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.49353895Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68192024-02-12T01:37:29.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493561349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68202024-02-12T01:37:29.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493574718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68212024-02-12T01:37:29.597Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493594597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68222024-02-12T01:37:29.597Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493607257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68232024-02-12T01:37:29.597Z{"msg":"dfb188d4-3ce5-4f5c-b039-faf4daef3566 is now active with session: 4d84ba4f-948a-40cd-8b70-9c0dbd415c71","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493620406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68242024-02-12T01:37:29.597Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493638715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383,"session_id":"4d84ba4f-948a-40cd-8b70-9c0dbd415c71"}
68252024-02-12T01:37:29.597Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:28.493651435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
68262024-02-12T01:37:29.597ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
68272024-02-12T01:37:29.597ZLoading write count information from file "/tmp/test_repair_verify.data"
68282024-02-12T01:37:29.597ZRun Repair workload
68292024-02-12T01:37:29.597Z01/30 Write block 221 len 4096 data: 4
68302024-02-12T01:37:29.607Z02/30 Flush
68312024-02-12T01:37:29.607Z03/30 Write block 173 len 36864 data: 3 3 7 8 8 8 8 6 7
68322024-02-12T01:37:29.607Z04/30 Flush
68332024-02-12T01:37:29.607Z05/30 Write block 96 len 16384 data: 4 3 4 3
68342024-02-12T01:37:29.607Z06/30 Write block 20 len 16384 data: 3 3 4 4
68352024-02-12T01:37:29.607Z07/30 Read block 518 len 36864
68362024-02-12T01:37:29.607Z08/30 Read block 546 len 24576
68372024-02-12T01:37:29.610Z09/30 Write block 164 len 12288 data: 6 7 7
68382024-02-12T01:37:29.610Z10/30 Read block 203 len 20480
68392024-02-12T01:37:29.610Z11/30 Read block 411 len 16384
68402024-02-12T01:37:29.610Z12/30 Write block 592 len 4096 data: 4
68412024-02-12T01:37:29.611Z13/30 Write block 184 len 40960 data: 6 5 5 8 9 9 7 6 6 5
68422024-02-12T01:37:29.611Z14/30 Read block 531 len 12288
68432024-02-12T01:37:29.611Z15/30 Read block 360 len 40960
68442024-02-12T01:37:29.613Z16/30 Read block 242 len 4096
68452024-02-12T01:37:29.613Z17/30 Write block 568 len 28672 data: 7 7 7 6 6 9 8
68462024-02-12T01:37:29.613Z18/30 Read block 317 len 36864
68472024-02-12T01:37:29.616Z19/30 Write block 196 len 16384 data: 2 3 3 4
68482024-02-12T01:37:29.616Z20/30 Read block 290 len 12288
68492024-02-12T01:37:29.616Z21/30 Write block 148 len 16384 data: 4 5 6 7
68502024-02-12T01:37:29.616Z22/30 Flush
68512024-02-12T01:37:29.619Z23/30 Write block 96 len 12288 data: 5 4 5
68522024-02-12T01:37:29.622Z24/30 Write block 391 len 32768 data: 8 7 6 5 5 4 4 4
68532024-02-12T01:37:29.622Z25/30 Read block 471 len 8192
68542024-02-12T01:37:29.622Z26/30 Write block 8 len 4096 data: 6
68552024-02-12T01:37:29.622Z27/30 Read block 463 len 40960
68562024-02-12T01:37:29.625Z28/30 Read block 320 len 32768
68572024-02-12T01:37:29.625Z29/30 Write block 325 len 8192 data: 4 4
68582024-02-12T01:37:29.626Z30/30 Write block 398 len 36864 data: 5 4 4 3 5 4 5 5 5
68592024-02-12T01:37:29.626Z----------------------------------------------------------------
68602024-02-12T01:37:29.626Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:26
68612024-02-12T01:37:29.626ZStates: Active Active Active
68622024-02-12T01:37:29.626ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
68632024-02-12T01:37:29.626Z 5 Acked 1004 Write 4 Done Done Done false
68642024-02-12T01:37:29.626Z 6 Acked 1005 Write 4 Done Done Done false
68652024-02-12T01:37:29.626Z 7 Acked 1006 Read 9 Done Done Done false
68662024-02-12T01:37:29.626Z 8 Acked 1007 Read 6 Done Done Done false
68672024-02-12T01:37:29.626Z 9 Acked 1008 Write 3 Done Done Done false
68682024-02-12T01:37:29.626Z 10 Acked 1009 Read 5 Done Done Sent false
68692024-02-12T01:37:29.626Z 11 Acked 1010 Read 4 Done Done Sent false
68702024-02-12T01:37:29.626Z 12 Acked 1011 Write 1 Done Done Sent false
68712024-02-12T01:37:29.626Z 13 Acked 1012 Write 10 Done Done Sent false
68722024-02-12T01:37:29.626Z 14 Acked 1013 Read 3 Done Done Sent false
68732024-02-12T01:37:29.626Z 15 Acked 1014 Read 10 Done Done Sent false
68742024-02-12T01:37:29.626Z 16 Acked 1015 Read 1 Done Done Sent false
68752024-02-12T01:37:29.626Z 17 Acked 1016 Write 7 Done Done Sent false
68762024-02-12T01:37:29.626Z 18 Acked 1017 Read 9 Done Done Sent false
68772024-02-12T01:37:29.626Z 19 Acked 1018 Write 4 Done Done Sent false
68782024-02-12T01:37:29.626Z 20 Acked 1019 Read 3 Done Done Sent false
68792024-02-12T01:37:29.626Z 21 Acked 1020 Write 4 Done Done Sent false
68802024-02-12T01:37:29.626Z 22 Acked 1021 Flush 0 Done Done Sent false
68812024-02-12T01:37:29.626Z 23 Acked 1022 Write 3 Done Done Sent false
68822024-02-12T01:37:29.627Z 24 Acked 1023 Write 8 Done Done Sent false
68832024-02-12T01:37:29.627Z 25 Acked 1024 Read 2 Done Done Sent false
68842024-02-12T01:37:29.627Z 26 Acked 1025 Write 1 Done Done Sent false
68852024-02-12T01:37:29.627Z 27 Acked 1026 Read 10 Done Done Sent false
68862024-02-12T01:37:29.627Z 28 Acked 1027 Read 8 Done Done Sent false
68872024-02-12T01:37:29.627Z 29 Acked 1028 Write 2 Done Done Sent false
68882024-02-12T01:37:29.627Z 30 Acked 1029 Write 9 Sent Sent Sent false
68892024-02-12T01:37:29.627Z STATES DS:0 DS:1 DS:2 TOTAL
68902024-02-12T01:37:29.627Z New 0 0 0 0
68912024-02-12T01:37:29.627Z Sent 1 1 21 23
68922024-02-12T01:37:29.627Z Done 25 25 5 55
68932024-02-12T01:37:29.627Z Skipped 0 0 0 0
68942024-02-12T01:37:29.627Z Error 0 0 0 0
68952024-02-12T01:37:29.627ZLast Flush: 1021 1021 1003
68962024-02-12T01:37:29.627ZDownstairs last five completed: 1003 1002 1001 1000
68972024-02-12T01:37:29.627ZUpstairs last five completed: 30 29 28 27 26
68982024-02-12T01:37:29.630Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:28.526088177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1383}
68992024-02-12T01:37:29.630ZWrote out file "/tmp/test_repair_verify.data"
69002024-02-12T01:37:29.630Z
69012024-02-12T01:37:29.634ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
69022024-02-12T01:37:29.651ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69032024-02-12T01:37:29.651Z 0 000-019 61 61 61 104 104 104 T T F <---
69042024-02-12T01:37:29.651Z 1 020-039 62 62 59 107 107 102 F F T <---
69052024-02-12T01:37:29.651Z 2 040-059 58 58 58 101 101 101 F F F
69062024-02-12T01:37:29.652Z 3 060-079 58 58 58 101 101 101 F F F
69072024-02-12T01:37:29.652Z 4 080-099 62 62 52 107 107 86 T T T <---
69082024-02-12T01:37:29.652Z 5 100-119 59 59 59 102 102 102 F F F
69092024-02-12T01:37:29.652Z 6 120-139 59 59 59 102 102 102 F F F
69102024-02-12T01:37:29.652Z 7 140-159 62 62 61 107 107 104 F F F <---
69112024-02-12T01:37:29.652Z 8 160-179 62 62 62 107 107 106 F F T <---
69122024-02-12T01:37:29.652Z 9 180-199 62 62 62 107 107 106 F F F <---
69132024-02-12T01:37:29.652Z 10 200-219 61 61 61 104 104 104 F F F
69142024-02-12T01:37:29.655Z 11 220-239 62 62 62 105 105 105 F F F
69152024-02-12T01:37:29.655Z 12 240-259 59 59 59 102 102 102 F F F
69162024-02-12T01:37:29.655Z 13 260-279 60 60 60 103 103 103 F F F
69172024-02-12T01:37:29.655Z 14 280-299 56 56 56 98 98 98 F F F
69182024-02-12T01:37:29.655Z 15 300-319 55 55 55 97 97 97 F F F
69192024-02-12T01:37:29.655Z 16 320-339 58 58 58 101 101 101 T T F <---
69202024-02-12T01:37:29.655Z 17 340-359 52 52 52 85 85 85 F F F
69212024-02-12T01:37:29.655Z 18 360-379 61 61 61 104 104 104 F F F
69222024-02-12T01:37:29.655Z 19 380-399 57 57 57 100 100 100 T T F <---
69232024-02-12T01:37:29.655Z 20 400-419 57 57 57 100 100 100 T T F <---
69242024-02-12T01:37:29.655Z 21 420-439 61 61 61 104 104 104 F F F
69252024-02-12T01:37:29.656Z 22 440-459 56 56 56 98 98 98 F F F
69262024-02-12T01:37:29.656Z 23 460-479 56 56 56 98 98 98 F F F
69272024-02-12T01:37:29.656Z 24 480-499 54 54 54 94 94 94 F F F
69282024-02-12T01:37:29.656Z 25 500-519 56 56 56 99 99 99 F F F
69292024-02-12T01:37:29.656Z 26 520-539 61 61 61 104 104 104 F F F
69302024-02-12T01:37:29.656Z 27 540-559 51 51 51 83 83 83 F F F
69312024-02-12T01:37:29.656Z 28 560-579 62 62 61 107 107 104 F F F <---
69322024-02-12T01:37:29.656Z 29 580-599 62 62 58 107 107 101 F F F <---
69332024-02-12T01:37:29.656ZMax gen: 62, Max flush: 107
69342024-02-12T01:37:29.656ZError: Difference in extent metadata found!
69352024-02-12T01:37:29.656ZA Difference in extent metadata is expected here
69362024-02-12T01:37:29.656ZOn loop 31
69372024-02-12T01:37:29.656Z
69382024-02-12T01:37:29.659ZVerifying data now
69392024-02-12T01:37:35.212ZNo change after verify
69402024-02-12T01:37:35.212ZLoop: 31 Downstairs dump after verify (and repair):
69412024-02-12T01:37:35.227ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
69422024-02-12T01:37:35.227Z 0 000-019 63 63 63 108 108 108 F F F
69432024-02-12T01:37:35.230Z 1 020-039 62 62 62 107 107 107 F F F
69442024-02-12T01:37:35.230Z 2 040-059 58 58 58 101 101 101 F F F
69452024-02-12T01:37:35.230Z 3 060-079 58 58 58 101 101 101 F F F
69462024-02-12T01:37:35.230Z 4 080-099 63 63 63 108 108 108 F F F
69472024-02-12T01:37:35.230Z 5 100-119 59 59 59 102 102 102 F F F
69482024-02-12T01:37:35.230Z 6 120-139 59 59 59 102 102 102 F F F
69492024-02-12T01:37:35.230Z 7 140-159 62 62 62 107 107 107 F F F
69502024-02-12T01:37:35.230Z 8 160-179 62 62 62 107 107 107 F F F
69512024-02-12T01:37:35.230Z 9 180-199 62 62 62 107 107 107 F F F
69522024-02-12T01:37:35.231Z 10 200-219 61 61 61 104 104 104 F F F
69532024-02-12T01:37:35.231Z 11 220-239 62 62 62 105 105 105 F F F
69542024-02-12T01:37:35.231Z 12 240-259 59 59 59 102 102 102 F F F
69552024-02-12T01:37:35.231Z 13 260-279 60 60 60 103 103 103 F F F
69562024-02-12T01:37:35.231Z 14 280-299 56 56 56 98 98 98 F F F
69572024-02-12T01:37:35.231Z 15 300-319 55 55 55 97 97 97 F F F
69582024-02-12T01:37:35.231Z 16 320-339 63 63 63 108 108 108 F F F
69592024-02-12T01:37:35.231Z 17 340-359 52 52 52 85 85 85 F F F
69602024-02-12T01:37:35.231Z 18 360-379 61 61 61 104 104 104 F F F
69612024-02-12T01:37:35.231Z 19 380-399 63 63 63 108 108 108 F F F
69622024-02-12T01:37:35.231Z 20 400-419 63 63 63 108 108 108 F F F
69632024-02-12T01:37:35.231Z 21 420-439 61 61 61 104 104 104 F F F
69642024-02-12T01:37:35.231Z 22 440-459 56 56 56 98 98 98 F F F
69652024-02-12T01:37:35.231Z 23 460-479 56 56 56 98 98 98 F F F
69662024-02-12T01:37:35.231Z 24 480-499 54 54 54 94 94 94 F F F
69672024-02-12T01:37:35.231Z 25 500-519 56 56 56 99 99 99 F F F
69682024-02-12T01:37:35.231Z 26 520-539 61 61 61 104 104 104 F F F
69692024-02-12T01:37:35.231Z 27 540-559 51 51 51 83 83 83 F F F
69702024-02-12T01:37:35.231Z 28 560-579 62 62 62 107 107 107 F F F
69712024-02-12T01:37:35.231Z 29 580-599 62 62 62 107 107 107 F F F
69722024-02-12T01:37:35.231ZMax gen: 63, Max flush: 108
69732024-02-12T01:37:35.234Z
69742024-02-12T01:37:35.235ZBegin loop 32
69752024-02-12T01:37:35.235ZDownstairs to restart: 1
69762024-02-12T01:37:35.249Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.148702878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
69772024-02-12T01:37:35.249Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
69782024-02-12T01:37:35.254Z9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149063781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
69792024-02-12T01:37:35.254Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149112898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
69802024-02-12T01:37:35.254Z{"msg":"Crucible ed42ceac-727a-49cc-a487-72059ddaf834 has session id: ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149148416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69812024-02-12T01:37:35.254Z{"msg":"Upstairs opts: Upstairs UUID: ed42ceac-727a-49cc-a487-72059ddaf834, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149176505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69822024-02-12T01:37:35.254Z{"msg":"Crucible stats registered with UUID: ed42ceac-727a-49cc-a487-72059ddaf834","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149202594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69832024-02-12T01:37:35.254Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149218753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
69842024-02-12T01:37:35.254Z{"msg":"ed42ceac-727a-49cc-a487-72059ddaf834 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149246412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69852024-02-12T01:37:35.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.14927886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69862024-02-12T01:37:35.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149304799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69872024-02-12T01:37:35.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149340598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69882024-02-12T01:37:35.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149367956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69892024-02-12T01:37:35.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149399804Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69902024-02-12T01:37:35.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149416864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69912024-02-12T01:37:35.254Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149449022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69922024-02-12T01:37:35.254Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149505499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69932024-02-12T01:37:35.254Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149529788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69942024-02-12T01:37:35.255Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149564507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69952024-02-12T01:37:35.255Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149741848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69962024-02-12T01:37:35.255Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:34.149804425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69972024-02-12T01:37:35.255Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:34.149849803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69982024-02-12T01:37:35.255Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:34.149892961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
69992024-02-12T01:37:35.255Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.14991801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70002024-02-12T01:37:35.255Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.149942509Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70012024-02-12T01:37:35.255Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.15012469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70022024-02-12T01:37:35.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150262154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70032024-02-12T01:37:35.255Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150311691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70042024-02-12T01:37:35.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150346999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70052024-02-12T01:37:35.255Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150402637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70062024-02-12T01:37:35.255Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150846486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70072024-02-12T01:37:35.255Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.150978689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70082024-02-12T01:37:35.255Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151455486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70092024-02-12T01:37:35.255Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151489995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70102024-02-12T01:37:35.255Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151504984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70112024-02-12T01:37:35.255Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151535673Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70122024-02-12T01:37:35.255Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151568531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70132024-02-12T01:37:35.255Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:34.151601019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70142024-02-12T01:37:36.252Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.151358792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70152024-02-12T01:37:36.252Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.151451388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70162024-02-12T01:37:36.257Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.152343885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70172024-02-12T01:37:36.257Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.152379063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70182024-02-12T01:37:36.257Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.152830032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70192024-02-12T01:37:36.257Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153399724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70202024-02-12T01:37:36.257Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153439913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70212024-02-12T01:37:36.257Z{"msg":"[0]R flush_numbers[0..12]: [108, 107, 101, 101, 108, 102, 102, 107, 107, 107, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153459862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70222024-02-12T01:37:36.257Z{"msg":"[0]R generation[0..12]: [63, 62, 58, 58, 63, 59, 59, 62, 62, 62, 61, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.15348675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70232024-02-12T01:37:36.257Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153505719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70242024-02-12T01:37:36.257Z{"msg":"[1]R flush_numbers[0..12]: [108, 107, 101, 101, 108, 102, 102, 107, 107, 107, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153532348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"Wait for a query_work_queue command to finish before sending IO
70252024-02-12T01:37:36.257Zdownstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70262024-02-12T01:37:36.257Z{"msg":"[1]R generation[0..12]: [63, 62, 58, 58, 63, 59, 59, 62, 62, 62, 61, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153561777Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70272024-02-12T01:37:36.257Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153588315Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70282024-02-12T01:37:36.257Z{"msg":"[2]R flush_numbers[0..12]: [108, 107, 101, 101, 108, 102, 102, 107, 107, 107, 104, 105]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153607425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70292024-02-12T01:37:36.257Z{"msg":"[2]R generation[0..12]: [63, 62, 58, 58, 63, 59, 59, 62, 62, 62, 61, 62]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153639083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70302024-02-12T01:37:36.257Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153656822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70312024-02-12T01:37:36.257Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153687971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70322024-02-12T01:37:36.258Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.15370601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70332024-02-12T01:37:36.258Z{"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153744978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70342024-02-12T01:37:36.258Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153789366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70352024-02-12T01:37:36.258Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153830214Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70362024-02-12T01:37:36.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153865892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70372024-02-12T01:37:36.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.15390945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70382024-02-12T01:37:36.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153946998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70392024-02-12T01:37:36.258Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.153989087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70402024-02-12T01:37:36.258Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.154028345Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70412024-02-12T01:37:36.258ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
70422024-02-12T01:37:36.258Z{"msg":"ed42ceac-727a-49cc-a487-72059ddaf834 is now active with session: ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.154057403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70432024-02-12T01:37:36.258Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.154094461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
70442024-02-12T01:37:36.258Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:35.15411315Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
70452024-02-12T01:37:36.258ZLoading write count information from file "/tmp/test_repair_verify.data"
70462024-02-12T01:37:36.258ZRun Repair workload
70472024-02-12T01:37:36.265Z01/30 Write block 144 len 28672 data: 7 8 7 6 5 6 7
70482024-02-12T01:37:36.265Z02/30 Flush
70492024-02-12T01:37:36.265Z03/30 Write block 197 len 36864 data: 4 4 5 4 4 4 5 5 4
70502024-02-12T01:37:36.265Z04/30 Read block 117 len 32768
70512024-02-12T01:37:36.268Z05/30 Write block 287 len 28672 data: 7 7 6 6 5 3 4
70522024-02-12T01:37:36.268Z06/30 Read block 0 len 20480
70532024-02-12T01:37:36.268Z07/30 Read block 577 len 28672
70542024-02-12T01:37:36.268Z08/30 Write block 82 len 20480 data: 9 6 7 7 6
70552024-02-12T01:37:36.268Z09/30 Write block 178 len 16384 data: 9 9 7 8
70562024-02-12T01:37:36.268Z10/30 Write block 216 len 32768 data: 8 5 6 7 6 5 5 5
70572024-02-12T01:37:36.268Z11/30 Read block 568 len 16384
70582024-02-12T01:37:36.268Z12/30 Read block 183 len 36864
70592024-02-12T01:37:36.272Z13/30 Write block 146 len 36864 data: 8 7 6 7 8 8 8 6 6
70602024-02-12T01:37:36.272Z14/30 Read block 561 len 20480
70612024-02-12T01:37:36.272Z15/30 Flush
70622024-02-12T01:37:36.274Z16/30 Write block 137 len 36864 data: 5 6 5 5 7 8 8 8 9
70632024-02-12T01:37:36.274Z17/30 Read block 376 len 36864
70642024-02-12T01:37:36.277Z18/30 Write block 48 len 20480 data: 6 6 6 6 3
70652024-02-12T01:37:36.277Z19/30 Read block 483 len 8192
70662024-02-12T01:37:36.277Z20/30 Write block 206 len 16384 data: 6 8 8 7
70672024-02-12T01:37:36.280Z21/30 Read block 55 len 4096
70682024-02-12T01:37:36.280Z22/30 Write block 334 len 20480 data: 3 3 3 4 4
70692024-02-12T01:37:36.280Z23/30 Flush
70702024-02-12T01:37:36.283Z24/30 Write block 79 len 36864 data: 8 9 9 10 7 8 8 7 8
70712024-02-12T01:37:36.283Z25/30 Read block 328 len 20480
70722024-02-12T01:37:36.286Z26/30 Read block 548 len 16384
70732024-02-12T01:37:36.286Z27/30 Read block 392 len 16384
70742024-02-12T01:37:36.286Z28/30 Read block 485 len 24576
70752024-02-12T01:37:36.286Z29/30 Write block 236 len 20480 data: 6 7 8 7 5
70762024-02-12T01:37:36.292Z30/30 Write block 117 len 8192 data: 9 9
70772024-02-12T01:37:36.292Z----------------------------------------------------------------
70782024-02-12T01:37:36.292Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:28
70792024-02-12T01:37:36.292ZStates: Active Active Active
70802024-02-12T01:37:36.292ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
70812024-02-12T01:37:36.292Z 3 Acked 1002 Write 9 Done Done Done false
70822024-02-12T01:37:36.292Z 4 Acked 1003 Read 8 Done Done Done false
70832024-02-12T01:37:36.292Z 5 Acked 1004 Write 7 Done Done Done false
70842024-02-12T01:37:36.292Z 6 Acked 1005 Read 5 Done Done Done false
70852024-02-12T01:37:36.292Z 7 Acked 1006 Read 7 Done Sent Done false
70862024-02-12T01:37:36.293Z 8 Acked 1007 Write 5 Done Sent Done false
70872024-02-12T01:37:36.293Z 9 Acked 1008 Write 4 Done Sent Done false
70882024-02-12T01:37:36.293Z 10 Acked 1009 Write 8 Done Sent Done false
70892024-02-12T01:37:36.293Z 11 Acked 1010 Read 4 Done Sent Done false
70902024-02-12T01:37:36.293Z 12 Acked 1011 Read 9 Done Sent Done false
70912024-02-12T01:37:36.293Z 13 Acked 1012 Write 9 Done Sent Done false
70922024-02-12T01:37:36.293Z 14 Acked 1013 Read 5 Done Sent Done false
70932024-02-12T01:37:36.293Z 15 Acked 1014 Flush 0 Done Sent Done false
70942024-02-12T01:37:36.293Z 16 Acked 1015 Write 9 Done Sent Done false
70952024-02-12T01:37:36.293Z 17 Acked 1016 Read 9 Done Sent Done false
70962024-02-12T01:37:36.293Z 18 Acked 1017 Write 5 Done Sent Done false
70972024-02-12T01:37:36.293Z 19 Acked 1018 Read 2 Done Sent Done false
70982024-02-12T01:37:36.293Z 20 Acked 1019 Write 4 Done Sent Done false
70992024-02-12T01:37:36.293Z 21 Acked 1020 Read 1 Done Sent Done false
71002024-02-12T01:37:36.293Z 22 Acked 1021 Write 5 Done Sent Done false
71012024-02-12T01:37:36.293Z 23 Acked 1022 Flush 0 Done Sent Done false
71022024-02-12T01:37:36.293Z 24 Acked 1023 Write 9 Done Sent Done false
71032024-02-12T01:37:36.293Z 25 Acked 1024 Read 5 Done Sent Done false
71042024-02-12T01:37:36.293Z 26 Acked 1025 Read 4 Done Sent Done false
71052024-02-12T01:37:36.293Z 27 Acked 1026 Read 4 Done Sent Done false
71062024-02-12T01:37:36.293Z 28 Acked 1027 Read 6 Done Sent Done false
71072024-02-12T01:37:36.293Z 29 Acked 1028 Write 5 Sent Sent Sent false
71082024-02-12T01:37:36.293Z 30 Acked 1029 Write 2 Sent Sent Sent false
71092024-02-12T01:37:36.293Z STATES DS:0 DS:1 DS:2 TOTAL
71102024-02-12T01:37:36.293Z New 0 0 0 0
71112024-02-12T01:37:36.294Z Sent 2 24 2 28
71122024-02-12T01:37:36.294Z Done 26 4 26 56
71132024-02-12T01:37:36.294Z Skipped 0 0 0 0
71142024-02-12T01:37:36.294Z Error 0 0 0 0
71152024-02-12T01:37:36.294ZLast Flush: 1022 1001 1022
71162024-02-12T01:37:36.294ZDownstairs last five completed: 1001 1000
71172024-02-12T01:37:36.294ZUpstairs last five completed: 30 29 28 27 26
71182024-02-12T01:37:36.294Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187101157Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391}
71192024-02-12T01:37:36.294ZWrote out file "/tmp/test_repair_verify.data"
71202024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187554745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71212024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187591043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71222024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187627482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71232024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18765295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71242024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187674199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71252024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187696458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71262024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187728947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71272024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187747586Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71282024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187767445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71292024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187789024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71302024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187817193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71312024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187831552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71322024-02-12T01:37:36.294Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187844071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71332024-02-12T01:37:36.294Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18786212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71342024-02-12T01:37:36.294Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187888989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71352024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187902698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71362024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187919318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71372024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187946476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71382024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187959396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71392024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.187976375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71402024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188002384Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71412024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188015603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71422024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188027732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71432024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188039672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71442024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188051681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71452024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188064171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71462024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18807599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71472024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18808801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71482024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188107899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71492024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188120148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71502024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188132287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71512024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188151477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71522024-02-12T01:37:36.295Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188163516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71532024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188175276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71542024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188193935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71552024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188206134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71562024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188218343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71572024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188231813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71582024-02-12T01:37:36.295Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188250642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71592024-02-12T01:37:36.295Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188262761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71602024-02-12T01:37:36.296Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188274951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71612024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18829322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71622024-02-12T01:37:36.296Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188305289Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71632024-02-12T01:37:36.296Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188317359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71642024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188335978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71652024-02-12T01:37:36.296Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188348357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71662024-02-12T01:37:36.296Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188360387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71672024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188372436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71682024-02-12T01:37:36.296Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188384435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71692024-02-12T01:37:36.296Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188403434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71702024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188415554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71712024-02-12T01:37:36.296Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188427653Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71722024-02-12T01:37:36.296Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188446263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71732024-02-12T01:37:36.296Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188458432Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71742024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188470541Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71752024-02-12T01:37:36.296Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18848873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71762024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18850097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71772024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188512909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71782024-02-12T01:37:36.302Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188531219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71792024-02-12T01:37:36.302Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188543448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71802024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188555617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71812024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188567637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71822024-02-12T01:37:36.302Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188585806Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71832024-02-12T01:37:36.302Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188597925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71842024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188609935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71852024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188628454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71862024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188640633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71872024-02-12T01:37:36.302Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188652623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71882024-02-12T01:37:36.302Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188670772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71892024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188682791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71902024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188694811Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71912024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18870677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71922024-02-12T01:37:36.303Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188718819Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71932024-02-12T01:37:36.303Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188737319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71942024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188749478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71952024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188761437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71962024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188779936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71972024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188792086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71982024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188804046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
71992024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188817105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72002024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188835414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72012024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188847353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72022024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188859303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72032024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188871512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72042024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188883602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72052024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188901941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72062024-02-12T01:37:36.303Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18891424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72072024-02-12T01:37:36.303Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18892599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72082024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188938589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72092024-02-12T01:37:36.303Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188950568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72102024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188963168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72112024-02-12T01:37:36.303Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188975117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72122024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.188993477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72132024-02-12T01:37:36.303Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189005656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72142024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189018065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72152024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189030205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72162024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189042234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72172024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189054914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72182024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189066783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72192024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189085162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72202024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189097232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72212024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189115431Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72222024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18912757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72232024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189145639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72242024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189157809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72252024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189175978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72262024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189188117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72272024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189199927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72282024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189218186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72292024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189230245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72302024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189248674Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72312024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189260844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72322024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189279153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72332024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189291322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72342024-02-12T01:37:36.304Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189309511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72352024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189321621Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72362024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.18933984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72372024-02-12T01:37:36.304Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189351979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72382024-02-12T01:37:36.304Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189370168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72392024-02-12T01:37:36.305Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189382218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72402024-02-12T01:37:36.305Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189394807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72412024-02-12T01:37:36.305Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189413406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72422024-02-12T01:37:36.305Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189425546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72432024-02-12T01:37:36.305Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:35.189443705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1391,"session_id":"ab3a4bdf-42f3-4a35-b6d7-b4e6ad6ac3cf"}
72442024-02-12T01:37:36.305Z
72452024-02-12T01:37:36.305ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72462024-02-12T01:37:36.316ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72472024-02-12T01:37:36.316Z 0 000-019 63 63 63 108 108 108 F F F
72482024-02-12T01:37:36.316Z 1 020-039 62 62 62 107 107 107 F F F
72492024-02-12T01:37:36.316Z 2 040-059 64 58 64 111 101 111 F F F <---
72502024-02-12T01:37:36.316Z 3 060-079 58 58 58 101 101 101 T F T <---
72512024-02-12T01:37:36.316Z 4 080-099 64 63 64 110 108 110 T F T <---
72522024-02-12T01:37:36.316Z 5 100-119 59 59 59 102 102 102 T F T <---
72532024-02-12T01:37:36.316Z 6 120-139 64 59 64 111 102 111 F F F <---
72542024-02-12T01:37:36.316Z 7 140-159 64 64 64 111 109 111 F F F <---
72552024-02-12T01:37:36.316Z 8 160-179 64 62 64 110 107 110 F F F <---
72562024-02-12T01:37:36.320Z 9 180-199 64 62 64 110 107 110 F T F <---
72572024-02-12T01:37:36.320Z 10 200-219 64 61 64 111 104 111 F T F <---
72582024-02-12T01:37:36.320Z 11 220-239 64 62 64 110 105 110 T F T <---
72592024-02-12T01:37:36.320Z 12 240-259 59 59 59 102 102 102 T F T <---
72602024-02-12T01:37:36.320Z 13 260-279 60 60 60 103 103 103 F F F
72612024-02-12T01:37:36.320Z 14 280-299 64 56 64 110 98 110 F T F <---
72622024-02-12T01:37:36.320Z 15 300-319 55 55 55 97 97 97 F F F
72632024-02-12T01:37:36.320Z 16 320-339 64 63 64 111 108 111 F F F <---
72642024-02-12T01:37:36.320Z 17 340-359 52 52 52 85 85 85 F F F
72652024-02-12T01:37:36.320Z 18 360-379 61 61 61 104 104 104 F F F
72662024-02-12T01:37:36.320Z 19 380-399 63 63 63 108 108 108 F F F
72672024-02-12T01:37:36.320Z 20 400-419 63 63 63 108 108 108 F F F
72682024-02-12T01:37:36.320Z 21 420-439 61 61 61 104 104 104 F F F
72692024-02-12T01:37:36.320Z 22 440-459 56 56 56 98 98 98 F F F
72702024-02-12T01:37:36.320Z 23 460-479 56 56 56 98 98 98 F F F
72712024-02-12T01:37:36.320Z 24 480-499 54 54 54 94 94 94 F F F
72722024-02-12T01:37:36.320Z 25 500-519 56 56 56 99 99 99 F F F
72732024-02-12T01:37:36.320Z 26 520-539 61 61 61 104 104 104 F F F
72742024-02-12T01:37:36.320Z 27 540-559 51 51 51 83 83 83 F F F
72752024-02-12T01:37:36.320Z 28 560-579 62 62 62 107 107 107 F F F
72762024-02-12T01:37:36.320Z 29 580-599 62 62 62 107 107 107 F F F
72772024-02-12T01:37:36.321ZMax gen: 64, Max flush: 111
72782024-02-12T01:37:36.321ZError: Difference in extent metadata found!
72792024-02-12T01:37:36.321ZA Difference in extent metadata is expected here
72802024-02-12T01:37:36.321ZOn loop 32
72812024-02-12T01:37:36.321Z
72822024-02-12T01:37:36.325ZVerifying data now
72832024-02-12T01:37:41.945ZNo change after verify
72842024-02-12T01:37:41.945ZLoop: 32 Downstairs dump after verify (and repair):
72852024-02-12T01:37:41.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72862024-02-12T01:37:41.961Z 0 000-019 63 63 63 108 108 108 F F F
72872024-02-12T01:37:41.961Z 1 020-039 62 62 62 107 107 107 F F F
72882024-02-12T01:37:41.961Z 2 040-059 64 64 64 111 111 111 F F F
72892024-02-12T01:37:41.961Z 3 060-079 65 65 65 112 112 112 F F F
72902024-02-12T01:37:41.961Z 4 080-099 65 65 65 112 112 112 F F F
72912024-02-12T01:37:41.961Z 5 100-119 65 65 65 112 112 112 F F F
72922024-02-12T01:37:41.961Z 6 120-139 64 64 64 111 111 111 F F F
72932024-02-12T01:37:41.961Z 7 140-159 64 64 64 111 111 111 F F F
72942024-02-12T01:37:41.961Z 8 160-179 64 64 64 110 110 110 F F F
72952024-02-12T01:37:41.961Z 9 180-199 64 64 64 110 110 110 F F F
72962024-02-12T01:37:41.961Z 10 200-219 64 64 64 111 111 111 F F F
72972024-02-12T01:37:41.961Z 11 220-239 65 65 65 112 112 112 F F F
72982024-02-12T01:37:41.961Z 12 240-259 65 65 65 112 112 112 F F F
72992024-02-12T01:37:41.961Z 13 260-279 60 60 60 103 103 103 F F F
73002024-02-12T01:37:41.961Z 14 280-299 64 64 64 110 110 110 F F F
73012024-02-12T01:37:41.964Z 15 300-319 55 55 55 97 97 97 F F F
73022024-02-12T01:37:41.964Z 16 320-339 64 64 64 111 111 111 F F F
73032024-02-12T01:37:41.964Z 17 340-359 52 52 52 85 85 85 F F F
73042024-02-12T01:37:41.964Z 18 360-379 61 61 61 104 104 104 F F F
73052024-02-12T01:37:41.964Z 19 380-399 63 63 63 108 108 108 F F F
73062024-02-12T01:37:41.964Z 20 400-419 63 63 63 108 108 108 F F F
73072024-02-12T01:37:41.964Z 21 420-439 61 61 61 104 104 104 F F F
73082024-02-12T01:37:41.964Z 22 440-459 56 56 56 98 98 98 F F F
73092024-02-12T01:37:41.964Z 23 460-479 56 56 56 98 98 98 F F F
73102024-02-12T01:37:41.964Z 24 480-499 54 54 54 94 94 94 F F F
73112024-02-12T01:37:41.964Z 25 500-519 56 56 56 99 99 99 F F F
73122024-02-12T01:37:41.965Z 26 520-539 61 61 61 104 104 104 F F F
73132024-02-12T01:37:41.965Z 27 540-559 51 51 51 83 83 83 F F F
73142024-02-12T01:37:41.965Z 28 560-579 62 62 62 107 107 107 F F F
73152024-02-12T01:37:41.965Z 29 580-599 62 62 62 107 107 107 F F F
73162024-02-12T01:37:41.965ZMax gen: 65, Max flush: 112
73172024-02-12T01:37:41.965Z
73182024-02-12T01:37:41.965ZBegin loop 33
73192024-02-12T01:37:41.965ZDownstairs to restart: 1
73202024-02-12T01:37:41.982Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880516539Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
73212024-02-12T01:37:41.982Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
73222024-02-12T01:37:41.982Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880812105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
73232024-02-12T01:37:41.987Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880855863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
73242024-02-12T01:37:41.987Z{"msg":"Crucible 01b7e015-bbf6-4d68-ba55-b072cb3d5e24 has session id: c7460843-e995-464e-962c-3fc4b3c72cf8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880909671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73252024-02-12T01:37:41.987Z{"msg":"Upstairs opts: Upstairs UUID: 01b7e015-bbf6-4d68-ba55-b072cb3d5e24, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880954479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73262024-02-12T01:37:41.987Z{"msg":"Crucible stats registered with UUID: 01b7e015-bbf6-4d68-ba55-b072cb3d5e24","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.880973487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73272024-02-12T01:37:41.987Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881002016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
73282024-02-12T01:37:41.987Z{"msg":"01b7e015-bbf6-4d68-ba55-b072cb3d5e24 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881026615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73292024-02-12T01:37:41.987Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881047764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73302024-02-12T01:37:41.987Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881098242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73312024-02-12T01:37:41.987Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.8811254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73322024-02-12T01:37:41.987Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881152039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73332024-02-12T01:37:41.987Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881176868Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73342024-02-12T01:37:41.987Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881217576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73352024-02-12T01:37:41.987Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881250564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73362024-02-12T01:37:41.988Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881284953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73372024-02-12T01:37:41.988Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881327601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73382024-02-12T01:37:41.988Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881356469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73392024-02-12T01:37:41.988Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881383148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73402024-02-12T01:37:41.988Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.881407217Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73412024-02-12T01:37:41.988Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882058226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73422024-02-12T01:37:41.988Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882093464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73432024-02-12T01:37:41.988Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882234247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73442024-02-12T01:37:41.988Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882271386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73452024-02-12T01:37:41.988Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882642448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73462024-02-12T01:37:41.988Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.882787661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73472024-02-12T01:37:41.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883097336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73482024-02-12T01:37:41.988Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883139764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73492024-02-12T01:37:41.988Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883164633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73502024-02-12T01:37:41.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883292577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73512024-02-12T01:37:41.988Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883329445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73522024-02-12T01:37:41.988Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883365064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73532024-02-12T01:37:41.988Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.88343406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73542024-02-12T01:37:41.988Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883469859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73552024-02-12T01:37:41.988Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.883923787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73562024-02-12T01:37:42.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884353506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73572024-02-12T01:37:42.005Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884397434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73582024-02-12T01:37:42.005Z{"msg":"[0]R flush_numbers[0..12]: [108, 107, 111, 112, 112, 112, 111, 111, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884423413Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73592024-02-12T01:37:42.005Z{"msg":"[0]R generation[0..12]: [63, 62, 64, 65, 65, 65, 64, 64, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884438962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73602024-02-12T01:37:42.005Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884461351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73612024-02-12T01:37:42.005Z{"msg":"[1]R flush_numbers[0..12]: [108, 107, 111, 112, 112, 112, 111, 111, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884476011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73622024-02-12T01:37:42.005Z{"msg":"[1]R generation[0..12]: [63, 62, 64, 65, 65, 65, 64, 64, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884496399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73632024-02-12T01:37:42.005Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
73642024-02-12T01:37:42.005Z,"time":"2024-02-12T01:37:40.884510949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73652024-02-12T01:37:42.005Z{"msg":"[2]R flush_numbers[0..12]: [108, 107, 111, 112, 112, 112, 111, 111, 110, 110, 111, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884527618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73662024-02-12T01:37:42.005Z{"msg":"[2]R generation[0..12]: [63, 62, 64, 65, 65, 65, 64, 64, 64, 64, 64, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884548617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73672024-02-12T01:37:42.005Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884563166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73682024-02-12T01:37:42.005Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884583665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73692024-02-12T01:37:42.005Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884597485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73702024-02-12T01:37:42.005Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884610324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73712024-02-12T01:37:42.005Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884633133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73722024-02-12T01:37:42.006Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884649992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73732024-02-12T01:37:42.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884673521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73742024-02-12T01:37:42.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.88468831Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73752024-02-12T01:37:42.006Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884708599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73762024-02-12T01:37:42.006Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884722129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73772024-02-12T01:37:42.006Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884741098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73782024-02-12T01:37:42.006Z{"msg":"01b7e015-bbf6-4d68-ba55-b072cb3d5e24 is now active with session: c7460843-e995-464e-962c-3fc4b3c72cf8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884754077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73792024-02-12T01:37:42.006Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884766477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399,"session_id":"c7460843-e995-464e-962c-3fc4b3c72cf8"}
73802024-02-12T01:37:42.006Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:40.884779206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
73812024-02-12T01:37:42.006ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73822024-02-12T01:37:42.006ZLoading write count information from file "/tmp/test_repair_verify.data"
73832024-02-12T01:37:42.006ZRun Repair workload
73842024-02-12T01:37:42.006Z01/30 Flush
73852024-02-12T01:37:42.006Z02/30 Write block 326 len 24576 data: 5 4 4 4 4 4
73862024-02-12T01:37:42.006Z03/30 Write block 67 len 4096 data: 6
73872024-02-12T01:37:42.006Z04/30 Write block 566 len 36864 data: 5 6 8 8 8 7 7 10 9
73882024-02-12T01:37:42.006Z05/30 Write block 553 len 4096 data: 7
73892024-02-12T01:37:42.006Z06/30 Write block 323 len 4096 data: 5
73902024-02-12T01:37:42.006Z07/30 Read block 232 len 24576
73912024-02-12T01:37:42.006Z08/30 Read block 209 len 24576
73922024-02-12T01:37:42.006Z09/30 Write block 279 len 16384 data: 8 8 7 9
73932024-02-12T01:37:42.006Z10/30 Read block 507 len 4096
73942024-02-12T01:37:42.006Z11/30 Flush
73952024-02-12T01:37:42.006Z12/30 Write block 537 len 16384 data: 9 9 8 7
73962024-02-12T01:37:42.006Z13/30 Write block 136 len 28672 data: 5 6 7 6 6 8 9
73972024-02-12T01:37:42.007Z14/30 Flush
73982024-02-12T01:37:42.011Z15/30 Write block 210 len 40960 data: 4 5 4 4 6 5 9 6 7 8
73992024-02-12T01:37:42.011Z16/30 Read block 236 len 20480
74002024-02-12T01:37:42.011Z17/30 Read block 312 len 28672
74012024-02-12T01:37:42.011Z18/30 Read block 350 len 32768
74022024-02-12T01:37:42.011Z19/30 Write block 444 len 32768 data: 4 5 4 5 5 5 5 5
74032024-02-12T01:37:42.011Z20/30 Write block 569 len 32768 data: 9 9 8 8 11 10 9 6
74042024-02-12T01:37:42.011Z21/30 Read block 595 len 8192
74052024-02-12T01:37:42.011Z22/30 Read block 246 len 4096
74062024-02-12T01:37:42.011Z23/30 Write block 118 len 4096 data: 10
74072024-02-12T01:37:42.011Z24/30 Flush
74082024-02-12T01:37:42.014Z25/30 Write block 486 len 4096 data: 8
74092024-02-12T01:37:42.014Z26/30 Write block 80 len 36864 data: 10 10 11 8 9 9 8 9 8
74102024-02-12T01:37:42.014Z27/30 Write block 595 len 16384 data: 4 5 4 3
74112024-02-12T01:37:42.014Z28/30 Write block 413 len 28672 data: 7 5 6 6 6 4 3
74122024-02-12T01:37:42.014Z29/30 Read block 114 len 16384
74132024-02-12T01:37:42.017Z30/30 Read block 79 len 8192
74142024-02-12T01:37:42.018Z----------------------------------------------------------------
74152024-02-12T01:37:42.018Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:29
74162024-02-12T01:37:42.018ZStates: Active Active Active
74172024-02-12T01:37:42.018ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74182024-02-12T01:37:42.018Z 2 Acked 1001 Write 6 Done Done Done false
74192024-02-12T01:37:42.018Z 3 Acked 1002 Write 1 Done Done Done false
74202024-02-12T01:37:42.018Z 4 Acked 1003 Write 9 Done Done Done false
74212024-02-12T01:37:42.018Z 5 Acked 1004 Write 1 Done Sent Done false
74222024-02-12T01:37:42.018Z 6 Acked 1005 Write 1 Done Sent Done false
74232024-02-12T01:37:42.018Z 7 Acked 1006 Read 6 Done Sent Done false
74242024-02-12T01:37:42.018Z 8 Acked 1007 Read 6 Done Sent Done false
74252024-02-12T01:37:42.018Z 9 Acked 1008 Write 4 Done Sent Done false
74262024-02-12T01:37:42.018Z 10 Acked 1009 Read 1 Done Sent Done false
74272024-02-12T01:37:42.018Z 11 Acked 1010 Flush 0 Done Sent Done false
74282024-02-12T01:37:42.018Z 12 Acked 1011 Write 4 Done Sent Done false
74292024-02-12T01:37:42.018Z 13 Acked 1012 Write 7 Done Sent Done false
74302024-02-12T01:37:42.018Z 14 Acked 1013 Flush 0 Done Sent Done false
74312024-02-12T01:37:42.018Z 15 Acked 1014 Write 10 Done Sent Done false
74322024-02-12T01:37:42.018Z 16 Acked 1015 Read 5 Done Sent Done false
74332024-02-12T01:37:42.018Z 17 Acked 1016 Read 7 Done Sent Done false
74342024-02-12T01:37:42.018Z 18 Acked 1017 Read 8 Done Sent Done false
74352024-02-12T01:37:42.018Z 19 Acked 1018 Write 8 Done Sent Done false
74362024-02-12T01:37:42.018Z 20 Acked 1019 Write 8 Done Sent Done false
74372024-02-12T01:37:42.018Z 21 Acked 1020 Read 2 Done Sent Done false
74382024-02-12T01:37:42.019Z 22 Acked 1021 Read 1 Done Sent Done false
74392024-02-12T01:37:42.019Z 23 Acked 1022 Write 1 Done Sent Done false
74402024-02-12T01:37:42.019Z 24 Acked 1023 Flush 0 Done Sent Done false
74412024-02-12T01:37:42.019Z 25 Acked 1024 Write 1 Done Sent Done false
74422024-02-12T01:37:42.019Z 26 Acked 1025 Write 9 Done Sent Done false
74432024-02-12T01:37:42.019Z 27 Acked 1026 Write 4 Done Sent Done false
74442024-02-12T01:37:42.019Z 28 Acked 1027 Write 7 Done Sent Done false
74452024-02-12T01:37:42.019Z 29 Acked 1028 Read 4 Done Sent Done false
74462024-02-12T01:37:42.019Z 30 Acked 1029 Read 2 Done Sent Sent false
74472024-02-12T01:37:42.019Z STATES DS:0 DS:1 DS:2 TOTAL
74482024-02-12T01:37:42.019Z New 0 0 0 0
74492024-02-12T01:37:42.019Z Sent 0 26 1 27
74502024-02-12T01:37:42.019Z Done 29 3 28 60
74512024-02-12T01:37:42.019Z Skipped 0 0 0 0
74522024-02-12T01:37:42.019Z Error 0 0 0 0
74532024-02-12T01:37:42.019ZLast Flush: 1023 1000 1023
74542024-02-12T01:37:42.019ZDownstairs last five completed: 1000
74552024-02-12T01:37:42.019ZUpstairs last five completed: 30 29 28 27 26
74562024-02-12T01:37:42.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:40.915452093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1399}
74572024-02-12T01:37:42.019ZWrote out file "/tmp/test_repair_verify.data"
74582024-02-12T01:37:42.022Z
74592024-02-12T01:37:42.025ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74602024-02-12T01:37:42.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74612024-02-12T01:37:42.043Z 0 000-019 63 63 63 108 108 108 F F F
74622024-02-12T01:37:42.043Z 1 020-039 62 62 62 107 107 107 F F F
74632024-02-12T01:37:42.043Z 2 040-059 64 64 64 111 111 111 F F F
74642024-02-12T01:37:42.043Z 3 060-079 66 65 66 114 112 114 F T F <---
74652024-02-12T01:37:42.043Z 4 080-099 65 65 65 112 112 112 T F T <---
74662024-02-12T01:37:42.043Z 5 100-119 66 65 66 116 112 116 F F F <---
74672024-02-12T01:37:42.043Z 6 120-139 66 64 66 115 111 115 F F F <---
74682024-02-12T01:37:42.043Z 7 140-159 66 64 66 115 111 115 F F F <---
74692024-02-12T01:37:42.043Z 8 160-179 64 64 64 110 110 110 F F F
74702024-02-12T01:37:42.043Z 9 180-199 64 64 64 110 110 110 F F F
74712024-02-12T01:37:42.046Z 10 200-219 66 64 66 116 111 116 F F F <---
74722024-02-12T01:37:42.046Z 11 220-239 65 65 65 112 112 112 F F F
74732024-02-12T01:37:42.046Z 12 240-259 65 65 65 112 112 112 F F F
74742024-02-12T01:37:42.046Z 13 260-279 66 60 66 114 103 114 F F F <---
74752024-02-12T01:37:42.046Z 14 280-299 66 64 66 114 110 114 F F F <---
74762024-02-12T01:37:42.046Z 15 300-319 55 55 55 97 97 97 F F F
74772024-02-12T01:37:42.046Z 16 320-339 66 64 66 114 111 114 F T F <---
74782024-02-12T01:37:42.047Z 17 340-359 52 52 52 85 85 85 F F F
74792024-02-12T01:37:42.047Z 18 360-379 61 61 61 104 104 104 F F F
74802024-02-12T01:37:42.047Z 19 380-399 63 63 63 108 108 108 F F F
74812024-02-12T01:37:42.047Z 20 400-419 63 63 63 108 108 108 T F T <---
74822024-02-12T01:37:42.047Z 21 420-439 61 61 61 104 104 104 F F F
74832024-02-12T01:37:42.047Z 22 440-459 66 56 66 116 98 116 F F F <---
74842024-02-12T01:37:42.047Z 23 460-479 56 56 56 98 98 98 F F F
74852024-02-12T01:37:42.047Z 24 480-499 54 54 54 94 94 94 T F T <---
74862024-02-12T01:37:42.047Z 25 500-519 56 56 56 99 99 99 F F F
74872024-02-12T01:37:42.047Z 26 520-539 66 61 66 115 104 115 F F F <---
74882024-02-12T01:37:42.047Z 27 540-559 66 51 66 115 83 115 F F F <---
74892024-02-12T01:37:42.047Z 28 560-579 66 62 66 116 107 116 F T F <---
74902024-02-12T01:37:42.047Z 29 580-599 62 62 62 107 107 107 T F T <---
74912024-02-12T01:37:42.047ZMax gen: 66, Max flush: 116
74922024-02-12T01:37:42.047ZError: Difference in extent metadata found!
74932024-02-12T01:37:42.047ZA Difference in extent metadata is expected here
74942024-02-12T01:37:42.047ZOn loop 33
74952024-02-12T01:37:42.047Z
74962024-02-12T01:37:42.050ZVerifying data now
74972024-02-12T01:37:47.778ZNo change after verify
74982024-02-12T01:37:47.778ZLoop: 33 Downstairs dump after verify (and repair):
74992024-02-12T01:37:47.793ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75002024-02-12T01:37:47.793Z 0 000-019 63 63 63 108 108 108 F F F
75012024-02-12T01:37:47.793Z 1 020-039 62 62 62 107 107 107 F F F
75022024-02-12T01:37:47.793Z 2 040-059 64 64 64 111 111 111 F F F
75032024-02-12T01:37:47.793Z 3 060-079 66 66 66 114 114 114 F F F
75042024-02-12T01:37:47.793Z 4 080-099 67 67 67 117 117 117 F F F
75052024-02-12T01:37:47.793Z 5 100-119 66 66 66 116 116 116 F F F
75062024-02-12T01:37:47.793Z 6 120-139 66 66 66 115 115 115 F F F
75072024-02-12T01:37:47.793Z 7 140-159 66 66 66 115 115 115 F F F
75082024-02-12T01:37:47.793Z 8 160-179 64 64 64 110 110 110 F F F
75092024-02-12T01:37:47.793Z 9 180-199 64 64 64 110 110 110 F F F
75102024-02-12T01:37:47.793Z 10 200-219 66 66 66 116 116 116 F F F
75112024-02-12T01:37:47.793Z 11 220-239 65 65 65 112 112 112 F F F
75122024-02-12T01:37:47.793Z 12 240-259 65 65 65 112 112 112 F F F
75132024-02-12T01:37:47.793Z 13 260-279 66 66 66 114 114 114 F F F
75142024-02-12T01:37:47.794Z 14 280-299 66 66 66 114 114 114 F F F
75152024-02-12T01:37:47.794Z 15 300-319 55 55 55 97 97 97 F F F
75162024-02-12T01:37:47.797Z 16 320-339 66 66 66 114 114 114 F F F
75172024-02-12T01:37:47.797Z 17 340-359 52 52 52 85 85 85 F F F
75182024-02-12T01:37:47.797Z 18 360-379 61 61 61 104 104 104 F F F
75192024-02-12T01:37:47.797Z 19 380-399 63 63 63 108 108 108 F F F
75202024-02-12T01:37:47.797Z 20 400-419 67 67 67 117 117 117 F F F
75212024-02-12T01:37:47.797Z 21 420-439 61 61 61 104 104 104 F F F
75222024-02-12T01:37:47.797Z 22 440-459 66 66 66 116 116 116 F F F
75232024-02-12T01:37:47.797Z 23 460-479 56 56 56 98 98 98 F F F
75242024-02-12T01:37:47.797Z 24 480-499 67 67 67 117 117 117 F F F
75252024-02-12T01:37:47.797Z 25 500-519 56 56 56 99 99 99 F F F
75262024-02-12T01:37:47.797Z 26 520-539 66 66 66 115 115 115 F F F
75272024-02-12T01:37:47.797Z 27 540-559 66 66 66 115 115 115 F F F
75282024-02-12T01:37:47.797Z 28 560-579 66 66 66 116 116 116 F F F
75292024-02-12T01:37:47.797Z 29 580-599 67 67 67 117 117 117 F F F
75302024-02-12T01:37:47.797ZMax gen: 67, Max flush: 117
75312024-02-12T01:37:47.797Z
75322024-02-12T01:37:47.797ZBegin loop 34
75332024-02-12T01:37:47.797ZDownstairs to restart: 2
75342024-02-12T01:37:47.814Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.7129258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
75352024-02-12T01:37:47.814Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713200347Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
75362024-02-12T01:37:47.815ZCrucible runtime is spawned
75372024-02-12T01:37:47.820Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713255605Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
75382024-02-12T01:37:47.820Z{"msg":"Crucible 5ac0c760-ef2c-4c1c-a0b9-f65f69b4928a has session id: 21e55f29-d7ac-4b83-881e-9378e1ccdf89","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713300742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75392024-02-12T01:37:47.820Z{"msg":"Upstairs opts: Upstairs UUID: 5ac0c760-ef2c-4c1c-a0b9-f65f69b4928a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71335234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75402024-02-12T01:37:47.820Z{"msg":"Crucible stats registered with UUID: 5ac0c760-ef2c-4c1c-a0b9-f65f69b4928a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713375019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75412024-02-12T01:37:47.820Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713405717Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
75422024-02-12T01:37:47.820Z{"msg":"5ac0c760-ef2c-4c1c-a0b9-f65f69b4928a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713503213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75432024-02-12T01:37:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713540131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75442024-02-12T01:37:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713571669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75452024-02-12T01:37:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713615287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75462024-02-12T01:37:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713649076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75472024-02-12T01:37:47.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713693184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75482024-02-12T01:37:47.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713742681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75492024-02-12T01:37:47.820Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713781589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75502024-02-12T01:37:47.820Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713861376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75512024-02-12T01:37:47.820Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713896994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75522024-02-12T01:37:47.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713935782Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75532024-02-12T01:37:47.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.713988359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75542024-02-12T01:37:47.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.714041237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75552024-02-12T01:37:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.714781302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75562024-02-12T01:37:47.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.714824919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75572024-02-12T01:37:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.714855648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75582024-02-12T01:37:47.821Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.714895156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75592024-02-12T01:37:47.821Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.715345505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75602024-02-12T01:37:47.821Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.715486258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75612024-02-12T01:37:47.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.715899478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75622024-02-12T01:37:47.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.715939716Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75632024-02-12T01:37:47.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.715956225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75642024-02-12T01:37:47.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.716046001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75652024-02-12T01:37:47.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71607327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75662024-02-12T01:37:47.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.716088569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75672024-02-12T01:37:47.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.716109758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75682024-02-12T01:37:47.821Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.716124637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75692024-02-12T01:37:47.826Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.716608545Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75702024-02-12T01:37:47.826Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717028645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75712024-02-12T01:37:47.826Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717074682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75722024-02-12T01:37:47.826Z{"msg":"[0]R flush_numbers[0..12]: [108, 107, 111, 114, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717094621Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75732024-02-12T01:37:47.826Z{"msg":"[0]R generation[0..12]: [63, 62, 64, 66, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71711761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75742024-02-12T01:37:47.826Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71713278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75752024-02-12T01:37:47.826Z{"msg":"[1]R flush_numbers[0..12]: [108, 107, 111, 114, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717154159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75762024-02-12T01:37:47.826Z{"msg":"[1]R generation[0..12]: [63, 62, 64, 66, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717168298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75772024-02-12T01:37:47.826Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717182797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75782024-02-12T01:37:47.826Z{"msg":"[2]R flush_numbers[0..12]: [108, 107, 111, 114, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717204166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75792024-02-12T01:37:47.826Z{"msg":"[2]R generation[0..12]: [63, 62, 64, 66, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717218205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407Wait for a query_work_queue command to finish before sending IO
75802024-02-12T01:37:47.826Z,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75812024-02-12T01:37:47.826Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717235844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75822024-02-12T01:37:47.826Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717249964Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75832024-02-12T01:37:47.826Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717270093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75842024-02-12T01:37:47.827Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717285542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75852024-02-12T01:37:47.827Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717318281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75862024-02-12T01:37:47.827Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717344149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75872024-02-12T01:37:47.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717364888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75882024-02-12T01:37:47.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717397257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75892024-02-12T01:37:47.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717421326Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75902024-02-12T01:37:47.827Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717450064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75912024-02-12T01:37:47.827Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717470163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75922024-02-12T01:37:47.827Z{"msg":"5ac0c760-ef2c-4c1c-a0b9-f65f69b4928a is now active with session: 21e55f29-d7ac-4b83-881e-9378e1ccdf89","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.717495722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75932024-02-12T01:37:47.827Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71752873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407,"session_id":"21e55f29-d7ac-4b83-881e-9378e1ccdf89"}
75942024-02-12T01:37:47.827Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:46.71754387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
75952024-02-12T01:37:47.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75962024-02-12T01:37:47.827ZLoading write count information from file "/tmp/test_repair_verify.data"
75972024-02-12T01:37:47.827ZRun Repair workload
75982024-02-12T01:37:47.827Z01/30 Write block 360 len 28672 data: 4 4 4 5 7 9 7
75992024-02-12T01:37:47.827Z02/30 Flush
76002024-02-12T01:37:47.827Z03/30 Write block 331 len 40960 data: 5 4 4 4 4 4 5 5 4 3
76012024-02-12T01:37:47.827Z04/30 Read block 410 len 16384
76022024-02-12T01:37:47.830Z05/30 Read block 508 len 16384
76032024-02-12T01:37:47.831Z06/30 Read block 540 len 16384
76042024-02-12T01:37:47.831Z07/30 Read block 186 len 12288
76052024-02-12T01:37:47.831Z08/30 Flush
76062024-02-12T01:37:47.831Z09/30 Write block 295 len 8192 data: 5 4
76072024-02-12T01:37:47.833Z10/30 Write block 384 len 40960 data: 4 5 6 7 7 6 9 9 8 7
76082024-02-12T01:37:47.834Z11/30 Read block 491 len 28672
76092024-02-12T01:37:47.834Z12/30 Write block 266 len 12288 data: 6 6 5
76102024-02-12T01:37:47.834Z13/30 Read block 340 len 24576
76112024-02-12T01:37:47.837Z14/30 Read block 492 len 28672
76122024-02-12T01:37:47.837Z15/30 Flush
76132024-02-12T01:37:47.839Z16/30 Write block 67 len 40960 data: 7 6 7 6 8 5 5 5 6 8
76142024-02-12T01:37:47.840Z17/30 Read block 291 len 8192
76152024-02-12T01:37:47.842Z18/30 Write block 57 len 40960 data: 5 9 7 7 6 8 7 7 8 6
76162024-02-12T01:37:47.842Z19/30 Read block 104 len 16384
76172024-02-12T01:37:47.843Z20/30 Write block 3 len 8192 data: 3 4
76182024-02-12T01:37:47.843Z21/30 Read block 499 len 28672
76192024-02-12T01:37:47.845Z22/30 Write block 291 len 20480 data: 6 4 5 4 6
76202024-02-12T01:37:47.846Z23/30 Write block 38 len 24576 data: 7 6 6 6 5 9
76212024-02-12T01:37:47.846Z24/30 Write block 410 len 28672 data: 6 7 6 8 6 7 7
76222024-02-12T01:37:47.846Z25/30 Write block 323 len 24576 data: 6 5 5 6 5 5
76232024-02-12T01:37:47.846Z26/30 Read block 44 len 12288
76242024-02-12T01:37:47.849Z27/30 Read block 358 len 24576
76252024-02-12T01:37:47.849Z28/30 Write block 374 len 36864 data: 7 9 9 8 7 6 5 5 4
76262024-02-12T01:37:47.849Z29/30 Read block 333 len 28672
76272024-02-12T01:37:47.852Z30/30 Read block 434 len 20480
76282024-02-12T01:37:47.852Z----------------------------------------------------------------
76292024-02-12T01:37:47.853Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:28
76302024-02-12T01:37:47.853ZStates: Active Active Active
76312024-02-12T01:37:47.853ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76322024-02-12T01:37:47.853Z 3 Acked 1002 Write 10 Done Done Sent false
76332024-02-12T01:37:47.853Z 4 Acked 1003 Read 4 Done Done Sent false
76342024-02-12T01:37:47.853Z 5 Acked 1004 Read 4 Done Done Sent false
76352024-02-12T01:37:47.853Z 6 Acked 1005 Read 4 Done Done Sent false
76362024-02-12T01:37:47.853Z 7 Acked 1006 Read 3 Done Done Sent false
76372024-02-12T01:37:47.853Z 8 Acked 1007 Flush 0 Done Done Sent false
76382024-02-12T01:37:47.853Z 9 Acked 1008 Write 2 Done Done Sent false
76392024-02-12T01:37:47.853Z 10 Acked 1009 Write 10 Done Done Sent false
76402024-02-12T01:37:47.853Z 11 Acked 1010 Read 7 Done Done Sent false
76412024-02-12T01:37:47.853Z 12 Acked 1011 Write 3 Done Done Sent false
76422024-02-12T01:37:47.853Z 13 Acked 1012 Read 6 Done Done Sent false
76432024-02-12T01:37:47.853Z 14 Acked 1013 Read 7 Done Done Sent false
76442024-02-12T01:37:47.853Z 15 Acked 1014 Flush 0 Done Done Sent false
76452024-02-12T01:37:47.853Z 16 Acked 1015 Write 10 Done Done Sent false
76462024-02-12T01:37:47.853Z 17 Acked 1016 Read 2 Done Done Sent false
76472024-02-12T01:37:47.853Z 18 Acked 1017 Write 10 Done Done Sent false
76482024-02-12T01:37:47.853Z 19 Acked 1018 Read 4 Done Done Sent false
76492024-02-12T01:37:47.853Z 20 Acked 1019 Write 2 Done Done Sent false
76502024-02-12T01:37:47.853Z 21 Acked 1020 Read 7 Done Done Sent false
76512024-02-12T01:37:47.853Z 22 Acked 1021 Write 5 Done Done Sent false
76522024-02-12T01:37:47.854Z 23 Acked 1022 Write 6 Done Done Sent false
76532024-02-12T01:37:47.854Z 24 Acked 1023 Write 7 Done Done Sent false
76542024-02-12T01:37:47.854Z 25 Acked 1024 Write 6 Done Done Sent false
76552024-02-12T01:37:47.854Z 26 Acked 1025 Read 3 Done Done Sent false
76562024-02-12T01:37:47.854Z 27 Acked 1026 Read 6 Done Done Sent false
76572024-02-12T01:37:47.854Z 28 Acked 1027 Write 9 Done Done Sent false
76582024-02-12T01:37:47.854Z 29 Acked 1028 Read 7 Done Done Sent false
76592024-02-12T01:37:47.854Z 30 Acked 1029 Read 5 Done Done Sent false
76602024-02-12T01:37:47.854Z STATES DS:0 DS:1 DS:2 TOTAL
76612024-02-12T01:37:47.854Z New 0 0 0 0
76622024-02-12T01:37:47.854Z Sent 0 0 28 28
76632024-02-12T01:37:47.854Z Done 28 28 0 56
76642024-02-12T01:37:47.854Z Skipped 0 0 0 0
76652024-02-12T01:37:47.854Z Error 0 0 0 0
76662024-02-12T01:37:47.854ZLast Flush: 1014 1014 1001
76672024-02-12T01:37:47.854ZDownstairs last five completed: 1001 1000
76682024-02-12T01:37:47.854ZUpstairs last five completed: 30 29 28 27 26
76692024-02-12T01:37:47.854Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:46.749622789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1407}
76702024-02-12T01:37:47.854ZWrote out file "/tmp/test_repair_verify.data"
76712024-02-12T01:37:47.857Z
76722024-02-12T01:37:47.860ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76732024-02-12T01:37:47.877ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76742024-02-12T01:37:47.878Z 0 000-019 63 63 63 108 108 108 T T F <---
76752024-02-12T01:37:47.878Z 1 020-039 62 62 62 107 107 107 T T F <---
76762024-02-12T01:37:47.878Z 2 040-059 64 64 64 111 111 111 T T F <---
76772024-02-12T01:37:47.878Z 3 060-079 66 66 66 114 114 114 T T F <---
76782024-02-12T01:37:47.878Z 4 080-099 67 67 67 117 117 117 F F F
76792024-02-12T01:37:47.878Z 5 100-119 66 66 66 116 116 116 F F F
76802024-02-12T01:37:47.878Z 6 120-139 66 66 66 115 115 115 F F F
76812024-02-12T01:37:47.878Z 7 140-159 66 66 66 115 115 115 F F F
76822024-02-12T01:37:47.878Z 8 160-179 64 64 64 110 110 110 F F F
76832024-02-12T01:37:47.878Z 9 180-199 64 64 64 110 110 110 F F F
76842024-02-12T01:37:47.881Z 10 200-219 66 66 66 116 116 116 F F F
76852024-02-12T01:37:47.881Z 11 220-239 65 65 65 112 112 112 F F F
76862024-02-12T01:37:47.881Z 12 240-259 65 65 65 112 112 112 F F F
76872024-02-12T01:37:47.881Z 13 260-279 68 68 66 120 120 114 F F F <---
76882024-02-12T01:37:47.881Z 14 280-299 68 68 66 120 120 114 T T F <---
76892024-02-12T01:37:47.881Z 15 300-319 55 55 55 97 97 97 F F F
76902024-02-12T01:37:47.881Z 16 320-339 68 68 66 119 119 114 T T F <---
76912024-02-12T01:37:47.881Z 17 340-359 68 68 52 119 119 85 F F F <---
76922024-02-12T01:37:47.881Z 18 360-379 68 68 68 118 118 118 T T F <---
76932024-02-12T01:37:47.882Z 19 380-399 68 68 63 120 120 108 T T F <---
76942024-02-12T01:37:47.882Z 20 400-419 67 67 67 117 117 117 T T F <---
76952024-02-12T01:37:47.882Z 21 420-439 61 61 61 104 104 104 F F F
76962024-02-12T01:37:47.882Z 22 440-459 66 66 66 116 116 116 F F F
76972024-02-12T01:37:47.882Z 23 460-479 56 56 56 98 98 98 F F F
76982024-02-12T01:37:47.882Z 24 480-499 67 67 67 117 117 117 F F F
76992024-02-12T01:37:47.882Z 25 500-519 56 56 56 99 99 99 F F F
77002024-02-12T01:37:47.882Z 26 520-539 66 66 66 115 115 115 F F F
77012024-02-12T01:37:47.882Z 27 540-559 66 66 66 115 115 115 F F F
77022024-02-12T01:37:47.882Z 28 560-579 66 66 66 116 116 116 F F F
77032024-02-12T01:37:47.882Z 29 580-599 67 67 67 117 117 117 F F F
77042024-02-12T01:37:47.882ZMax gen: 68, Max flush: 120
77052024-02-12T01:37:47.882ZError: Difference in extent metadata found!
77062024-02-12T01:37:47.882ZA Difference in extent metadata is expected here
77072024-02-12T01:37:47.882ZOn loop 34
77082024-02-12T01:37:47.882Z
77092024-02-12T01:37:47.885ZVerifying data now
77102024-02-12T01:37:52.438ZNo change after verify
77112024-02-12T01:37:52.438ZLoop: 34 Downstairs dump after verify (and repair):
77122024-02-12T01:37:52.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77132024-02-12T01:37:52.454Z 0 000-019 69 69 69 121 121 121 F F F
77142024-02-12T01:37:52.454Z 1 020-039 69 69 69 121 121 121 F F F
77152024-02-12T01:37:52.454Z 2 040-059 69 69 69 121 121 121 F F F
77162024-02-12T01:37:52.454Z 3 060-079 69 69 69 121 121 121 F F F
77172024-02-12T01:37:52.454Z 4 080-099 67 67 67 117 117 117 F F F
77182024-02-12T01:37:52.454Z 5 100-119 66 66 66 116 116 116 F F F
77192024-02-12T01:37:52.457Z 6 120-139 66 66 66 115 115 115 F F F
77202024-02-12T01:37:52.457Z 7 140-159 66 66 66 115 115 115 F F F
77212024-02-12T01:37:52.457Z 8 160-179 64 64 64 110 110 110 F F F
77222024-02-12T01:37:52.457Z 9 180-199 64 64 64 110 110 110 F F F
77232024-02-12T01:37:52.457Z 10 200-219 66 66 66 116 116 116 F F F
77242024-02-12T01:37:52.457Z 11 220-239 65 65 65 112 112 112 F F F
77252024-02-12T01:37:52.457Z 12 240-259 65 65 65 112 112 112 F F F
77262024-02-12T01:37:52.457Z 13 260-279 68 68 68 120 120 120 F F F
77272024-02-12T01:37:52.457Z 14 280-299 69 69 69 121 121 121 F F F
77282024-02-12T01:37:52.457Z 15 300-319 55 55 55 97 97 97 F F F
77292024-02-12T01:37:52.457Z 16 320-339 69 69 69 121 121 121 F F F
77302024-02-12T01:37:52.458Z 17 340-359 68 68 68 119 119 119 F F F
77312024-02-12T01:37:52.458Z 18 360-379 69 69 69 121 121 121 F F F
77322024-02-12T01:37:52.458Z 19 380-399 69 69 69 121 121 121 F F F
77332024-02-12T01:37:52.458Z 20 400-419 69 69 69 121 121 121 F F F
77342024-02-12T01:37:52.458Z 21 420-439 61 61 61 104 104 104 F F F
77352024-02-12T01:37:52.458Z 22 440-459 66 66 66 116 116 116 F F F
77362024-02-12T01:37:52.458Z 23 460-479 56 56 56 98 98 98 F F F
77372024-02-12T01:37:52.458Z 24 480-499 67 67 67 117 117 117 F F F
77382024-02-12T01:37:52.458Z 25 500-519 56 56 56 99 99 99 F F F
77392024-02-12T01:37:52.458Z 26 520-539 66 66 66 115 115 115 F F F
77402024-02-12T01:37:52.458Z 27 540-559 66 66 66 115 115 115 F F F
77412024-02-12T01:37:52.458Z 28 560-579 66 66 66 116 116 116 F F F
77422024-02-12T01:37:52.458Z 29 580-599 67 67 67 117 117 117 F F F
77432024-02-12T01:37:52.458ZMax gen: 69, Max flush: 121
77442024-02-12T01:37:52.461Z
77452024-02-12T01:37:52.461ZBegin loop 35
77462024-02-12T01:37:52.461ZDownstairs to restart: 0
77472024-02-12T01:37:52.477Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.374823836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
77482024-02-12T01:37:52.477Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375047815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
77492024-02-12T01:37:52.477Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375068464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
77502024-02-12T01:37:52.482Z{"msg":"Crucible 1e089291-9b24-44d4-89c9-1b4c5d4f690a has session id: 2fce122e-1cfb-4e7a-8d20-4309396bf31a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375101862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"Crucible runtime is spawned
77512024-02-12T01:37:52.482Z}
77522024-02-12T01:37:52.482Z{"msg":"Upstairs opts: Upstairs UUID: 1e089291-9b24-44d4-89c9-1b4c5d4f690a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.37514807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77532024-02-12T01:37:52.482Z{"msg":"Crucible stats registered with UUID: 1e089291-9b24-44d4-89c9-1b4c5d4f690a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375171379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77542024-02-12T01:37:52.482Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375204428Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
77552024-02-12T01:37:52.482Z{"msg":"1e089291-9b24-44d4-89c9-1b4c5d4f690a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.37536504Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77562024-02-12T01:37:52.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375396568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77572024-02-12T01:37:52.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375414818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77582024-02-12T01:37:52.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375457996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77592024-02-12T01:37:52.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375486704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77602024-02-12T01:37:52.482Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375521802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77612024-02-12T01:37:52.482Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375547981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77622024-02-12T01:37:52.482Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375588489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77632024-02-12T01:37:52.482Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375621088Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77642024-02-12T01:37:52.482Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375647756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77652024-02-12T01:37:52.482Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:51.375677345Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77662024-02-12T01:37:52.482Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:51.375709353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77672024-02-12T01:37:52.482Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:51.375734612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77682024-02-12T01:37:52.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375756981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77692024-02-12T01:37:52.482Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.37577643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77702024-02-12T01:37:52.482Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375810339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77712024-02-12T01:37:52.482Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375836487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77722024-02-12T01:37:52.483Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.375947822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77732024-02-12T01:37:52.483Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.376373612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77742024-02-12T01:37:52.483Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.37641941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77752024-02-12T01:37:52.483Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.376517285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77762024-02-12T01:37:52.483Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.37661872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77772024-02-12T01:37:52.483Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.376973493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77782024-02-12T01:37:52.483Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377032221Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77792024-02-12T01:37:52.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377649071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77802024-02-12T01:37:52.483Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377696939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77812024-02-12T01:37:52.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377733657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77822024-02-12T01:37:52.483Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377749766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77832024-02-12T01:37:52.483Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377771205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77842024-02-12T01:37:52.483Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:51.377785025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77852024-02-12T01:37:53.478Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.376392642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77862024-02-12T01:37:53.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.376483327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77872024-02-12T01:37:53.484Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.377358126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77882024-02-12T01:37:53.484Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.377402744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77892024-02-12T01:37:53.484Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.377880881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77902024-02-12T01:37:53.484Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378408306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77912024-02-12T01:37:53.484Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378449734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77922024-02-12T01:37:53.484Z{"msg":"[0]R flush_numbers[0..12]: [121, 121, 121, 121, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378477062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77932024-02-12T01:37:53.484Z{"msg":"[0]R generation[0..12]: [69, 69, 69, 69, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378512161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77942024-02-12T01:37:53.484Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378538269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77952024-02-12T01:37:53.484Z{"msg":"[1]R flush_numbers[0..12]: [121, 121, 121, 121, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378568118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77962024-02-12T01:37:53.484Z{"msg":"[1]R generation[0..12]: [69, 69, 69, 69, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378593627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77972024-02-12T01:37:53.484ZWait for a query_work_queue command to finish before sending IO
77982024-02-12T01:37:53.484Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378626995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
77992024-02-12T01:37:53.485Z{"msg":"[2]R flush_numbers[0..12]: [121, 121, 121, 121, 117, 116, 115, 115, 110, 110, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378654794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78002024-02-12T01:37:53.485Z{"msg":"[2]R generation[0..12]: [69, 69, 69, 69, 67, 66, 66, 66, 64, 64, 66, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378678443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78012024-02-12T01:37:53.485Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378701202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78022024-02-12T01:37:53.485Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378723481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78032024-02-12T01:37:53.485Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378755759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78042024-02-12T01:37:53.485Z{"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378779498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78052024-02-12T01:37:53.485Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378803827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78062024-02-12T01:37:53.485Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378827116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78072024-02-12T01:37:53.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378848685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78082024-02-12T01:37:53.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378880033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78092024-02-12T01:37:53.485Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378906092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78102024-02-12T01:37:53.485Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378929921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78112024-02-12T01:37:53.485Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378960749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78122024-02-12T01:37:53.485Z{"msg":"1e089291-9b24-44d4-89c9-1b4c5d4f690a is now active with session: 2fce122e-1cfb-4e7a-8d20-4309396bf31a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.378984578Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78132024-02-12T01:37:53.485ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78142024-02-12T01:37:53.485Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.379025436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415,"session_id":"2fce122e-1cfb-4e7a-8d20-4309396bf31a"}
78152024-02-12T01:37:53.485Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:52.379054465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
78162024-02-12T01:37:53.485ZLoading write count information from file "/tmp/test_repair_verify.data"
78172024-02-12T01:37:53.488ZRun Repair workload
78182024-02-12T01:37:53.489Z01/30 Write block 519 len 24576 data: 5 6 6 7 7 7
78192024-02-12T01:37:53.489Z02/30 Read block 527 len 8192
78202024-02-12T01:37:53.489Z03/30 Read block 277 len 24576
78212024-02-12T01:37:53.489Z04/30 Write block 371 len 12288 data: 6 7 8
78222024-02-12T01:37:53.489Z05/30 Read block 511 len 20480
78232024-02-12T01:37:53.489Z06/30 Write block 173 len 16384 data: 4 4 8 9
78242024-02-12T01:37:53.489Z07/30 Write block 139 len 24576 data: 7 7 9 10 9 9
78252024-02-12T01:37:53.489Z08/30 Read block 484 len 40960
78262024-02-12T01:37:53.492Z09/30 Flush
78272024-02-12T01:37:53.498Z10/30 Write block 466 len 40960 data: 6 8 8 7 6 5 7 6 6 5
78282024-02-12T01:37:53.498Z11/30 Read block 249 len 16384
78292024-02-12T01:37:53.498Z12/30 Flush
78302024-02-12T01:37:53.501Z13/30 Write block 215 len 24576 data: 6 10 7 8 9 7
78312024-02-12T01:37:53.501Z14/30 Write block 89 len 4096 data: 8
78322024-02-12T01:37:53.501Z15/30 Read block 189 len 28672
78332024-02-12T01:37:53.501Z16/30 Write block 128 len 36864 data: 8 7 7 8 8 5 5 4 6
78342024-02-12T01:37:53.501Z17/30 Read block 246 len 12288
78352024-02-12T01:37:53.504Z18/30 Write block 143 len 36864 data: 10 10 10 9 8 7 8 9 9
78362024-02-12T01:37:53.504Z19/30 Read block 205 len 32768
78372024-02-12T01:37:53.504Z20/30 Read block 364 len 36864
78382024-02-12T01:37:53.507Z21/30 Write block 452 len 4096 data: 5
78392024-02-12T01:37:53.507Z22/30 Read block 574 len 20480
78402024-02-12T01:37:53.507Z23/30 Read block 221 len 12288
78412024-02-12T01:37:53.507Z24/30 Flush
78422024-02-12T01:37:53.510Z25/30 Write block 577 len 4096 data: 5
78432024-02-12T01:37:53.510Z26/30 Read block 432 len 16384
78442024-02-12T01:37:53.513Z27/30 Read block 62 len 8192
78452024-02-12T01:37:53.513Z28/30 Write block 494 len 20480 data: 4 7 6 5 6
78462024-02-12T01:37:53.513Z29/30 Read block 262 len 4096
78472024-02-12T01:37:53.513Z30/30 Read block 149 len 24576
78482024-02-12T01:37:53.517Z----------------------------------------------------------------
78492024-02-12T01:37:53.517Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
78502024-02-12T01:37:53.517ZStates: Active Active Active
78512024-02-12T01:37:53.517ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78522024-02-12T01:37:53.517Z 1 Acked 1000 Write 6 Sent Done Done false
78532024-02-12T01:37:53.517Z 2 Acked 1001 Read 2 Sent Done Done false
78542024-02-12T01:37:53.517Z 3 Acked 1002 Read 6 Sent Done Done false
78552024-02-12T01:37:53.517Z 4 Acked 1003 Write 3 Sent Done Done false
78562024-02-12T01:37:53.517Z 5 Acked 1004 Read 5 Sent Done Done false
78572024-02-12T01:37:53.517Z 6 Acked 1005 Write 4 Sent Done Done false
78582024-02-12T01:37:53.517Z 7 Acked 1006 Write 6 Sent Done Done false
78592024-02-12T01:37:53.517Z 8 Acked 1007 Read 10 Sent Done Done false
78602024-02-12T01:37:53.517Z 9 Acked 1008 Flush 0 Sent Done Done false
78612024-02-12T01:37:53.517Z 10 Acked 1009 Write 10 Sent Done Done false
78622024-02-12T01:37:53.517Z 11 Acked 1010 Read 4 Sent Done Done false
78632024-02-12T01:37:53.517Z 12 Acked 1011 Flush 0 Sent Done Done false
78642024-02-12T01:37:53.517Z 13 Acked 1012 Write 6 Sent Done Done false
78652024-02-12T01:37:53.517Z 14 Acked 1013 Write 1 Sent Done Done false
78662024-02-12T01:37:53.517Z 15 Acked 1014 Read 7 Sent Done Done false
78672024-02-12T01:37:53.517Z 16 Acked 1015 Write 9 Sent Done Done false
78682024-02-12T01:37:53.518Z 17 Acked 1016 Read 3 Sent Done Done false
78692024-02-12T01:37:53.518Z 18 Acked 1017 Write 9 Sent Done Done false
78702024-02-12T01:37:53.518Z 19 Acked 1018 Read 8 Sent Done Done false
78712024-02-12T01:37:53.518Z 20 Acked 1019 Read 9 Sent Done Done false
78722024-02-12T01:37:53.518Z 21 Acked 1020 Write 1 Sent Done Done false
78732024-02-12T01:37:53.518Z 22 Acked 1021 Read 5 Sent Done Done false
78742024-02-12T01:37:53.518Z 23 Acked 1022 Read 3 Sent Done Done false
78752024-02-12T01:37:53.518Z 24 Acked 1023 Flush 0 Sent Done Done false
78762024-02-12T01:37:53.518Z 25 Acked 1024 Write 1 Sent Done Done false
78772024-02-12T01:37:53.518Z 26 Acked 1025 Read 4 Sent Done Done false
78782024-02-12T01:37:53.518Z 27 Acked 1026 Read 2 Sent Done Done false
78792024-02-12T01:37:53.518Z 28 Acked 1027 Write 5 Sent Done Done false
78802024-02-12T01:37:53.518Z 29 Acked 1028 Read 1 Sent Done Done false
78812024-02-12T01:37:53.518Z 30 Acked 1029 Read 6 Sent Sent Done false
78822024-02-12T01:37:53.518Z STATES DS:0 DS:1 DS:2 TOTAL
78832024-02-12T01:37:53.518Z New 0 0 0 0
78842024-02-12T01:37:53.518Z Sent 30 1 0 31
78852024-02-12T01:37:53.518Z Done 0 29 30 59
78862024-02-12T01:37:53.518Z Skipped 0 0 0 0
78872024-02-12T01:37:53.518Z Error 0 0 0 0
78882024-02-12T01:37:53.518ZLast Flush: 0 1023 1023
78892024-02-12T01:37:53.518ZDownstairs last five completed:
78902024-02-12T01:37:53.519ZUpstairs last five completed: 30 29 28 27 26
78912024-02-12T01:37:53.519Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:52.411968974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1415}
78922024-02-12T01:37:53.519ZWrote out file "/tmp/test_repair_verify.data"
78932024-02-12T01:37:53.522Z
78942024-02-12T01:37:53.525ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78952024-02-12T01:37:53.538ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78962024-02-12T01:37:53.538Z 0 000-019 69 69 69 121 121 121 F F F
78972024-02-12T01:37:53.538Z 1 020-039 69 69 69 121 121 121 F F F
78982024-02-12T01:37:53.538Z 2 040-059 69 69 69 121 121 121 F F F
78992024-02-12T01:37:53.538Z 3 060-079 69 69 69 121 121 121 F F F
79002024-02-12T01:37:53.538Z 4 080-099 67 70 70 117 124 124 F F F <---
79012024-02-12T01:37:53.538Z 5 100-119 66 66 66 116 116 116 F F F
79022024-02-12T01:37:53.538Z 6 120-139 66 70 70 115 124 124 F F F <---
79032024-02-12T01:37:53.538Z 7 140-159 66 70 70 115 124 124 F F F <---
79042024-02-12T01:37:53.538Z 8 160-179 64 70 70 110 122 122 F F F <---
79052024-02-12T01:37:53.542Z 9 180-199 64 64 64 110 110 110 F F F
79062024-02-12T01:37:53.542Z 10 200-219 66 70 70 116 124 124 F F F <---
79072024-02-12T01:37:53.542Z 11 220-239 65 70 70 112 124 124 F F F <---
79082024-02-12T01:37:53.542Z 12 240-259 65 65 65 112 112 112 F F F
79092024-02-12T01:37:53.542Z 13 260-279 68 68 68 120 120 120 F F F
79102024-02-12T01:37:53.542Z 14 280-299 69 69 69 121 121 121 F F F
79112024-02-12T01:37:53.542Z 15 300-319 55 55 55 97 97 97 F F F
79122024-02-12T01:37:53.542Z 16 320-339 69 69 69 121 121 121 F F F
79132024-02-12T01:37:53.542Z 17 340-359 68 68 68 119 119 119 F F F
79142024-02-12T01:37:53.542Z 18 360-379 69 70 70 121 122 122 F F F <---
79152024-02-12T01:37:53.542Z 19 380-399 69 69 69 121 121 121 F F F
79162024-02-12T01:37:53.542Z 20 400-419 69 69 69 121 121 121 F F F
79172024-02-12T01:37:53.543Z 21 420-439 61 61 61 104 104 104 F F F
79182024-02-12T01:37:53.543Z 22 440-459 66 70 70 116 124 124 F F F <---
79192024-02-12T01:37:53.543Z 23 460-479 56 70 70 98 123 123 F F F <---
79202024-02-12T01:37:53.543Z 24 480-499 67 67 67 117 117 117 F T T <---
79212024-02-12T01:37:53.543Z 25 500-519 56 70 70 99 122 122 F F F <---
79222024-02-12T01:37:53.543Z 26 520-539 66 70 70 115 122 122 F F F <---
79232024-02-12T01:37:53.543Z 27 540-559 66 66 66 115 115 115 F F F
79242024-02-12T01:37:53.543Z 28 560-579 66 66 66 116 116 116 F T T <---
79252024-02-12T01:37:53.543Z 29 580-599 67 67 67 117 117 117 F F F
79262024-02-12T01:37:53.543ZMax gen: 70, Max flush: 124
79272024-02-12T01:37:53.543ZError: Difference in extent metadata found!
79282024-02-12T01:37:53.543ZA Difference in extent metadata is expected here
79292024-02-12T01:37:53.543ZOn loop 35
79302024-02-12T01:37:53.543Z
79312024-02-12T01:37:53.546ZVerifying data now
79322024-02-12T01:37:58.163ZNo change after verify
79332024-02-12T01:37:58.163ZLoop: 35 Downstairs dump after verify (and repair):
79342024-02-12T01:37:58.178ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79352024-02-12T01:37:58.178Z 0 000-019 69 69 69 121 121 121 F F F
79362024-02-12T01:37:58.178Z 1 020-039 69 69 69 121 121 121 F F F
79372024-02-12T01:37:58.178Z 2 040-059 69 69 69 121 121 121 F F F
79382024-02-12T01:37:58.178Z 3 060-079 69 69 69 121 121 121 F F F
79392024-02-12T01:37:58.178Z 4 080-099 70 70 70 124 124 124 F F F
79402024-02-12T01:37:58.178Z 5 100-119 66 66 66 116 116 116 F F F
79412024-02-12T01:37:58.179Z 6 120-139 70 70 70 124 124 124 F F F
79422024-02-12T01:37:58.179Z 7 140-159 70 70 70 124 124 124 F F F
79432024-02-12T01:37:58.179Z 8 160-179 70 70 70 122 122 122 F F F
79442024-02-12T01:37:58.185Z 9 180-199 64 64 64 110 110 110 F F F
79452024-02-12T01:37:58.185Z 10 200-219 70 70 70 124 124 124 F F F
79462024-02-12T01:37:58.185Z 11 220-239 70 70 70 124 124 124 F F F
79472024-02-12T01:37:58.185Z 12 240-259 65 65 65 112 112 112 F F F
79482024-02-12T01:37:58.185Z 13 260-279 68 68 68 120 120 120 F F F
79492024-02-12T01:37:58.185Z 14 280-299 69 69 69 121 121 121 F F F
79502024-02-12T01:37:58.185Z 15 300-319 55 55 55 97 97 97 F F F
79512024-02-12T01:37:58.185Z 16 320-339 69 69 69 121 121 121 F F F
79522024-02-12T01:37:58.185Z 17 340-359 68 68 68 119 119 119 F F F
79532024-02-12T01:37:58.185Z 18 360-379 70 70 70 122 122 122 F F F
79542024-02-12T01:37:58.185Z 19 380-399 69 69 69 121 121 121 F F F
79552024-02-12T01:37:58.185Z 20 400-419 69 69 69 121 121 121 F F F
79562024-02-12T01:37:58.185Z 21 420-439 61 61 61 104 104 104 F F F
79572024-02-12T01:37:58.185Z 22 440-459 70 70 70 124 124 124 F F F
79582024-02-12T01:37:58.185Z 23 460-479 70 70 70 123 123 123 F F F
79592024-02-12T01:37:58.185Z 24 480-499 71 71 71 125 125 125 F F F
79602024-02-12T01:37:58.185Z 25 500-519 70 70 70 122 122 122 F F F
79612024-02-12T01:37:58.185Z 26 520-539 70 70 70 122 122 122 F F F
79622024-02-12T01:37:58.186Z 27 540-559 66 66 66 115 115 115 F F F
79632024-02-12T01:37:58.186Z 28 560-579 71 71 71 125 125 125 F F F
79642024-02-12T01:37:58.186Z 29 580-599 67 67 67 117 117 117 F F F
79652024-02-12T01:37:58.186ZMax gen: 71, Max flush: 125
79662024-02-12T01:37:58.186Z
79672024-02-12T01:37:58.186ZBegin loop 36
79682024-02-12T01:37:58.186ZDownstairs to restart: 1
79692024-02-12T01:37:58.201Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097217944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
79702024-02-12T01:37:58.201Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"Crucible runtime is spawned
79712024-02-12T01:37:58.201Zstable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097540048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
79722024-02-12T01:37:58.201Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097577787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
79732024-02-12T01:37:58.201Z{"msg":"Crucible ac93d57b-b66b-47c1-a421-8f8aab6a2e9b has session id: be8fbd34-58bd-4d69-a9b8-63db462a19e4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097598726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79742024-02-12T01:37:58.202Z{"msg":"Upstairs opts: Upstairs UUID: ac93d57b-b66b-47c1-a421-8f8aab6a2e9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097633104Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79752024-02-12T01:37:58.202Z{"msg":"Crucible stats registered with UUID: ac93d57b-b66b-47c1-a421-8f8aab6a2e9b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097662533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79762024-02-12T01:37:58.202Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097685161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
79772024-02-12T01:37:58.202Z{"msg":"ac93d57b-b66b-47c1-a421-8f8aab6a2e9b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.09771201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79782024-02-12T01:37:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097734329Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79792024-02-12T01:37:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097776517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79802024-02-12T01:37:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097802786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79812024-02-12T01:37:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097834274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79822024-02-12T01:37:58.202Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097858643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79832024-02-12T01:37:58.202Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097899511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79842024-02-12T01:37:58.202Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.09792596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79852024-02-12T01:37:58.202Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097960728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79862024-02-12T01:37:58.202Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.097986147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79872024-02-12T01:37:58.202Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098083393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79882024-02-12T01:37:58.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098153529Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79892024-02-12T01:37:58.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098216306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79902024-02-12T01:37:58.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098796748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79912024-02-12T01:37:58.208Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098829337Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79922024-02-12T01:37:58.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098857096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79932024-02-12T01:37:58.208Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.098911823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79942024-02-12T01:37:58.208Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099360712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79952024-02-12T01:37:58.208Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099514954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79962024-02-12T01:37:58.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099798691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79972024-02-12T01:37:58.209Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099838399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79982024-02-12T01:37:58.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099872417Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
79992024-02-12T01:37:58.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.099953213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80002024-02-12T01:37:58.209Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.10001766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80012024-02-12T01:37:58.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.100051519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80022024-02-12T01:37:58.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.100066128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80032024-02-12T01:37:58.209Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.100093157Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80042024-02-12T01:37:58.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.100539036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80052024-02-12T01:37:58.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.100984194Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80062024-02-12T01:37:58.209Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101022012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80072024-02-12T01:37:58.209Z{"msg":"[0]R flush_numbers[0..12]: [121, 121, 121, 121, 124, 116, 124, 124, 122, 110, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101039112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80082024-02-12T01:37:58.209Z{"msg":"[0]R generation[0..12]: [69, 69, 69, 69, 70, 66, 70, 70, 70, 64, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101054351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80092024-02-12T01:37:58.209Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.10107679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80102024-02-12T01:37:58.209Z{"msg":"[1]R flush_numbers[0..12]: [121, 121, 121, 121, 124, 116, 124, 124, 122, 110, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101092319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80112024-02-12T01:37:58.209Z{"msg":"[1]R generation[0..12]: [69, 69, 69, 69, 70, 66, 70, 70, 70, 64, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101106398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80122024-02-12T01:37:58.209Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101129117Z","hostname":"Wait for a query_work_queue command to finish before sending IO
80132024-02-12T01:37:58.209Zip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80142024-02-12T01:37:58.209Z{"msg":"[2]R flush_numbers[0..12]: [121, 121, 121, 121, 124, 116, 124, 124, 122, 110, 124, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101147886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80152024-02-12T01:37:58.209Z{"msg":"[2]R generation[0..12]: [69, 69, 69, 69, 70, 66, 70, 70, 70, 64, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101162076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80162024-02-12T01:37:58.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101176955Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80172024-02-12T01:37:58.209Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101190284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80182024-02-12T01:37:58.210Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101212433Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80192024-02-12T01:37:58.210Z{"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101225483Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80202024-02-12T01:37:58.210Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101238772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80212024-02-12T01:37:58.210Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101259721Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80222024-02-12T01:37:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.10127213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80232024-02-12T01:37:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.10128529Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80242024-02-12T01:37:58.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101305689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80252024-02-12T01:37:58.210Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101319158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80262024-02-12T01:37:58.210Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101332098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80272024-02-12T01:37:58.210Z{"msg":"ac93d57b-b66b-47c1-a421-8f8aab6a2e9b is now active with session: be8fbd34-58bd-4d69-a9b8-63db462a19e4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101352897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80282024-02-12T01:37:58.210Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101365816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423,"session_id":"be8fbd34-58bd-4d69-a9b8-63db462a19e4"}
80292024-02-12T01:37:58.210Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:37:57.101378475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
80302024-02-12T01:37:58.210ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80312024-02-12T01:37:58.210ZLoading write count information from file "/tmp/test_repair_verify.data"
80322024-02-12T01:37:58.210ZRun Repair workload
80332024-02-12T01:37:58.210Z01/30 Write block 49 len 12288 data: 7 7 7
80342024-02-12T01:37:58.210Z02/30 Write block 384 len 40960 data: 5 6 7 8 8 7 10 10 9 8
80352024-02-12T01:37:58.213Z03/30 Write block 228 len 8192 data: 6 5
80362024-02-12T01:37:58.213Z04/30 Read block 48 len 24576
80372024-02-12T01:37:58.213Z05/30 Read block 212 len 28672
80382024-02-12T01:37:58.214Z06/30 Write block 405 len 8192 data: 6 6
80392024-02-12T01:37:58.214Z07/30 Read block 26 len 32768
80402024-02-12T01:37:58.214Z08/30 Write block 9 len 20480 data: 6 5 6 7 6
80412024-02-12T01:37:58.214Z09/30 Write block 14 len 12288 data: 7 7 6
80422024-02-12T01:37:58.214Z10/30 Write block 318 len 8192 data: 4 5
80432024-02-12T01:37:58.214Z11/30 Read block 20 len 12288
80442024-02-12T01:37:58.217Z12/30 Read block 430 len 8192
80452024-02-12T01:37:58.218Z13/30 Read block 126 len 8192
80462024-02-12T01:37:58.218Z14/30 Read block 320 len 8192
80472024-02-12T01:37:58.218Z15/30 Write block 69 len 16384 data: 8 7 9 6
80482024-02-12T01:37:58.218Z16/30 Flush
80492024-02-12T01:37:58.221Z17/30 Write block 561 len 40960 data: 6 5 6 5 5 6 7 9 10 10
80502024-02-12T01:37:58.221Z18/30 Flush
80512024-02-12T01:37:58.224Z19/30 Write block 170 len 40960 data: 5 4 3 5 5 9 10 9 10 10
80522024-02-12T01:37:58.224Z20/30 Write block 385 len 4096 data: 7
80532024-02-12T01:37:58.224Z21/30 Read block 534 len 40960
80542024-02-12T01:37:58.226Z22/30 Write block 412 len 16384 data: 7 9 7 8
80552024-02-12T01:37:58.226Z23/30 Flush
80562024-02-12T01:37:58.229Z24/30 Write block 315 len 32768 data: 3 3 4 5 6 5 6 7
80572024-02-12T01:37:58.229Z25/30 Flush
80582024-02-12T01:37:58.232Z26/30 Write block 379 len 40960 data: 7 6 6 5 5 6 8 8 9 9
80592024-02-12T01:37:58.232Z27/30 Read block 424 len 16384
80602024-02-12T01:37:58.235Z28/30 Write block 503 len 12288 data: 4 4 4
80612024-02-12T01:37:58.235Z29/30 Write block 221 len 28672 data: 6 6 6 5 5 5 5
80622024-02-12T01:37:58.235Z30/30 Read block 301 len 12288
80632024-02-12T01:37:58.238Z----------------------------------------------------------------
80642024-02-12T01:37:58.238Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
80652024-02-12T01:37:58.238ZStates: Active Active Active
80662024-02-12T01:37:58.239ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80672024-02-12T01:37:58.239Z 1 Acked 1000 Write 3 Done Done Done false
80682024-02-12T01:37:58.239Z 2 Acked 1001 Write 10 Done Done Done false
80692024-02-12T01:37:58.239Z 3 Acked 1002 Write 2 Done Done Done false
80702024-02-12T01:37:58.239Z 4 Acked 1003 Read 6 Done Done Done false
80712024-02-12T01:37:58.239Z 5 Acked 1004 Read 7 Done Done Done false
80722024-02-12T01:37:58.239Z 6 Acked 1005 Write 2 Done Done Done false
80732024-02-12T01:37:58.239Z 7 Acked 1006 Read 8 Done Done Done false
80742024-02-12T01:37:58.239Z 8 Acked 1007 Write 5 Done Sent Done false
80752024-02-12T01:37:58.239Z 9 Acked 1008 Write 3 Done Sent Done false
80762024-02-12T01:37:58.239Z 10 Acked 1009 Write 2 Done Sent Done false
80772024-02-12T01:37:58.239Z 11 Acked 1010 Read 3 Done Sent Done false
80782024-02-12T01:37:58.239Z 12 Acked 1011 Read 2 Done Sent Done false
80792024-02-12T01:37:58.239Z 13 Acked 1012 Read 2 Done Sent Done false
80802024-02-12T01:37:58.239Z 14 Acked 1013 Read 2 Done Sent Done false
80812024-02-12T01:37:58.239Z 15 Acked 1014 Write 4 Done Sent Done false
80822024-02-12T01:37:58.240Z 16 Acked 1015 Flush 0 Done Sent Done false
80832024-02-12T01:37:58.240Z 17 Acked 1016 Write 10 Done Sent Done false
80842024-02-12T01:37:58.240Z 18 Acked 1017 Flush 0 Done Sent Done false
80852024-02-12T01:37:58.240Z 19 Acked 1018 Write 10 Done Sent Done false
80862024-02-12T01:37:58.240Z 20 Acked 1019 Write 1 Done Sent Done false
80872024-02-12T01:37:58.240Z 21 Acked 1020 Read 10 Done Sent Done false
80882024-02-12T01:37:58.240Z 22 Acked 1021 Write 4 Done Sent Done false
80892024-02-12T01:37:58.240Z 23 Acked 1022 Flush 0 Done Sent Done false
80902024-02-12T01:37:58.240Z 24 Acked 1023 Write 8 Done Sent Done false
80912024-02-12T01:37:58.240Z 25 Acked 1024 Flush 0 Done Sent Done false
80922024-02-12T01:37:58.240Z 26 Acked 1025 Write 10 Done Sent Done false
80932024-02-12T01:37:58.240Z 27 Acked 1026 Read 4 Done Sent Done false
80942024-02-12T01:37:58.240Z 28 Acked 1027 Write 3 Done Sent Done false
80952024-02-12T01:37:58.240Z 29 Acked 1028 Write 7 Done Sent Done false
80962024-02-12T01:37:58.240Z 30 Acked 1029 Read 3 Done Sent Sent false
80972024-02-12T01:37:58.240Z STATES DS:0 DS:1 DS:2 TOTAL
80982024-02-12T01:37:58.241Z New 0 0 0 0
80992024-02-12T01:37:58.241Z Sent 0 23 1 24
81002024-02-12T01:37:58.241Z Done 30 7 29 66
81012024-02-12T01:37:58.241Z Skipped 0 0 0 0
81022024-02-12T01:37:58.241Z Error 0 0 0 0
81032024-02-12T01:37:58.241ZLast Flush: 1024 0 1024
81042024-02-12T01:37:58.241ZDownstairs last five completed:
81052024-02-12T01:37:58.241ZUpstairs last five completed: 30 29 28 27 26
81062024-02-12T01:37:58.241Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:37:57.133027835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1423}
81072024-02-12T01:37:58.241ZWrote out file "/tmp/test_repair_verify.data"
81082024-02-12T01:37:58.244Z
81092024-02-12T01:37:58.244ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
81102024-02-12T01:37:58.272ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81112024-02-12T01:37:58.272Z 0 000-019 72 69 72 126 121 126 F F F <---
81122024-02-12T01:37:58.272Z 1 020-039 69 69 69 121 121 121 F F F
81132024-02-12T01:37:58.273Z 2 040-059 72 69 72 126 121 126 F T F <---
81142024-02-12T01:37:58.273Z 3 060-079 72 69 72 126 121 126 F F F <---
81152024-02-12T01:37:58.273Z 4 080-099 70 70 70 124 124 124 F F F
81162024-02-12T01:37:58.273Z 5 100-119 66 66 66 116 116 116 F F F
81172024-02-12T01:37:58.273Z 6 120-139 70 70 70 124 124 124 F F F
81182024-02-12T01:37:58.273Z 7 140-159 70 70 70 124 124 124 F F F
81192024-02-12T01:37:58.273Z 8 160-179 72 70 72 128 122 128 F F F <---
81202024-02-12T01:37:58.273Z 9 180-199 64 64 64 110 110 110 F F F
81212024-02-12T01:37:58.273Z 10 200-219 70 70 70 124 124 124 F F F
81222024-02-12T01:37:58.273Z 11 220-239 72 70 72 126 124 126 T T T <---
81232024-02-12T01:37:58.277Z 12 240-259 65 65 65 112 112 112 F F F
81242024-02-12T01:37:58.277Z 13 260-279 68 68 68 120 120 120 F F F
81252024-02-12T01:37:58.277Z 14 280-299 69 69 69 121 121 121 F F F
81262024-02-12T01:37:58.277Z 15 300-319 72 55 72 129 97 129 F F F <---
81272024-02-12T01:37:58.277Z 16 320-339 72 69 72 129 121 129 F F F <---
81282024-02-12T01:37:58.277Z 17 340-359 68 68 68 119 119 119 F F F
81292024-02-12T01:37:58.277Z 18 360-379 70 70 70 122 122 122 T F T <---
81302024-02-12T01:37:58.277Z 19 380-399 72 69 72 128 121 128 T T T <---
81312024-02-12T01:37:58.277Z 20 400-419 72 69 72 128 121 128 F T F <---
81322024-02-12T01:37:58.277Z 21 420-439 61 61 61 104 104 104 F F F
81332024-02-12T01:37:58.277Z 22 440-459 70 70 70 124 124 124 F F F
81342024-02-12T01:37:58.277Z 23 460-479 70 70 70 123 123 123 F F F
81352024-02-12T01:37:58.277Z 24 480-499 71 71 71 125 125 125 F F F
81362024-02-12T01:37:58.277Z 25 500-519 70 70 70 122 122 122 T F T <---
81372024-02-12T01:37:58.278Z 26 520-539 70 70 70 122 122 122 F F F
81382024-02-12T01:37:58.278Z 27 540-559 66 66 66 115 115 115 F F F
81392024-02-12T01:37:58.278Z 28 560-579 72 71 72 127 125 127 F F F <---
81402024-02-12T01:37:58.278Z 29 580-599 67 67 67 117 117 117 F F F
81412024-02-12T01:37:58.278ZMax gen: 72, Max flush: 129
81422024-02-12T01:37:58.278ZError: Difference in extent metadata found!
81432024-02-12T01:37:58.278ZA Difference in extent metadata is expected here
81442024-02-12T01:37:58.278ZOn loop 36
81452024-02-12T01:37:58.278Z
81462024-02-12T01:37:58.278ZVerifying data now
81472024-02-12T01:38:03.844ZNo change after verify
81482024-02-12T01:38:03.844ZLoop: 36 Downstairs dump after verify (and repair):
81492024-02-12T01:38:03.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81502024-02-12T01:38:03.860Z 0 000-019 72 72 72 126 126 126 F F F
81512024-02-12T01:38:03.860Z 1 020-039 69 69 69 121 121 121 F F F
81522024-02-12T01:38:03.860Z 2 040-059 72 72 72 126 126 126 F F F
81532024-02-12T01:38:03.860Z 3 060-079 72 72 72 126 126 126 F F F
81542024-02-12T01:38:03.860Z 4 080-099 70 70 70 124 124 124 F F F
81552024-02-12T01:38:03.860Z 5 100-119 66 66 66 116 116 116 F F F
81562024-02-12T01:38:03.860Z 6 120-139 70 70 70 124 124 124 F F F
81572024-02-12T01:38:03.860Z 7 140-159 70 70 70 124 124 124 F F F
81582024-02-12T01:38:03.860Z 8 160-179 72 72 72 128 128 128 F F F
81592024-02-12T01:38:03.860Z 9 180-199 64 64 64 110 110 110 F F F
81602024-02-12T01:38:03.860Z 10 200-219 70 70 70 124 124 124 F F F
81612024-02-12T01:38:03.860Z 11 220-239 73 73 73 130 130 130 F F F
81622024-02-12T01:38:03.860Z 12 240-259 65 65 65 112 112 112 F F F
81632024-02-12T01:38:03.861Z 13 260-279 68 68 68 120 120 120 F F F
81642024-02-12T01:38:03.861Z 14 280-299 69 69 69 121 121 121 F F F
81652024-02-12T01:38:03.864Z 15 300-319 72 72 72 129 129 129 F F F
81662024-02-12T01:38:03.864Z 16 320-339 72 72 72 129 129 129 F F F
81672024-02-12T01:38:03.864Z 17 340-359 68 68 68 119 119 119 F F F
81682024-02-12T01:38:03.864Z 18 360-379 73 73 73 130 130 130 F F F
81692024-02-12T01:38:03.864Z 19 380-399 73 73 73 130 130 130 F F F
81702024-02-12T01:38:03.864Z 20 400-419 72 72 72 128 128 128 F F F
81712024-02-12T01:38:03.864Z 21 420-439 61 61 61 104 104 104 F F F
81722024-02-12T01:38:03.865Z 22 440-459 70 70 70 124 124 124 F F F
81732024-02-12T01:38:03.865Z 23 460-479 70 70 70 123 123 123 F F F
81742024-02-12T01:38:03.865Z 24 480-499 71 71 71 125 125 125 F F F
81752024-02-12T01:38:03.865Z 25 500-519 73 73 73 130 130 130 F F F
81762024-02-12T01:38:03.865Z 26 520-539 70 70 70 122 122 122 F F F
81772024-02-12T01:38:03.865Z 27 540-559 66 66 66 115 115 115 F F F
81782024-02-12T01:38:03.865Z 28 560-579 72 72 72 127 127 127 F F F
81792024-02-12T01:38:03.865Z 29 580-599 67 67 67 117 117 117 F F F
81802024-02-12T01:38:03.865ZMax gen: 73, Max flush: 130
81812024-02-12T01:38:03.865Z
81822024-02-12T01:38:03.865ZBegin loop 37
81832024-02-12T01:38:03.865ZDownstairs to restart: 0
81842024-02-12T01:38:03.882Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778263556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
81852024-02-12T01:38:03.882Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778516944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
81862024-02-12T01:38:03.887Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778544923Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
81872024-02-12T01:38:03.887ZCrucible runtime is spawned
81882024-02-12T01:38:03.888Z{"msg":"Crucible c036ee71-4424-4735-9511-b51f0dfdb5d3 has session id: 862a3846-2a94-4fac-a8df-4dea4442d4ff","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778587841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81892024-02-12T01:38:03.888Z{"msg":"Upstairs opts: Upstairs UUID: c036ee71-4424-4735-9511-b51f0dfdb5d3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778632058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81902024-02-12T01:38:03.888Z{"msg":"Crucible stats registered with UUID: c036ee71-4424-4735-9511-b51f0dfdb5d3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778650458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81912024-02-12T01:38:03.888Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778677286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
81922024-02-12T01:38:03.888Z{"msg":"c036ee71-4424-4735-9511-b51f0dfdb5d3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778738503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81932024-02-12T01:38:03.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778766932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81942024-02-12T01:38:03.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778792161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81952024-02-12T01:38:03.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778827499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81962024-02-12T01:38:03.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778872187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81972024-02-12T01:38:03.888Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778898426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81982024-02-12T01:38:03.888Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778923705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
81992024-02-12T01:38:03.888Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778964723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82002024-02-12T01:38:03.888Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.778999131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82012024-02-12T01:38:03.888Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779040129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82022024-02-12T01:38:03.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779064198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82032024-02-12T01:38:03.888Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779124635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82042024-02-12T01:38:03.888Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:02.779272288Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82052024-02-12T01:38:03.888Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:02.779330685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82062024-02-12T01:38:03.888Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:02.779381033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82072024-02-12T01:38:03.888Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779412531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82082024-02-12T01:38:03.888Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.77942899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82092024-02-12T01:38:03.888Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779559324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82102024-02-12T01:38:03.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779963115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82112024-02-12T01:38:03.889Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.779991114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82122024-02-12T01:38:03.889Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.780084999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82132024-02-12T01:38:03.889Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.780106418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82142024-02-12T01:38:03.889Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.780586365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82152024-02-12T01:38:03.889Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.780786965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82162024-02-12T01:38:03.889Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.781067092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82172024-02-12T01:38:03.889Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.78110339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82182024-02-12T01:38:03.889Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.781122269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82192024-02-12T01:38:03.889Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.781250143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82202024-02-12T01:38:03.889Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.781290452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82212024-02-12T01:38:03.889Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:02.781306481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82222024-02-12T01:38:04.885Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.78092111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82232024-02-12T01:38:04.885Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.781037325Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82242024-02-12T01:38:04.892Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.781782799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82252024-02-12T01:38:04.892Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.781827297Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82262024-02-12T01:38:04.892Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782289595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82272024-02-12T01:38:04.892Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782772502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82282024-02-12T01:38:04.892Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782835079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82292024-02-12T01:38:04.892Z{"msg":"[0]R flush_numbers[0..12]: [126, 121, 126, 126, 124, 116, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782850368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82302024-02-12T01:38:04.892Z{"msg":"[0]R generation[0..12]: [72, 69, 72, 72, 70, 66, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782866557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82312024-02-12T01:38:04.892Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782880917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82322024-02-12T01:38:04.892Z{"msg":"[1]R flush_numbers[0..12]: [126, 121, 126, 126, 124, 116, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782895536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82332024-02-12T01:38:04.892Z{"msg":"[1]R generation[0..12]: [72, 69, 72, 72, 70, 66, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782917805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82342024-02-12T01:38:04.892Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782933194Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82352024-02-12T01:38:04.892Z{"msg":"[2]R flush_numbers[0..12]: [126, 121, 126, 126, 124, 116, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782955883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82362024-02-12T01:38:04.892Z{"msg":"[2]R generation[0..12]: [72, 69, 72, 72, 70, 66, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
82372024-02-12T01:38:04.892Z,"time":"2024-02-12T01:38:03.782970823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82382024-02-12T01:38:04.892Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.782987172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82392024-02-12T01:38:04.892Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783008521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82402024-02-12T01:38:04.892Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.78302303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82412024-02-12T01:38:04.892Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783048379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82422024-02-12T01:38:04.892Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783062158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82432024-02-12T01:38:04.892Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783076898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82442024-02-12T01:38:04.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783089837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82452024-02-12T01:38:04.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783115776Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82462024-02-12T01:38:04.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783129935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82472024-02-12T01:38:04.893Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783144974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82482024-02-12T01:38:04.893Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783158403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82492024-02-12T01:38:04.893Z{"msg":"c036ee71-4424-4735-9511-b51f0dfdb5d3 is now active with session: 862a3846-2a94-4fac-a8df-4dea4442d4ff","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783179273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82502024-02-12T01:38:04.893Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783192392Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431,"session_id":"862a3846-2a94-4fac-a8df-4dea4442d4ff"}
82512024-02-12T01:38:04.893Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:03.783212211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
82522024-02-12T01:38:04.893ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82532024-02-12T01:38:04.893ZLoading write count information from file "/tmp/test_repair_verify.data"
82542024-02-12T01:38:04.893ZRun Repair workload
82552024-02-12T01:38:04.893Z01/30 Write block 511 len 28672 data: 5 5 4 3 6 5 5
82562024-02-12T01:38:04.896Z02/30 Read block 269 len 24576
82572024-02-12T01:38:04.896Z03/30 Write block 87 len 28672 data: 10 9 9 7 7 6 5
82582024-02-12T01:38:04.896Z04/30 Read block 38 len 16384
82592024-02-12T01:38:04.896Z05/30 Read block 499 len 32768
82602024-02-12T01:38:04.896Z06/30 Write block 513 len 12288 data: 5 4 7
82612024-02-12T01:38:04.896Z07/30 Write block 544 len 32768 data: 5 6 8 8 8 7 9 8
82622024-02-12T01:38:04.897Z08/30 Write block 447 len 16384 data: 6 6 6 6
82632024-02-12T01:38:04.897Z09/30 Write block 512 len 12288 data: 6 6 5
82642024-02-12T01:38:04.899Z10/30 Write block 473 len 28672 data: 7 7 6 4 4 5 8
82652024-02-12T01:38:04.899Z11/30 Write block 572 len 4096 data: 9
82662024-02-12T01:38:04.899Z12/30 Read block 371 len 28672
82672024-02-12T01:38:04.900Z13/30 Read block 586 len 40960
82682024-02-12T01:38:04.902Z14/30 Write block 428 len 12288 data: 2 2 2
82692024-02-12T01:38:04.902Z15/30 Write block 409 len 28672 data: 6 7 8 8 10 8 9
82702024-02-12T01:38:04.903Z16/30 Write block 359 len 24576 data: 4 5 5 5 6 8
82712024-02-12T01:38:04.903Z17/30 Write block 104 len 12288 data: 6 6 5
82722024-02-12T01:38:04.903Z18/30 Read block 497 len 4096
82732024-02-12T01:38:04.909Z19/30 Read block 389 len 12288
82742024-02-12T01:38:04.909Z20/30 Read block 356 len 36864
82752024-02-12T01:38:04.909Z21/30 Read block 212 len 12288
82762024-02-12T01:38:04.912Z22/30 Read block 250 len 16384
82772024-02-12T01:38:04.913Z23/30 Read block 458 len 28672
82782024-02-12T01:38:04.913Z24/30 Read block 466 len 8192
82792024-02-12T01:38:04.913Z25/30 Read block 552 len 20480
82802024-02-12T01:38:04.913Z26/30 Read block 1 len 20480
82812024-02-12T01:38:04.913Z27/30 Write block 327 len 24576 data: 6 6 5 5 6 5
82822024-02-12T01:38:04.913Z28/30 Write block 350 len 8192 data: 5 6
82832024-02-12T01:38:04.913Z29/30 Read block 445 len 8192
82842024-02-12T01:38:04.913Z30/30 Read block 44 len 32768
82852024-02-12T01:38:04.917Z----------------------------------------------------------------
82862024-02-12T01:38:04.917Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
82872024-02-12T01:38:04.917ZStates: Active Active Active
82882024-02-12T01:38:04.917ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82892024-02-12T01:38:04.917Z 1 Acked 1000 Write 7 Sent Done Done false
82902024-02-12T01:38:04.917Z 2 Acked 1001 Read 6 Sent Done Done false
82912024-02-12T01:38:04.917Z 3 Acked 1002 Write 7 Sent Done Done false
82922024-02-12T01:38:04.917Z 4 Acked 1003 Read 4 Sent Done Done false
82932024-02-12T01:38:04.917Z 5 Acked 1004 Read 8 Sent Done Done false
82942024-02-12T01:38:04.917Z 6 Acked 1005 Write 3 Sent Done Done false
82952024-02-12T01:38:04.917Z 7 Acked 1006 Write 8 Sent Done Done false
82962024-02-12T01:38:04.917Z 8 Acked 1007 Write 4 Sent Done Done false
82972024-02-12T01:38:04.917Z 9 Acked 1008 Write 3 Sent Done Done false
82982024-02-12T01:38:04.917Z 10 Acked 1009 Write 7 Sent Done Done false
82992024-02-12T01:38:04.917Z 11 Acked 1010 Write 1 Sent Done Done false
83002024-02-12T01:38:04.917Z 12 Acked 1011 Read 7 Sent Done Done false
83012024-02-12T01:38:04.917Z 13 Acked 1012 Read 10 Sent Done Done false
83022024-02-12T01:38:04.917Z 14 Acked 1013 Write 3 Sent Done Done false
83032024-02-12T01:38:04.917Z 15 Acked 1014 Write 7 Sent Done Done false
83042024-02-12T01:38:04.918Z 16 Acked 1015 Write 6 Sent Done Done false
83052024-02-12T01:38:04.918Z 17 Acked 1016 Write 3 Sent Done Done false
83062024-02-12T01:38:04.918Z 18 Acked 1017 Read 1 Sent Done Done false
83072024-02-12T01:38:04.918Z 19 Acked 1018 Read 3 Sent Done Done false
83082024-02-12T01:38:04.918Z 20 Acked 1019 Read 9 Sent Done Done false
83092024-02-12T01:38:04.918Z 21 Acked 1020 Read 3 Sent Done Done false
83102024-02-12T01:38:04.918Z 22 Acked 1021 Read 4 Sent Done Done false
83112024-02-12T01:38:04.918Z 23 Acked 1022 Read 7 Sent Done Done false
83122024-02-12T01:38:04.918Z 24 Acked 1023 Read 2 Sent Done Done false
83132024-02-12T01:38:04.918Z 25 Acked 1024 Read 5 Sent Done Done false
83142024-02-12T01:38:04.918Z 26 Acked 1025 Read 5 Sent Done Done false
83152024-02-12T01:38:04.918Z 27 Acked 1026 Write 6 Sent Done Done false
83162024-02-12T01:38:04.918Z 28 Acked 1027 Write 2 Sent Done Done false
83172024-02-12T01:38:04.918Z 29 Acked 1028 Read 2 Sent Done Done false
83182024-02-12T01:38:04.918Z 30 Acked 1029 Read 8 Sent Done Done false
83192024-02-12T01:38:04.918Z STATES DS:0 DS:1 DS:2 TOTAL
83202024-02-12T01:38:04.918Z New 0 0 0 0
83212024-02-12T01:38:04.918Z Sent 30 0 0 30
83222024-02-12T01:38:04.918Z Done 0 30 30 60
83232024-02-12T01:38:04.918Z Skipped 0 0 0 0
83242024-02-12T01:38:04.918Z Error 0 0 0 0
83252024-02-12T01:38:04.918ZLast Flush: 0 0 0
83262024-02-12T01:38:04.918ZDownstairs last five completed:
83272024-02-12T01:38:04.918ZUpstairs last five completed: 30 29 28 27 26
83282024-02-12T01:38:04.918Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:03.809961355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1431}
83292024-02-12T01:38:04.918ZWrote out file "/tmp/test_repair_verify.data"
83302024-02-12T01:38:04.923Z
83312024-02-12T01:38:04.923ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83322024-02-12T01:38:04.941ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83332024-02-12T01:38:04.941Z 0 000-019 72 72 72 126 126 126 F F F
83342024-02-12T01:38:04.941Z 1 020-039 69 69 69 121 121 121 F F F
83352024-02-12T01:38:04.941Z 2 040-059 72 72 72 126 126 126 F F F
83362024-02-12T01:38:04.941Z 3 060-079 72 72 72 126 126 126 F F F
83372024-02-12T01:38:04.941Z 4 080-099 70 70 70 124 124 124 F T T <---
83382024-02-12T01:38:04.941Z 5 100-119 66 66 66 116 116 116 F T T <---
83392024-02-12T01:38:04.941Z 6 120-139 70 70 70 124 124 124 F F F
83402024-02-12T01:38:04.941Z 7 140-159 70 70 70 124 124 124 F F F
83412024-02-12T01:38:04.941Z 8 160-179 72 72 72 128 128 128 F F F
83422024-02-12T01:38:04.941Z 9 180-199 64 64 64 110 110 110 F F F
83432024-02-12T01:38:04.945Z 10 200-219 70 70 70 124 124 124 F F F
83442024-02-12T01:38:04.945Z 11 220-239 73 73 73 130 130 130 F F F
83452024-02-12T01:38:04.945Z 12 240-259 65 65 65 112 112 112 F F F
83462024-02-12T01:38:04.945Z 13 260-279 68 68 68 120 120 120 F F F
83472024-02-12T01:38:04.945Z 14 280-299 69 69 69 121 121 121 F F F
83482024-02-12T01:38:04.945Z 15 300-319 72 72 72 129 129 129 F F F
83492024-02-12T01:38:04.945Z 16 320-339 72 72 72 129 129 129 F T T <---
83502024-02-12T01:38:04.945Z 17 340-359 68 68 68 119 119 119 F T T <---
83512024-02-12T01:38:04.945Z 18 360-379 73 73 73 130 130 130 F T T <---
83522024-02-12T01:38:04.945Z 19 380-399 73 73 73 130 130 130 F F F
83532024-02-12T01:38:04.945Z 20 400-419 72 72 72 128 128 128 F T T <---
83542024-02-12T01:38:04.945Z 21 420-439 61 61 61 104 104 104 F T T <---
83552024-02-12T01:38:04.945Z 22 440-459 70 70 70 124 124 124 F T T <---
83562024-02-12T01:38:04.945Z 23 460-479 70 70 70 123 123 123 F T T <---
83572024-02-12T01:38:04.945Z 24 480-499 71 71 71 125 125 125 F F F
83582024-02-12T01:38:04.945Z 25 500-519 73 73 73 130 130 130 F T T <---
83592024-02-12T01:38:04.945Z 26 520-539 70 70 70 122 122 122 F F F
83602024-02-12T01:38:04.946Z 27 540-559 66 66 66 115 115 115 F T T <---
83612024-02-12T01:38:04.946Z 28 560-579 72 72 72 127 127 127 F T T <---
83622024-02-12T01:38:04.946Z 29 580-599 67 67 67 117 117 117 F F F
83632024-02-12T01:38:04.946ZMax gen: 73, Max flush: 130
83642024-02-12T01:38:04.946ZError: Difference in extent metadata found!
83652024-02-12T01:38:04.946ZA Difference in extent metadata is expected here
83662024-02-12T01:38:04.946ZOn loop 37
83672024-02-12T01:38:04.946Z
83682024-02-12T01:38:04.949ZVerifying data now
83692024-02-12T01:38:09.552ZNo change after verify
83702024-02-12T01:38:09.552ZLoop: 37 Downstairs dump after verify (and repair):
83712024-02-12T01:38:09.570ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83722024-02-12T01:38:09.570Z 0 000-019 72 72 72 126 126 126 F F F
83732024-02-12T01:38:09.570Z 1 020-039 69 69 69 121 121 121 F F F
83742024-02-12T01:38:09.570Z 2 040-059 72 72 72 126 126 126 F F F
83752024-02-12T01:38:09.570Z 3 060-079 72 72 72 126 126 126 F F F
83762024-02-12T01:38:09.570Z 4 080-099 74 74 74 131 131 131 F F F
83772024-02-12T01:38:09.570Z 5 100-119 74 74 74 131 131 131 F F F
83782024-02-12T01:38:09.570Z 6 120-139 70 70 70 124 124 124 F F F
83792024-02-12T01:38:09.570Z 7 140-159 70 70 70 124 124 124 F F F
83802024-02-12T01:38:09.570Z 8 160-179 72 72 72 128 128 128 F F F
83812024-02-12T01:38:09.570Z 9 180-199 64 64 64 110 110 110 F F F
83822024-02-12T01:38:09.570Z 10 200-219 70 70 70 124 124 124 F F F
83832024-02-12T01:38:09.574Z 11 220-239 73 73 73 130 130 130 F F F
83842024-02-12T01:38:09.574Z 12 240-259 65 65 65 112 112 112 F F F
83852024-02-12T01:38:09.574Z 13 260-279 68 68 68 120 120 120 F F F
83862024-02-12T01:38:09.574Z 14 280-299 69 69 69 121 121 121 F F F
83872024-02-12T01:38:09.574Z 15 300-319 72 72 72 129 129 129 F F F
83882024-02-12T01:38:09.574Z 16 320-339 74 74 74 131 131 131 F F F
83892024-02-12T01:38:09.574Z 17 340-359 74 74 74 131 131 131 F F F
83902024-02-12T01:38:09.574Z 18 360-379 74 74 74 131 131 131 F F F
83912024-02-12T01:38:09.574Z 19 380-399 73 73 73 130 130 130 F F F
83922024-02-12T01:38:09.574Z 20 400-419 74 74 74 131 131 131 F F F
83932024-02-12T01:38:09.574Z 21 420-439 74 74 74 131 131 131 F F F
83942024-02-12T01:38:09.574Z 22 440-459 74 74 74 131 131 131 F F F
83952024-02-12T01:38:09.574Z 23 460-479 74 74 74 131 131 131 F F F
83962024-02-12T01:38:09.574Z 24 480-499 71 71 71 125 125 125 F F F
83972024-02-12T01:38:09.574Z 25 500-519 74 74 74 131 131 131 F F F
83982024-02-12T01:38:09.574Z 26 520-539 70 70 70 122 122 122 F F F
83992024-02-12T01:38:09.574Z 27 540-559 74 74 74 131 131 131 F F F
84002024-02-12T01:38:09.574Z 28 560-579 74 74 74 131 131 131 F F F
84012024-02-12T01:38:09.574Z 29 580-599 67 67 67 117 117 117 F F F
84022024-02-12T01:38:09.574ZMax gen: 74, Max flush: 131
84032024-02-12T01:38:09.574Z
84042024-02-12T01:38:09.575ZBegin loop 38
84052024-02-12T01:38:09.575ZDownstairs to restart: 0
84062024-02-12T01:38:09.587Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.48339121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
84072024-02-12T01:38:09.587Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483646707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
84082024-02-12T01:38:09.591Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483711374Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
84092024-02-12T01:38:09.592Z{"msg":"Crucible ed81ebf0-0660-4e5f-89f6-f2db51eb7023 has session id: 8377693e-7060-480b-8f95-60427e637dd4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483747543Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84102024-02-12T01:38:09.592ZCrucible runtime is spawned
84112024-02-12T01:38:09.592Z{"msg":"Upstairs opts: Upstairs UUID: ed81ebf0-0660-4e5f-89f6-f2db51eb7023, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483838248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84122024-02-12T01:38:09.592Z{"msg":"Crucible stats registered with UUID: ed81ebf0-0660-4e5f-89f6-f2db51eb7023","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483891786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84132024-02-12T01:38:09.592Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483918305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
84142024-02-12T01:38:09.592Z{"msg":"ed81ebf0-0660-4e5f-89f6-f2db51eb7023 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.483968832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84152024-02-12T01:38:09.592Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484001861Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84162024-02-12T01:38:09.592Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484040069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84172024-02-12T01:38:09.592Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484068387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84182024-02-12T01:38:09.592Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484107236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84192024-02-12T01:38:09.592Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484135744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84202024-02-12T01:38:09.592Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484161053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84212024-02-12T01:38:09.592Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.48421692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84222024-02-12T01:38:09.592Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484270808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84232024-02-12T01:38:09.592Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484312826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84242024-02-12T01:38:09.592Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484351754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84252024-02-12T01:38:09.592Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484390382Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84262024-02-12T01:38:09.592Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484419901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84272024-02-12T01:38:09.592Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.484998963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84282024-02-12T01:38:09.592Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.485037241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84292024-02-12T01:38:09.592Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.485076949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84302024-02-12T01:38:09.592Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.485116507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84312024-02-12T01:38:09.592Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.485544177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84322024-02-12T01:38:09.593Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.485704579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84332024-02-12T01:38:09.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486083691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84342024-02-12T01:38:09.593Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486130699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84352024-02-12T01:38:09.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486157538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84362024-02-12T01:38:09.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486189226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84372024-02-12T01:38:09.593Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486220845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84382024-02-12T01:38:09.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486253323Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84392024-02-12T01:38:09.593Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486453873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84402024-02-12T01:38:09.593Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486488162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84412024-02-12T01:38:09.597Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.486961629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84422024-02-12T01:38:09.598Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487387479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84432024-02-12T01:38:09.598Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487424577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84442024-02-12T01:38:09.598Z{"msg":"[0]R flush_numbers[0..12]: [126, 121, 126, 126, 131, 131, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487440986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84452024-02-12T01:38:09.598Z{"msg":"[0]R generation[0..12]: [72, 69, 72, 72, 74, 74, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487458366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84462024-02-12T01:38:09.598Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487490404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84472024-02-12T01:38:09.598Z{"msg":"[1]R flush_numbers[0..12]: [126, 121, 126, 126, 131, 131, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487513083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84482024-02-12T01:38:09.598Z{"msg":"[1]R generation[0..12]: [72, 69, 72, 72, 74, 74, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487543321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84492024-02-12T01:38:09.598Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487565411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
84502024-02-12T01:38:09.598Z,"pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84512024-02-12T01:38:09.598Z{"msg":"[2]R flush_numbers[0..12]: [126, 121, 126, 126, 131, 131, 124, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487597269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84522024-02-12T01:38:09.598Z{"msg":"[2]R generation[0..12]: [72, 69, 72, 72, 74, 74, 70, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487640127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84532024-02-12T01:38:09.598Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487674635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84542024-02-12T01:38:09.598Z{"msg":"Max found gen is 75","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487706344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84552024-02-12T01:38:09.598Z{"msg":"Generation requested: 76 >= found:75","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487738522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84562024-02-12T01:38:09.598Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487755022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84572024-02-12T01:38:09.598Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487823198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84582024-02-12T01:38:09.598Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487857837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84592024-02-12T01:38:09.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487872776Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84602024-02-12T01:38:09.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487893745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84612024-02-12T01:38:09.598Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487907494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84622024-02-12T01:38:09.598Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487927903Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84632024-02-12T01:38:09.599Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487940833Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84642024-02-12T01:38:09.599Z{"msg":"ed81ebf0-0660-4e5f-89f6-f2db51eb7023 is now active with session: 8377693e-7060-480b-8f95-60427e637dd4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487960222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84652024-02-12T01:38:09.599Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.487972451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439,"session_id":"8377693e-7060-480b-8f95-60427e637dd4"}
84662024-02-12T01:38:09.599Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:08.48799168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
84672024-02-12T01:38:09.599ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84682024-02-12T01:38:09.599ZLoading write count information from file "/tmp/test_repair_verify.data"
84692024-02-12T01:38:09.599ZRun Repair workload
84702024-02-12T01:38:09.599Z01/30 Write block 272 len 36864 data: 4 5 6 7 7 6 7 9 9
84712024-02-12T01:38:09.599Z02/30 Read block 123 len 40960
84722024-02-12T01:38:09.599Z03/30 Read block 533 len 8192
84732024-02-12T01:38:09.599Z04/30 Write block 566 len 4096 data: 7
84742024-02-12T01:38:09.602Z05/30 Write block 310 len 24576 data: 6 6 5 5 3 4
84752024-02-12T01:38:09.602Z06/30 Read block 521 len 24576
84762024-02-12T01:38:09.602Z07/30 Read block 261 len 32768
84772024-02-12T01:38:09.602Z08/30 Write block 546 len 32768 data: 9 9 9 8 10 9 8 8
84782024-02-12T01:38:09.602Z09/30 Write block 350 len 16384 data: 6 7 9 7
84792024-02-12T01:38:09.602Z10/30 Read block 259 len 28672
84802024-02-12T01:38:09.603Z11/30 Read block 55 len 40960
84812024-02-12T01:38:09.605Z12/30 Write block 531 len 4096 data: 7
84822024-02-12T01:38:09.605Z13/30 Read block 572 len 36864
84832024-02-12T01:38:09.605Z14/30 Read block 44 len 28672
84842024-02-12T01:38:09.605Z15/30 Read block 140 len 4096
84852024-02-12T01:38:09.608Z16/30 Write block 285 len 24576 data: 8 7 8 8 7 7
84862024-02-12T01:38:09.609Z17/30 Read block 91 len 12288
84872024-02-12T01:38:09.609Z18/30 Write block 550 len 8192 data: 11 10
84882024-02-12T01:38:09.609Z19/30 Read block 429 len 32768
84892024-02-12T01:38:09.612Z20/30 Write block 123 len 28672 data: 10 9 8 9 9 9 8
84902024-02-12T01:38:09.612Z21/30 Write block 97 len 20480 data: 5 6 4 3 5
84912024-02-12T01:38:09.612Z22/30 Write block 488 len 12288 data: 9 6 7
84922024-02-12T01:38:09.612Z23/30 Write block 404 len 28672 data: 6 7 7 5 6 7 8
84932024-02-12T01:38:09.615Z24/30 Read block 313 len 20480
84942024-02-12T01:38:09.615Z25/30 Flush
84952024-02-12T01:38:09.617Z26/30 Write block 510 len 28672 data: 4 6 7 7 6 8 6
84962024-02-12T01:38:09.620Z27/30 Write block 549 len 28672 data: 9 12 11 9 9 6 4
84972024-02-12T01:38:09.620Z28/30 Read block 313 len 4096
84982024-02-12T01:38:09.620Z29/30 Read block 350 len 36864
84992024-02-12T01:38:09.624Z30/30 Write block 58 len 4096 data: 10
85002024-02-12T01:38:09.624Z----------------------------------------------------------------
85012024-02-12T01:38:09.624Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
85022024-02-12T01:38:09.624ZStates: Active Active Active
85032024-02-12T01:38:09.624ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85042024-02-12T01:38:09.624Z 1 Acked 1000 Write 9 Done Done Done false
85052024-02-12T01:38:09.624Z 2 Acked 1001 Read 10 Done Done Done false
85062024-02-12T01:38:09.624Z 3 Acked 1002 Read 2 Done Done Done false
85072024-02-12T01:38:09.624Z 4 Acked 1003 Write 1 Sent Done Done false
85082024-02-12T01:38:09.624Z 5 Acked 1004 Write 6 Sent Done Done false
85092024-02-12T01:38:09.624Z 6 Acked 1005 Read 6 Sent Done Done false
85102024-02-12T01:38:09.624Z 7 Acked 1006 Read 8 Sent Done Done false
85112024-02-12T01:38:09.624Z 8 Acked 1007 Write 8 Sent Done Done false
85122024-02-12T01:38:09.624Z 9 Acked 1008 Write 4 Sent Done Done false
85132024-02-12T01:38:09.624Z 10 Acked 1009 Read 7 Sent Done Done false
85142024-02-12T01:38:09.624Z 11 Acked 1010 Read 10 Sent Done Done false
85152024-02-12T01:38:09.624Z 12 Acked 1011 Write 1 Sent Done Done false
85162024-02-12T01:38:09.624Z 13 Acked 1012 Read 9 Sent Done Done false
85172024-02-12T01:38:09.624Z 14 Acked 1013 Read 7 Sent Done Done false
85182024-02-12T01:38:09.624Z 15 Acked 1014 Read 1 Sent Done Done false
85192024-02-12T01:38:09.624Z 16 Acked 1015 Write 6 Sent Done Done false
85202024-02-12T01:38:09.624Z 17 Acked 1016 Read 3 Sent Done Done false
85212024-02-12T01:38:09.624Z 18 Acked 1017 Write 2 Sent Done Done false
85222024-02-12T01:38:09.625Z 19 Acked 1018 Read 8 Sent Done Done false
85232024-02-12T01:38:09.625Z 20 Acked 1019 Write 7 Sent Done Done false
85242024-02-12T01:38:09.625Z 21 Acked 1020 Write 5 Sent Done Done false
85252024-02-12T01:38:09.625Z 22 Acked 1021 Write 3 Sent Done Done false
85262024-02-12T01:38:09.625Z 23 Acked 1022 Write 7 Sent Done Done false
85272024-02-12T01:38:09.625Z 24 Acked 1023 Read 5 Sent Done Done false
85282024-02-12T01:38:09.625Z 25 Acked 1024 Flush 0 Sent Done Done false
85292024-02-12T01:38:09.625Z 26 Acked 1025 Write 7 Sent Done Done false
85302024-02-12T01:38:09.625Z 27 Acked 1026 Write 7 Sent Done Done false
85312024-02-12T01:38:09.625Z 28 Acked 1027 Read 1 Sent Done Done false
85322024-02-12T01:38:09.625Z 29 Acked 1028 Read 9 Sent Done Done false
85332024-02-12T01:38:09.625Z 30 Acked 1029 Write 1 Sent Sent Sent false
85342024-02-12T01:38:09.625Z STATES DS:0 DS:1 DS:2 TOTAL
85352024-02-12T01:38:09.625Z New 0 0 0 0
85362024-02-12T01:38:09.625Z Sent 27 1 1 29
85372024-02-12T01:38:09.625Z Done 3 29 29 61
85382024-02-12T01:38:09.625Z Skipped 0 0 0 0
85392024-02-12T01:38:09.625Z Error 0 0 0 0
85402024-02-12T01:38:09.625ZLast Flush: 0 1024 1024
85412024-02-12T01:38:09.625ZDownstairs last five completed:
85422024-02-12T01:38:09.625ZUpstairs last five completed: 30 29 28 27 26
85432024-02-12T01:38:09.625Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:08.51796507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1439}
85442024-02-12T01:38:09.625ZWrote out file "/tmp/test_repair_verify.data"
85452024-02-12T01:38:09.629Z
85462024-02-12T01:38:09.631ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85472024-02-12T01:38:09.645ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85482024-02-12T01:38:09.645Z 0 000-019 72 72 72 126 126 126 F F F
85492024-02-12T01:38:09.645Z 1 020-039 69 69 69 121 121 121 F F F
85502024-02-12T01:38:09.645Z 2 040-059 72 72 72 126 126 126 F T T <---
85512024-02-12T01:38:09.645Z 3 060-079 72 72 72 126 126 126 F F F
85522024-02-12T01:38:09.645Z 4 080-099 74 76 76 131 132 132 F F F <---
85532024-02-12T01:38:09.645Z 5 100-119 74 76 76 131 132 132 F F F <---
85542024-02-12T01:38:09.645Z 6 120-139 70 76 76 124 132 132 F F F <---
85552024-02-12T01:38:09.646Z 7 140-159 70 70 70 124 124 124 F F F
85562024-02-12T01:38:09.646Z 8 160-179 72 72 72 128 128 128 F F F
85572024-02-12T01:38:09.646Z 9 180-199 64 64 64 110 110 110 F F F
85582024-02-12T01:38:09.649Z 10 200-219 70 70 70 124 124 124 F F F
85592024-02-12T01:38:09.649Z 11 220-239 73 73 73 130 130 130 F F F
85602024-02-12T01:38:09.649Z 12 240-259 65 65 65 112 112 112 F F F
85612024-02-12T01:38:09.649Z 13 260-279 68 76 76 120 132 132 T F F <---
85622024-02-12T01:38:09.649Z 14 280-299 69 76 76 121 132 132 T F F <---
85632024-02-12T01:38:09.649Z 15 300-319 72 76 76 129 132 132 F F F <---
85642024-02-12T01:38:09.649Z 16 320-339 74 74 74 131 131 131 F F F
85652024-02-12T01:38:09.649Z 17 340-359 74 76 76 131 132 132 F F F <---
85662024-02-12T01:38:09.649Z 18 360-379 74 74 74 131 131 131 F F F
85672024-02-12T01:38:09.649Z 19 380-399 73 73 73 130 130 130 F F F
85682024-02-12T01:38:09.649Z 20 400-419 74 76 76 131 132 132 F F F <---
85692024-02-12T01:38:09.649Z 21 420-439 74 74 74 131 131 131 F F F
85702024-02-12T01:38:09.649Z 22 440-459 74 74 74 131 131 131 F F F
85712024-02-12T01:38:09.649Z 23 460-479 74 74 74 131 131 131 F F F
85722024-02-12T01:38:09.649Z 24 480-499 71 76 76 125 132 132 F F F <---
85732024-02-12T01:38:09.649Z 25 500-519 74 74 74 131 131 131 F T T <---
85742024-02-12T01:38:09.649Z 26 520-539 70 76 76 122 132 132 F F F <---
85752024-02-12T01:38:09.649Z 27 540-559 74 76 76 131 132 132 F T T <---
85762024-02-12T01:38:09.650Z 28 560-579 74 76 76 131 132 132 F F F <---
85772024-02-12T01:38:09.650Z 29 580-599 67 67 67 117 117 117 F F F
85782024-02-12T01:38:09.650ZMax gen: 76, Max flush: 132
85792024-02-12T01:38:09.650ZError: Difference in extent metadata found!
85802024-02-12T01:38:09.650ZA Difference in extent metadata is expected here
85812024-02-12T01:38:09.650ZOn loop 38
85822024-02-12T01:38:09.650Z
85832024-02-12T01:38:09.656ZVerifying data now
85842024-02-12T01:38:14.315ZNo change after verify
85852024-02-12T01:38:14.316ZLoop: 38 Downstairs dump after verify (and repair):
85862024-02-12T01:38:14.333ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85872024-02-12T01:38:14.333Z 0 000-019 72 72 72 126 126 126 F F F
85882024-02-12T01:38:14.333Z 1 020-039 69 69 69 121 121 121 F F F
85892024-02-12T01:38:14.334Z 2 040-059 77 77 77 133 133 133 F F F
85902024-02-12T01:38:14.334Z 3 060-079 72 72 72 126 126 126 F F F
85912024-02-12T01:38:14.334Z 4 080-099 76 76 76 132 132 132 F F F
85922024-02-12T01:38:14.334Z 5 100-119 76 76 76 132 132 132 F F F
85932024-02-12T01:38:14.334Z 6 120-139 76 76 76 132 132 132 F F F
85942024-02-12T01:38:14.334Z 7 140-159 70 70 70 124 124 124 F F F
85952024-02-12T01:38:14.334Z 8 160-179 72 72 72 128 128 128 F F F
85962024-02-12T01:38:14.334Z 9 180-199 64 64 64 110 110 110 F F F
85972024-02-12T01:38:14.334Z 10 200-219 70 70 70 124 124 124 F F F
85982024-02-12T01:38:14.334Z 11 220-239 73 73 73 130 130 130 F F F
85992024-02-12T01:38:14.334Z 12 240-259 65 65 65 112 112 112 F F F
86002024-02-12T01:38:14.334Z 13 260-279 76 76 76 132 132 132 F F F
86012024-02-12T01:38:14.338Z 14 280-299 76 76 76 132 132 132 F F F
86022024-02-12T01:38:14.338Z 15 300-319 76 76 76 132 132 132 F F F
86032024-02-12T01:38:14.338Z 16 320-339 74 74 74 131 131 131 F F F
86042024-02-12T01:38:14.338Z 17 340-359 76 76 76 132 132 132 F F F
86052024-02-12T01:38:14.338Z 18 360-379 74 74 74 131 131 131 F F F
86062024-02-12T01:38:14.338Z 19 380-399 73 73 73 130 130 130 F F F
86072024-02-12T01:38:14.338Z 20 400-419 76 76 76 132 132 132 F F F
86082024-02-12T01:38:14.338Z 21 420-439 74 74 74 131 131 131 F F F
86092024-02-12T01:38:14.338Z 22 440-459 74 74 74 131 131 131 F F F
86102024-02-12T01:38:14.338Z 23 460-479 74 74 74 131 131 131 F F F
86112024-02-12T01:38:14.338Z 24 480-499 76 76 76 132 132 132 F F F
86122024-02-12T01:38:14.338Z 25 500-519 77 77 77 133 133 133 F F F
86132024-02-12T01:38:14.338Z 26 520-539 76 76 76 132 132 132 F F F
86142024-02-12T01:38:14.338Z 27 540-559 77 77 77 133 133 133 F F F
86152024-02-12T01:38:14.338Z 28 560-579 76 76 76 132 132 132 F F F
86162024-02-12T01:38:14.338Z 29 580-599 67 67 67 117 117 117 F F F
86172024-02-12T01:38:14.339ZMax gen: 77, Max flush: 133
86182024-02-12T01:38:14.339Z
86192024-02-12T01:38:14.339ZBegin loop 39
86202024-02-12T01:38:14.339ZDownstairs to restart: 2
86212024-02-12T01:38:14.356Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251102875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
86222024-02-12T01:38:14.356Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251409161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
86232024-02-12T01:38:14.361Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
86242024-02-12T01:38:14.361Z,"time":"2024-02-12T01:38:13.251434999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
86252024-02-12T01:38:14.361Z{"msg":"Crucible 3d7cd6d0-1657-4dd1-ab39-75680cfdb5b3 has session id: 94c3972e-9775-4312-b5e0-cb13153edb93","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251480857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86262024-02-12T01:38:14.361Z{"msg":"Upstairs opts: Upstairs UUID: 3d7cd6d0-1657-4dd1-ab39-75680cfdb5b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251537465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86272024-02-12T01:38:14.361Z{"msg":"Crucible stats registered with UUID: 3d7cd6d0-1657-4dd1-ab39-75680cfdb5b3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251585122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86282024-02-12T01:38:14.361Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251616851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
86292024-02-12T01:38:14.361Z{"msg":"3d7cd6d0-1657-4dd1-ab39-75680cfdb5b3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251711906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86302024-02-12T01:38:14.361Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251747284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86312024-02-12T01:38:14.361Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251776993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86322024-02-12T01:38:14.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251819521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86332024-02-12T01:38:14.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251855099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86342024-02-12T01:38:14.362Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251928606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86352024-02-12T01:38:14.362Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.251976864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86362024-02-12T01:38:14.362Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252006022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86372024-02-12T01:38:14.362Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252062689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86382024-02-12T01:38:14.362Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252093988Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86392024-02-12T01:38:14.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252135366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86402024-02-12T01:38:14.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252180244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86412024-02-12T01:38:14.362Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252203083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86422024-02-12T01:38:14.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252715068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86432024-02-12T01:38:14.362Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252762726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86442024-02-12T01:38:14.362Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252794645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86452024-02-12T01:38:14.363Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.252868741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86462024-02-12T01:38:14.363Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.253385636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86472024-02-12T01:38:14.363Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.253422664Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86482024-02-12T01:38:14.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.253927531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86492024-02-12T01:38:14.363Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.253974498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86502024-02-12T01:38:14.363Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254001757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86512024-02-12T01:38:14.363Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254034115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86522024-02-12T01:38:14.363Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254059504Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86532024-02-12T01:38:14.363Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254088263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86542024-02-12T01:38:14.363Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254110362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86552024-02-12T01:38:14.363Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.25415154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86562024-02-12T01:38:14.363Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.254645126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86572024-02-12T01:38:14.368Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255070926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86582024-02-12T01:38:14.368Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255115564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86592024-02-12T01:38:14.368Z{"msg":"[0]R flush_numbers[0..12]: [126, 121, 133, 126, 132, 132, 132, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255160922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86602024-02-12T01:38:14.369Z{"msg":"[0]R generation[0..12]: [72, 69, 77, 72, 76, 76, 76, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.25519681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86612024-02-12T01:38:14.369Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255232238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86622024-02-12T01:38:14.369Z{"msg":"[1]R flush_numbers[0..12]: [126, 121, 133, 126, 132, 132, 132, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255269367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86632024-02-12T01:38:14.369Z{"msg":"[1]R generation[0..12]: [72, 69, 77, 72, 76, 76, 76, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
86642024-02-12T01:38:14.369Z,"time":"2024-02-12T01:38:13.255307645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86652024-02-12T01:38:14.369Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255349183Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86662024-02-12T01:38:14.369Z{"msg":"[2]R flush_numbers[0..12]: [126, 121, 133, 126, 132, 132, 132, 124, 128, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255387751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86672024-02-12T01:38:14.369Z{"msg":"[2]R generation[0..12]: [72, 69, 77, 72, 76, 76, 76, 70, 72, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255430879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86682024-02-12T01:38:14.369Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255458987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86692024-02-12T01:38:14.369Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255489736Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86702024-02-12T01:38:14.369Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255515275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86712024-02-12T01:38:14.369Z{"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255546873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86722024-02-12T01:38:14.370Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255571312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86732024-02-12T01:38:14.370Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255598821Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86742024-02-12T01:38:14.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.25562079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86752024-02-12T01:38:14.370ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86762024-02-12T01:38:14.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255651248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86772024-02-12T01:38:14.370Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255675027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86782024-02-12T01:38:14.370Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255702976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86792024-02-12T01:38:14.370Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255724925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86802024-02-12T01:38:14.370Z{"msg":"3d7cd6d0-1657-4dd1-ab39-75680cfdb5b3 is now active with session: 94c3972e-9775-4312-b5e0-cb13153edb93","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255757613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86812024-02-12T01:38:14.370Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255780002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447,"session_id":"94c3972e-9775-4312-b5e0-cb13153edb93"}
86822024-02-12T01:38:14.370Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:13.255806181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
86832024-02-12T01:38:14.370ZLoading write count information from file "/tmp/test_repair_verify.data"
86842024-02-12T01:38:14.370ZRun Repair workload
86852024-02-12T01:38:14.370Z01/30 Write block 33 len 16384 data: 10 10 6 7
86862024-02-12T01:38:14.370Z02/30 Read block 81 len 12288
86872024-02-12T01:38:14.370Z03/30 Read block 145 len 40960
86882024-02-12T01:38:14.370Z04/30 Write block 510 len 24576 data: 5 7 8 8 7 9
86892024-02-12T01:38:14.370Z05/30 Read block 510 len 20480
86902024-02-12T01:38:14.370Z06/30 Flush
86912024-02-12T01:38:14.376Z07/30 Write block 357 len 20480 data: 4 4 5 6 6
86922024-02-12T01:38:14.376Z08/30 Write block 165 len 36864 data: 8 8 6 6 6 6 5 4 6
86932024-02-12T01:38:14.376Z09/30 Read block 173 len 16384
86942024-02-12T01:38:14.377Z10/30 Read block 209 len 28672
86952024-02-12T01:38:14.377Z11/30 Read block 560 len 8192
86962024-02-12T01:38:14.377Z12/30 Read block 580 len 20480
86972024-02-12T01:38:14.377Z13/30 Read block 536 len 36864
86982024-02-12T01:38:14.377Z14/30 Read block 118 len 32768
86992024-02-12T01:38:14.383Z15/30 Write block 530 len 28672 data: 6 8 9 9 7 10 9
87002024-02-12T01:38:14.384Z16/30 Write block 101 len 28672 data: 6 4 4 7 7 6 6
87012024-02-12T01:38:14.384Z17/30 Read block 120 len 28672
87022024-02-12T01:38:14.384Z18/30 Read block 527 len 4096
87032024-02-12T01:38:14.384Z19/30 Read block 181 len 20480
87042024-02-12T01:38:14.387Z20/30 Write block 259 len 20480 data: 4 6 4 4 5
87052024-02-12T01:38:14.387Z21/30 Write block 52 len 24576 data: 4 2 3 4 4 6
87062024-02-12T01:38:14.387Z22/30 Write block 120 len 28672 data: 10 9 8 11 10 9 10
87072024-02-12T01:38:14.387Z23/30 Read block 209 len 20480
87082024-02-12T01:38:14.387Z24/30 Read block 338 len 16384
87092024-02-12T01:38:14.387Z25/30 Flush
87102024-02-12T01:38:14.390Z26/30 Write block 590 len 4096 data: 6
87112024-02-12T01:38:14.390Z27/30 Read block 173 len 24576
87122024-02-12T01:38:14.391Z28/30 Read block 235 len 16384
87132024-02-12T01:38:14.394Z29/30 Write block 530 len 20480 data: 7 9 10 10 8
87142024-02-12T01:38:14.394Z30/30 Read block 510 len 20480
87152024-02-12T01:38:14.394Z----------------------------------------------------------------
87162024-02-12T01:38:14.394Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
87172024-02-12T01:38:14.394ZStates: Active Active Active
87182024-02-12T01:38:14.394ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
87192024-02-12T01:38:14.395Z 1 Acked 1000 Write 4 Done Done Sent false
87202024-02-12T01:38:14.395Z 2 Acked 1001 Read 3 Done Done Sent false
87212024-02-12T01:38:14.395Z 3 Acked 1002 Read 10 Done Done Sent false
87222024-02-12T01:38:14.395Z 4 Acked 1003 Write 6 Done Done Sent false
87232024-02-12T01:38:14.395Z 5 Acked 1004 Read 5 Done Done Sent false
87242024-02-12T01:38:14.395Z 6 Acked 1005 Flush 0 Done Done Sent false
87252024-02-12T01:38:14.395Z 7 Acked 1006 Write 5 Done Done Sent false
87262024-02-12T01:38:14.395Z 8 Acked 1007 Write 9 Done Done Sent false
87272024-02-12T01:38:14.395Z 9 Acked 1008 Read 4 Done Done Sent false
87282024-02-12T01:38:14.395Z 10 Acked 1009 Read 7 Done Done Sent false
87292024-02-12T01:38:14.395Z 11 Acked 1010 Read 2 Done Done Sent false
87302024-02-12T01:38:14.395Z 12 Acked 1011 Read 5 Done Done Sent false
87312024-02-12T01:38:14.395Z 13 Acked 1012 Read 9 Done Done Sent false
87322024-02-12T01:38:14.396Z 14 Acked 1013 Read 8 Done Done Sent false
87332024-02-12T01:38:14.396Z 15 Acked 1014 Write 7 Done Done Sent false
87342024-02-12T01:38:14.396Z 16 Acked 1015 Write 7 Done Done Sent false
87352024-02-12T01:38:14.396Z 17 Acked 1016 Read 7 Done Done Sent false
87362024-02-12T01:38:14.396Z 18 Acked 1017 Read 1 Done Done Sent false
87372024-02-12T01:38:14.396Z 19 Acked 1018 Read 5 Done Done Sent false
87382024-02-12T01:38:14.396Z 20 Acked 1019 Write 5 Done Done Sent false
87392024-02-12T01:38:14.396Z 21 Acked 1020 Write 6 Done Done Sent false
87402024-02-12T01:38:14.396Z 22 Acked 1021 Write 7 Done Done Sent false
87412024-02-12T01:38:14.396Z 23 Acked 1022 Read 5 Done Done Sent false
87422024-02-12T01:38:14.396Z 24 Acked 1023 Read 4 Done Done Sent false
87432024-02-12T01:38:14.396Z 25 Acked 1024 Flush 0 Done Done Sent false
87442024-02-12T01:38:14.396Z 26 Acked 1025 Write 1 Done Done Sent false
87452024-02-12T01:38:14.397Z 27 Acked 1026 Read 6 Done Done Sent false
87462024-02-12T01:38:14.397Z 28 Acked 1027 Read 4 Done Done Sent false
87472024-02-12T01:38:14.397Z 29 Acked 1028 Write 5 Done Done Sent false
87482024-02-12T01:38:14.397Z 30 Acked 1029 Read 5 Done Done Sent false
87492024-02-12T01:38:14.397Z STATES DS:0 DS:1 DS:2 TOTAL
87502024-02-12T01:38:14.397Z New 0 0 0 0
87512024-02-12T01:38:14.397Z Sent 0 0 30 30
87522024-02-12T01:38:14.397Z Done 30 30 0 60
87532024-02-12T01:38:14.397Z Skipped 0 0 0 0
87542024-02-12T01:38:14.397Z Error 0 0 0 0
87552024-02-12T01:38:14.397ZLast Flush: 1024 1024 0
87562024-02-12T01:38:14.397ZDownstairs last five completed:
87572024-02-12T01:38:14.398ZUpstairs last five completed: 30 29 28 27 26
87582024-02-12T01:38:14.398Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:13.288292091Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1447}
87592024-02-12T01:38:14.398ZWrote out file "/tmp/test_repair_verify.data"
87602024-02-12T01:38:14.401Z
87612024-02-12T01:38:14.404ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87622024-02-12T01:38:14.417ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87632024-02-12T01:38:14.417Z 0 000-019 72 72 72 126 126 126 F F F
87642024-02-12T01:38:14.418Z 1 020-039 78 78 69 134 134 121 F F F <---
87652024-02-12T01:38:14.418Z 2 040-059 78 78 77 135 135 133 F F F <---
87662024-02-12T01:38:14.418Z 3 060-079 72 72 72 126 126 126 F F F
87672024-02-12T01:38:14.418Z 4 080-099 76 76 76 132 132 132 F F F
87682024-02-12T01:38:14.418Z 5 100-119 78 78 76 135 135 132 F F F <---
87692024-02-12T01:38:14.418Z 6 120-139 78 78 76 135 135 132 F F F <---
87702024-02-12T01:38:14.418Z 7 140-159 70 70 70 124 124 124 F F F
87712024-02-12T01:38:14.418Z 8 160-179 78 78 72 135 135 128 F F F <---
87722024-02-12T01:38:14.422Z 9 180-199 64 64 64 110 110 110 F F F
87732024-02-12T01:38:14.422Z 10 200-219 70 70 70 124 124 124 F F F
87742024-02-12T01:38:14.422Z 11 220-239 73 73 73 130 130 130 F F F
87752024-02-12T01:38:14.422Z 12 240-259 78 78 65 135 135 112 F F F <---
87762024-02-12T01:38:14.422Z 13 260-279 78 78 76 135 135 132 F F F <---
87772024-02-12T01:38:14.422Z 14 280-299 76 76 76 132 132 132 F F F
87782024-02-12T01:38:14.422Z 15 300-319 76 76 76 132 132 132 F F F
87792024-02-12T01:38:14.422Z 16 320-339 74 74 74 131 131 131 F F F
87802024-02-12T01:38:14.422Z 17 340-359 78 78 76 135 135 132 F F F <---
87812024-02-12T01:38:14.422Z 18 360-379 78 78 74 135 135 131 F F F <---
87822024-02-12T01:38:14.422Z 19 380-399 73 73 73 130 130 130 F F F
87832024-02-12T01:38:14.423Z 20 400-419 76 76 76 132 132 132 F F F
87842024-02-12T01:38:14.423Z 21 420-439 74 74 74 131 131 131 F F F
87852024-02-12T01:38:14.423Z 22 440-459 74 74 74 131 131 131 F F F
87862024-02-12T01:38:14.423Z 23 460-479 74 74 74 131 131 131 F F F
87872024-02-12T01:38:14.423Z 24 480-499 76 76 76 132 132 132 F F F
87882024-02-12T01:38:14.423Z 25 500-519 78 78 77 134 134 133 F F F <---
87892024-02-12T01:38:14.423Z 26 520-539 78 78 76 135 135 132 T T F <---
87902024-02-12T01:38:14.423Z 27 540-559 77 77 77 133 133 133 F F F
87912024-02-12T01:38:14.423Z 28 560-579 76 76 76 132 132 132 F F F
87922024-02-12T01:38:14.423Z 29 580-599 67 67 67 117 117 117 T T F <---
87932024-02-12T01:38:14.423ZMax gen: 78, Max flush: 135
87942024-02-12T01:38:14.423ZError: Difference in extent metadata found!
87952024-02-12T01:38:14.423ZA Difference in extent metadata is expected here
87962024-02-12T01:38:14.423ZOn loop 39
87972024-02-12T01:38:14.423Z
87982024-02-12T01:38:14.427ZVerifying data now
87992024-02-12T01:38:19.020ZNo change after verify
88002024-02-12T01:38:19.020ZLoop: 39 Downstairs dump after verify (and repair):
88012024-02-12T01:38:19.036ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88022024-02-12T01:38:19.036Z 0 000-019 72 72 72 126 126 126 F F F
88032024-02-12T01:38:19.036Z 1 020-039 78 78 78 134 134 134 F F F
88042024-02-12T01:38:19.036Z 2 040-059 78 78 78 135 135 135 F F F
88052024-02-12T01:38:19.036Z 3 060-079 72 72 72 126 126 126 F F F
88062024-02-12T01:38:19.036Z 4 080-099 76 76 76 132 132 132 F F F
88072024-02-12T01:38:19.036Z 5 100-119 78 78 78 135 135 135 F F F
88082024-02-12T01:38:19.036Z 6 120-139 78 78 78 135 135 135 F F F
88092024-02-12T01:38:19.036Z 7 140-159 70 70 70 124 124 124 F F F
88102024-02-12T01:38:19.036Z 8 160-179 78 78 78 135 135 135 F F F
88112024-02-12T01:38:19.036Z 9 180-199 64 64 64 110 110 110 F F F
88122024-02-12T01:38:19.039Z 10 200-219 70 70 70 124 124 124 F F F
88132024-02-12T01:38:19.039Z 11 220-239 73 73 73 130 130 130 F F F
88142024-02-12T01:38:19.039Z 12 240-259 78 78 78 135 135 135 F F F
88152024-02-12T01:38:19.039Z 13 260-279 78 78 78 135 135 135 F F F
88162024-02-12T01:38:19.039Z 14 280-299 76 76 76 132 132 132 F F F
88172024-02-12T01:38:19.039Z 15 300-319 76 76 76 132 132 132 F F F
88182024-02-12T01:38:19.039Z 16 320-339 74 74 74 131 131 131 F F F
88192024-02-12T01:38:19.040Z 17 340-359 78 78 78 135 135 135 F F F
88202024-02-12T01:38:19.040Z 18 360-379 78 78 78 135 135 135 F F F
88212024-02-12T01:38:19.040Z 19 380-399 73 73 73 130 130 130 F F F
88222024-02-12T01:38:19.040Z 20 400-419 76 76 76 132 132 132 F F F
88232024-02-12T01:38:19.040Z 21 420-439 74 74 74 131 131 131 F F F
88242024-02-12T01:38:19.040Z 22 440-459 74 74 74 131 131 131 F F F
88252024-02-12T01:38:19.040Z 23 460-479 74 74 74 131 131 131 F F F
88262024-02-12T01:38:19.040Z 24 480-499 76 76 76 132 132 132 F F F
88272024-02-12T01:38:19.040Z 25 500-519 78 78 78 134 134 134 F F F
88282024-02-12T01:38:19.040Z 26 520-539 79 79 79 136 136 136 F F F
88292024-02-12T01:38:19.040Z 27 540-559 77 77 77 133 133 133 F F F
88302024-02-12T01:38:19.040Z 28 560-579 76 76 76 132 132 132 F F F
88312024-02-12T01:38:19.040Z 29 580-599 79 79 79 136 136 136 F F F
88322024-02-12T01:38:19.040ZMax gen: 79, Max flush: 136
88332024-02-12T01:38:19.040Z
88342024-02-12T01:38:19.040ZBegin loop 40
88352024-02-12T01:38:19.040ZDownstairs to restart: 1
88362024-02-12T01:38:19.057Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952295195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
88372024-02-12T01:38:19.057Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
88382024-02-12T01:38:19.057Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952631108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
88392024-02-12T01:38:19.062Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952681826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
88402024-02-12T01:38:19.062Z{"msg":"Crucible af4f7770-6e28-4a54-820d-32d704d73e02 has session id: 26b43e44-a560-49e7-ab22-d1086eb5b83b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952710235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88412024-02-12T01:38:19.062Z{"msg":"Upstairs opts: Upstairs UUID: af4f7770-6e28-4a54-820d-32d704d73e02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952756182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88422024-02-12T01:38:19.062Z{"msg":"Crucible stats registered with UUID: af4f7770-6e28-4a54-820d-32d704d73e02","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952783341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88432024-02-12T01:38:19.062Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.95281327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
88442024-02-12T01:38:19.062Z{"msg":"af4f7770-6e28-4a54-820d-32d704d73e02 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952832229Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88452024-02-12T01:38:19.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952860008Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88462024-02-12T01:38:19.062Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952885106Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88472024-02-12T01:38:19.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952907255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88482024-02-12T01:38:19.062Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952931534Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88492024-02-12T01:38:19.062Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952963253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88502024-02-12T01:38:19.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.952987102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88512024-02-12T01:38:19.063Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953006101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88522024-02-12T01:38:19.063Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953038959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88532024-02-12T01:38:19.063Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953078917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88542024-02-12T01:38:19.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953117265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88552024-02-12T01:38:19.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953142894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88562024-02-12T01:38:19.063Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953284057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88572024-02-12T01:38:19.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953875589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88582024-02-12T01:38:19.063Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953917107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88592024-02-12T01:38:19.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953935506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88602024-02-12T01:38:19.063Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.953954505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88612024-02-12T01:38:19.063Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.954555117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88622024-02-12T01:38:19.063Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.954608484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88632024-02-12T01:38:19.063Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955034634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88642024-02-12T01:38:19.063Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955086321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88652024-02-12T01:38:19.063Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955207905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88662024-02-12T01:38:19.063Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955251783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88672024-02-12T01:38:19.063Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955286502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88682024-02-12T01:38:19.063Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.95532044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88692024-02-12T01:38:19.063Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955360718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88702024-02-12T01:38:19.063Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955394527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88712024-02-12T01:38:19.063Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.955611046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88722024-02-12T01:38:19.068Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956027317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88732024-02-12T01:38:19.068Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956065135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88742024-02-12T01:38:19.068Z{"msg":"[0]R flush_numbers[0..12]: [126, 134, 135, 126, 132, 135, 135, 124, 135, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956082134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88752024-02-12T01:38:19.068Z{"msg":"[0]R generation[0..12]: [72, 78, 78, 72, 76, 78, 78, 70, 78, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956104703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88762024-02-12T01:38:19.068Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956119822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88772024-02-12T01:38:19.068Z{"msg":"[1]R flush_numbers[0..12]: [126, 134, 135, 126, 132, 135, 135, 124, 135, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956140881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88782024-02-12T01:38:19.068Z{"msg":"[1]R generation[0..12]: [72, 78, 78, 72, 76, 78, 78, 70, 78, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.95615499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88792024-02-12T01:38:19.068Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956175969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88802024-02-12T01:38:19.068Z{"msg":"[2]R flush_numbers[0..12]: [126, 134, 135, 126, 132, 135, 135, 124, 135, 110, 124, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956190359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88812024-02-12T01:38:19.068Z{"msg":"[2]R generation[0..12]: [72, 78, 78, 72, 76, 78, 78, 70, 78, 64, 70, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956216837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,Wait for a query_work_queue command to finish before sending IO
88822024-02-12T01:38:19.068Z"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88832024-02-12T01:38:19.068Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956235887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88842024-02-12T01:38:19.068Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956249166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88852024-02-12T01:38:19.068Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956262565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88862024-02-12T01:38:19.069Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956275475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88872024-02-12T01:38:19.069Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956303773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88882024-02-12T01:38:19.069Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956327182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88892024-02-12T01:38:19.069Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956340771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88902024-02-12T01:38:19.069Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956354201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88912024-02-12T01:38:19.069Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.95638316Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88922024-02-12T01:38:19.069Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956405219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88932024-02-12T01:38:19.069Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956435487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88942024-02-12T01:38:19.069Z{"msg":"af4f7770-6e28-4a54-820d-32d704d73e02 is now active with session: 26b43e44-a560-49e7-ab22-d1086eb5b83b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956458216Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88952024-02-12T01:38:19.069Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956488115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455,"session_id":"26b43e44-a560-49e7-ab22-d1086eb5b83b"}
88962024-02-12T01:38:19.069Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:17.956507883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
88972024-02-12T01:38:19.069ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88982024-02-12T01:38:19.069ZLoading write count information from file "/tmp/test_repair_verify.data"
88992024-02-12T01:38:19.069ZRun Repair workload
89002024-02-12T01:38:19.069Z01/30 Write block 416 len 12288 data: 8 7 5
89012024-02-12T01:38:19.069Z02/30 Write block 427 len 16384 data: 3 3 3 3
89022024-02-12T01:38:19.069Z03/30 Read block 64 len 40960
89032024-02-12T01:38:19.069Z04/30 Write block 97 len 16384 data: 6 7 5 4
89042024-02-12T01:38:19.069Z05/30 Read block 35 len 36864
89052024-02-12T01:38:19.075Z06/30 Write block 88 len 16384 data: 10 10 8 8
89062024-02-12T01:38:19.075Z07/30 Read block 580 len 32768
89072024-02-12T01:38:19.076Z08/30 Write block 167 len 40960 data: 7 7 7 7 6 5 7 6 10 11
89082024-02-12T01:38:19.076Z09/30 Read block 563 len 28672
89092024-02-12T01:38:19.076Z10/30 Read block 329 len 20480
89102024-02-12T01:38:19.076Z11/30 Read block 518 len 32768
89112024-02-12T01:38:19.078Z12/30 Read block 138 len 24576
89122024-02-12T01:38:19.079Z13/30 Write block 220 len 20480 data: 8 7 7 7 6
89132024-02-12T01:38:19.079Z14/30 Write block 28 len 24576 data: 7 9 10 9 11 11
89142024-02-12T01:38:19.079Z15/30 Read block 337 len 36864
89152024-02-12T01:38:19.079Z16/30 Write block 196 len 28672 data: 3 5 5 6 5 5 5
89162024-02-12T01:38:19.079Z17/30 Read block 97 len 32768
89172024-02-12T01:38:19.082Z18/30 Read block 297 len 28672
89182024-02-12T01:38:19.082Z19/30 Read block 279 len 24576
89192024-02-12T01:38:19.082Z20/30 Read block 42 len 40960
89202024-02-12T01:38:19.085Z21/30 Write block 38 len 16384 data: 8 7 7 7
89212024-02-12T01:38:19.085Z22/30 Read block 98 len 4096
89222024-02-12T01:38:19.085Z23/30 Read block 239 len 20480
89232024-02-12T01:38:19.085Z24/30 Read block 355 len 8192
89242024-02-12T01:38:19.085Z25/30 Read block 296 len 32768
89252024-02-12T01:38:19.088Z26/30 Read block 103 len 8192
89262024-02-12T01:38:19.088Z27/30 Write block 367 len 36864 data: 9 7 7 7 7 8 9 8 10
89272024-02-12T01:38:19.088Z28/30 Read block 60 len 40960
89282024-02-12T01:38:19.091Z29/30 Write block 251 len 20480 data: 5 4 5 4 4
89292024-02-12T01:38:19.091Z30/30 Read block 34 len 20480
89302024-02-12T01:38:19.091Z----------------------------------------------------------------
89312024-02-12T01:38:19.091Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
89322024-02-12T01:38:19.091ZStates: Active Active Active
89332024-02-12T01:38:19.091ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89342024-02-12T01:38:19.092Z 1 Acked 1000 Write 3 Done Done Done false
89352024-02-12T01:38:19.092Z 2 Acked 1001 Write 4 Done Done Done false
89362024-02-12T01:38:19.092Z 3 Acked 1002 Read 10 Done Sent Done false
89372024-02-12T01:38:19.092Z 4 Acked 1003 Write 4 Done Sent Done false
89382024-02-12T01:38:19.092Z 5 Acked 1004 Read 9 Done Sent Done false
89392024-02-12T01:38:19.092Z 6 Acked 1005 Write 4 Done Sent Done false
89402024-02-12T01:38:19.092Z 7 Acked 1006 Read 8 Done Sent Done false
89412024-02-12T01:38:19.092Z 8 Acked 1007 Write 10 Done Sent Done false
89422024-02-12T01:38:19.092Z 9 Acked 1008 Read 7 Done Sent Done false
89432024-02-12T01:38:19.092Z 10 Acked 1009 Read 5 Done Sent Done false
89442024-02-12T01:38:19.092Z 11 Acked 1010 Read 8 Done Sent Done false
89452024-02-12T01:38:19.092Z 12 Acked 1011 Read 6 Done Sent Done false
89462024-02-12T01:38:19.092Z 13 Acked 1012 Write 5 Done Sent Done false
89472024-02-12T01:38:19.092Z 14 Acked 1013 Write 6 Done Sent Done false
89482024-02-12T01:38:19.092Z 15 Acked 1014 Read 9 Done Sent Done false
89492024-02-12T01:38:19.092Z 16 Acked 1015 Write 7 Done Sent Done false
89502024-02-12T01:38:19.092Z 17 Acked 1016 Read 8 Done Sent Done false
89512024-02-12T01:38:19.092Z 18 Acked 1017 Read 7 Done Sent Done false
89522024-02-12T01:38:19.092Z 19 Acked 1018 Read 6 Done Sent Done false
89532024-02-12T01:38:19.092Z 20 Acked 1019 Read 10 Done Sent Done false
89542024-02-12T01:38:19.092Z 21 Acked 1020 Write 4 Done Sent Done false
89552024-02-12T01:38:19.093Z 22 Acked 1021 Read 1 Done Sent Done false
89562024-02-12T01:38:19.093Z 23 Acked 1022 Read 5 Done Sent Done false
89572024-02-12T01:38:19.093Z 24 Acked 1023 Read 2 Done Sent Done false
89582024-02-12T01:38:19.093Z 25 Acked 1024 Read 8 Done Sent Done false
89592024-02-12T01:38:19.093Z 26 Acked 1025 Read 2 Done Sent Done false
89602024-02-12T01:38:19.093Z 27 Acked 1026 Write 9 Done Sent Done false
89612024-02-12T01:38:19.093Z 28 Acked 1027 Read 10 Done Sent Done false
89622024-02-12T01:38:19.093Z 29 Acked 1028 Write 5 Done Sent Done false
89632024-02-12T01:38:19.093Z 30 Acked 1029 Read 5 Done Sent Sent false
89642024-02-12T01:38:19.093Z STATES DS:0 DS:1 DS:2 TOTAL
89652024-02-12T01:38:19.093Z New 0 0 0 0
89662024-02-12T01:38:19.093Z Sent 0 28 1 29
89672024-02-12T01:38:19.093Z Done 30 2 29 61
89682024-02-12T01:38:19.093Z Skipped 0 0 0 0
89692024-02-12T01:38:19.093Z Error 0 0 0 0
89702024-02-12T01:38:19.093ZLast Flush: 0 0 0
89712024-02-12T01:38:19.093ZDownstairs last five completed:
89722024-02-12T01:38:19.093ZUpstairs last five completed: 30 29 28 27 26
89732024-02-12T01:38:19.093Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:17.985572847Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1455}
89742024-02-12T01:38:19.093ZWrote out file "/tmp/test_repair_verify.data"
89752024-02-12T01:38:19.096Z
89762024-02-12T01:38:19.099ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89772024-02-12T01:38:19.117ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89782024-02-12T01:38:19.117Z 0 000-019 72 72 72 126 126 126 F F F
89792024-02-12T01:38:19.117Z 1 020-039 78 78 78 134 134 134 T F T <---
89802024-02-12T01:38:19.117Z 2 040-059 78 78 78 135 135 135 T F T <---
89812024-02-12T01:38:19.117Z 3 060-079 72 72 72 126 126 126 F F F
89822024-02-12T01:38:19.117Z 4 080-099 76 76 76 132 132 132 T F T <---
89832024-02-12T01:38:19.117Z 5 100-119 78 78 78 135 135 135 T F T <---
89842024-02-12T01:38:19.117Z 6 120-139 78 78 78 135 135 135 F F F
89852024-02-12T01:38:19.120Z 7 140-159 70 70 70 124 124 124 F F F
89862024-02-12T01:38:19.120Z 8 160-179 78 78 78 135 135 135 T F T <---
89872024-02-12T01:38:19.120Z 9 180-199 64 64 64 110 110 110 T F T <---
89882024-02-12T01:38:19.121Z 10 200-219 70 70 70 124 124 124 T F T <---
89892024-02-12T01:38:19.121Z 11 220-239 73 73 73 130 130 130 T F T <---
89902024-02-12T01:38:19.121Z 12 240-259 78 78 78 135 135 135 T F T <---
89912024-02-12T01:38:19.121Z 13 260-279 78 78 78 135 135 135 F F F
89922024-02-12T01:38:19.121Z 14 280-299 76 76 76 132 132 132 F F F
89932024-02-12T01:38:19.121Z 15 300-319 76 76 76 132 132 132 F F F
89942024-02-12T01:38:19.121Z 16 320-339 74 74 74 131 131 131 F F F
89952024-02-12T01:38:19.121Z 17 340-359 78 78 78 135 135 135 F F F
89962024-02-12T01:38:19.121Z 18 360-379 78 78 78 135 135 135 T F T <---
89972024-02-12T01:38:19.121Z 19 380-399 73 73 73 130 130 130 F F F
89982024-02-12T01:38:19.121Z 20 400-419 76 76 76 132 132 132 T T T
89992024-02-12T01:38:19.121Z 21 420-439 74 74 74 131 131 131 T T T
90002024-02-12T01:38:19.121Z 22 440-459 74 74 74 131 131 131 F F F
90012024-02-12T01:38:19.121Z 23 460-479 74 74 74 131 131 131 F F F
90022024-02-12T01:38:19.121Z 24 480-499 76 76 76 132 132 132 F F F
90032024-02-12T01:38:19.121Z 25 500-519 78 78 78 134 134 134 F F F
90042024-02-12T01:38:19.121Z 26 520-539 79 79 79 136 136 136 F F F
90052024-02-12T01:38:19.121Z 27 540-559 77 77 77 133 133 133 F F F
90062024-02-12T01:38:19.121Z 28 560-579 76 76 76 132 132 132 F F F
90072024-02-12T01:38:19.121Z 29 580-599 79 79 79 136 136 136 F F F
90082024-02-12T01:38:19.121ZMax gen: 79, Max flush: 136
90092024-02-12T01:38:19.121ZError: Difference in extent metadata found!
90102024-02-12T01:38:19.121ZA Difference in extent metadata is expected here
90112024-02-12T01:38:19.121ZOn loop 40
90122024-02-12T01:38:19.122Z
90132024-02-12T01:38:19.125ZVerifying data now
90142024-02-12T01:38:24.743ZNo change after verify
90152024-02-12T01:38:24.743ZLoop: 40 Downstairs dump after verify (and repair):
90162024-02-12T01:38:24.761ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90172024-02-12T01:38:24.761Z 0 000-019 72 72 72 126 126 126 F F F
90182024-02-12T01:38:24.761Z 1 020-039 80 80 80 137 137 137 F F F
90192024-02-12T01:38:24.761Z 2 040-059 80 80 80 137 137 137 F F F
90202024-02-12T01:38:24.761Z 3 060-079 72 72 72 126 126 126 F F F
90212024-02-12T01:38:24.761Z 4 080-099 80 80 80 137 137 137 F F F
90222024-02-12T01:38:24.765Z 5 100-119 80 80 80 137 137 137 F F F
90232024-02-12T01:38:24.765Z 6 120-139 78 78 78 135 135 135 F F F
90242024-02-12T01:38:24.765Z 7 140-159 70 70 70 124 124 124 F F F
90252024-02-12T01:38:24.765Z 8 160-179 80 80 80 137 137 137 F F F
90262024-02-12T01:38:24.765Z 9 180-199 80 80 80 137 137 137 F F F
90272024-02-12T01:38:24.765Z 10 200-219 80 80 80 137 137 137 F F F
90282024-02-12T01:38:24.765Z 11 220-239 80 80 80 137 137 137 F F F
90292024-02-12T01:38:24.765Z 12 240-259 80 80 80 137 137 137 F F F
90302024-02-12T01:38:24.765Z 13 260-279 78 78 78 135 135 135 F F F
90312024-02-12T01:38:24.765Z 14 280-299 76 76 76 132 132 132 F F F
90322024-02-12T01:38:24.765Z 15 300-319 76 76 76 132 132 132 F F F
90332024-02-12T01:38:24.765Z 16 320-339 74 74 74 131 131 131 F F F
90342024-02-12T01:38:24.765Z 17 340-359 78 78 78 135 135 135 F F F
90352024-02-12T01:38:24.765Z 18 360-379 80 80 80 137 137 137 F F F
90362024-02-12T01:38:24.765Z 19 380-399 73 73 73 130 130 130 F F F
90372024-02-12T01:38:24.765Z 20 400-419 80 80 80 137 137 137 F F F
90382024-02-12T01:38:24.765Z 21 420-439 80 80 80 137 137 137 F F F
90392024-02-12T01:38:24.765Z 22 440-459 74 74 74 131 131 131 F F F
90402024-02-12T01:38:24.765Z 23 460-479 74 74 74 131 131 131 F F F
90412024-02-12T01:38:24.766Z 24 480-499 76 76 76 132 132 132 F F F
90422024-02-12T01:38:24.766Z 25 500-519 78 78 78 134 134 134 F F F
90432024-02-12T01:38:24.766Z 26 520-539 79 79 79 136 136 136 F F F
90442024-02-12T01:38:24.766Z 27 540-559 77 77 77 133 133 133 F F F
90452024-02-12T01:38:24.766Z 28 560-579 76 76 76 132 132 132 F F F
90462024-02-12T01:38:24.766Z 29 580-599 79 79 79 136 136 136 F F F
90472024-02-12T01:38:24.766ZMax gen: 80, Max flush: 137
90482024-02-12T01:38:24.766Z
90492024-02-12T01:38:24.766ZBegin loop 41
90502024-02-12T01:38:24.766ZDownstairs to restart: 2
90512024-02-12T01:38:24.782Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.676961135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
90522024-02-12T01:38:24.787Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677253011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
90532024-02-12T01:38:24.787Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
90542024-02-12T01:38:24.787Z,"time":"2024-02-12T01:38:23.677320338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
90552024-02-12T01:38:24.787Z{"msg":"Crucible d93d27fb-6a57-4d44-9d55-01fa8ab6b416 has session id: 32dccb53-655b-4578-b52b-75821d71b6f3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677452522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90562024-02-12T01:38:24.787Z{"msg":"Upstairs opts: Upstairs UUID: d93d27fb-6a57-4d44-9d55-01fa8ab6b416, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677497039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90572024-02-12T01:38:24.787Z{"msg":"Crucible stats registered with UUID: d93d27fb-6a57-4d44-9d55-01fa8ab6b416","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677529058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90582024-02-12T01:38:24.788Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677551907Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
90592024-02-12T01:38:24.788Z{"msg":"d93d27fb-6a57-4d44-9d55-01fa8ab6b416 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677578036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90602024-02-12T01:38:24.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677600405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90612024-02-12T01:38:24.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677628793Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90622024-02-12T01:38:24.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677653242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90632024-02-12T01:38:24.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.67769364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90642024-02-12T01:38:24.788Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677736478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90652024-02-12T01:38:24.788Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677782266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90662024-02-12T01:38:24.788Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677827654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90672024-02-12T01:38:24.788Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677879191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90682024-02-12T01:38:24.788Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677918679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90692024-02-12T01:38:24.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.677960528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90702024-02-12T01:38:24.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.678000906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90712024-02-12T01:38:24.788Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.678072382Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90722024-02-12T01:38:24.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.678587727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90732024-02-12T01:38:24.788Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.678675013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90742024-02-12T01:38:24.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.678725731Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90752024-02-12T01:38:24.788Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.67874894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90762024-02-12T01:38:24.788Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679253896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90772024-02-12T01:38:24.788Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.67938074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90782024-02-12T01:38:24.788Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679654577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90792024-02-12T01:38:24.788Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679708564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90802024-02-12T01:38:24.788Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679875056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90812024-02-12T01:38:24.788Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679914044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90822024-02-12T01:38:24.788Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679940253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90832024-02-12T01:38:24.789Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.679967242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90842024-02-12T01:38:24.789Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.67999164Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90852024-02-12T01:38:24.789Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680023749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90862024-02-12T01:38:24.789Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680233559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90872024-02-12T01:38:24.793Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680736175Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90882024-02-12T01:38:24.793Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680777723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90892024-02-12T01:38:24.793Z{"msg":"[0]R flush_numbers[0..12]: [126, 137, 137, 126, 137, 137, 135, 124, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680794662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90902024-02-12T01:38:24.793Z{"msg":"[0]R generation[0..12]: [72, 80, 80, 72, 80, 80, 78, 70, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680818451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90912024-02-12T01:38:24.793Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.68084812Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90922024-02-12T01:38:24.793Z{"msg":"[1]R flush_numbers[0..12]: [126, 137, 137, 126, 137, 137, 135, 124, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680863599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90932024-02-12T01:38:24.793Z{"msg":"[1]R generation[0..12]: [72, 80, 80, 72, 80, 80, 78, 70, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680895577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90942024-02-12T01:38:24.793Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680914476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90952024-02-12T01:38:24.794Z{"msg":"[2]R flush_numbers[0..12]: [126, 137, 137, 126, 137, 137, 135, 124, 137, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680946585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
90962024-02-12T01:38:24.794Zdownstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90972024-02-12T01:38:24.794Z{"msg":"[2]R generation[0..12]: [72, 80, 80, 72, 80, 80, 78, 70, 80, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680965294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90982024-02-12T01:38:24.794Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.680984463Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
90992024-02-12T01:38:24.794Z{"msg":"Max found gen is 81","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681014982Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91002024-02-12T01:38:24.794Z{"msg":"Generation requested: 82 >= found:81","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681030361Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91012024-02-12T01:38:24.794Z{"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.68104414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91022024-02-12T01:38:24.794Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681070299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91032024-02-12T01:38:24.794Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681084159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91042024-02-12T01:38:24.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681096348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91052024-02-12T01:38:24.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681110497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91062024-02-12T01:38:24.794Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681124557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91072024-02-12T01:38:24.794Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681138656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91082024-02-12T01:38:24.794Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681158385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91092024-02-12T01:38:24.794Z{"msg":"d93d27fb-6a57-4d44-9d55-01fa8ab6b416 is now active with session: 32dccb53-655b-4578-b52b-75821d71b6f3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681172434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91102024-02-12T01:38:24.794Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681184574Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463,"session_id":"32dccb53-655b-4578-b52b-75821d71b6f3"}
91112024-02-12T01:38:24.794Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:23.681203703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
91122024-02-12T01:38:24.794ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91132024-02-12T01:38:24.794ZLoading write count information from file "/tmp/test_repair_verify.data"
91142024-02-12T01:38:24.794ZRun Repair workload
91152024-02-12T01:38:24.794Z01/30 Write block 464 len 24576 data: 5 5 7 9 9 8
91162024-02-12T01:38:24.794Z02/30 Read block 96 len 16384
91172024-02-12T01:38:24.794Z03/30 Flush
91182024-02-12T01:38:24.798Z04/30 Write block 430 len 36864 data: 4 2 2 2 3 3 3 3 4
91192024-02-12T01:38:24.798Z05/30 Read block 29 len 16384
91202024-02-12T01:38:24.798Z06/30 Write block 445 len 24576 data: 6 5 7 7 7 7
91212024-02-12T01:38:24.798Z07/30 Read block 453 len 40960
91222024-02-12T01:38:24.798Z08/30 Read block 506 len 28672
91232024-02-12T01:38:24.801Z09/30 Flush
91242024-02-12T01:38:24.801Z10/30 Write block 27 len 20480 data: 6 8 10 11 10
91252024-02-12T01:38:24.803Z11/30 Write block 523 len 32768 data: 8 8 5 5 3 4 5 8
91262024-02-12T01:38:24.803Z12/30 Read block 312 len 32768
91272024-02-12T01:38:24.803Z13/30 Read block 548 len 12288
91282024-02-12T01:38:24.807Z14/30 Read block 507 len 36864
91292024-02-12T01:38:24.807Z15/30 Write block 291 len 8192 data: 7 5
91302024-02-12T01:38:24.807Z16/30 Read block 374 len 4096
91312024-02-12T01:38:24.807Z17/30 Flush
91322024-02-12T01:38:24.810Z18/30 Write block 126 len 8192 data: 11 10
91332024-02-12T01:38:24.810Z19/30 Write block 33 len 36864 data: 12 11 7 8 6 9 8 8 8
91342024-02-12T01:38:24.813Z20/30 Write block 160 len 16384 data: 5 6 6 6
91352024-02-12T01:38:24.813Z21/30 Flush
91362024-02-12T01:38:24.816Z22/30 Write block 265 len 28672 data: 6 7 7 6 4 4 5
91372024-02-12T01:38:24.816Z23/30 Read block 195 len 8192
91382024-02-12T01:38:24.816Z24/30 Write block 572 len 12288 data: 10 12 11
91392024-02-12T01:38:24.819Z25/30 Write block 477 len 4096 data: 5
91402024-02-12T01:38:24.819Z26/30 Read block 159 len 28672
91412024-02-12T01:38:24.819Z27/30 Read block 98 len 12288
91422024-02-12T01:38:24.819Z28/30 Read block 568 len 12288
91432024-02-12T01:38:24.822Z29/30 Write block 390 len 32768 data: 11 11 10 9 6 6 5 5
91442024-02-12T01:38:24.822Z30/30 Write block 66 len 40960 data: 7 8 7 9 8 10 7 6 6 7
91452024-02-12T01:38:24.823Z----------------------------------------------------------------
91462024-02-12T01:38:24.823Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:27
91472024-02-12T01:38:24.823ZStates: Active Active Active
91482024-02-12T01:38:24.823ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91492024-02-12T01:38:24.823Z 4 Acked 1003 Write 9 Done Done Done false
91502024-02-12T01:38:24.823Z 5 Acked 1004 Read 4 Done Done Done false
91512024-02-12T01:38:24.823Z 6 Acked 1005 Write 6 Done Done Done false
91522024-02-12T01:38:24.823Z 7 Acked 1006 Read 10 Done Done Done false
91532024-02-12T01:38:24.823Z 8 Acked 1007 Read 7 Done Done Sent false
91542024-02-12T01:38:24.823Z 9 Acked 1008 Flush 0 Done Done Sent false
91552024-02-12T01:38:24.823Z 10 Acked 1009 Write 5 Done Done Sent false
91562024-02-12T01:38:24.823Z 11 Acked 1010 Write 8 Done Done Sent false
91572024-02-12T01:38:24.823Z 12 Acked 1011 Read 8 Done Done Sent false
91582024-02-12T01:38:24.823Z 13 Acked 1012 Read 3 Done Done Sent false
91592024-02-12T01:38:24.823Z 14 Acked 1013 Read 9 Done Done Sent false
91602024-02-12T01:38:24.823Z 15 Acked 1014 Write 2 Done Done Sent false
91612024-02-12T01:38:24.823Z 16 Acked 1015 Read 1 Done Done Sent false
91622024-02-12T01:38:24.823Z 17 Acked 1016 Flush 0 Done Done Sent false
91632024-02-12T01:38:24.823Z 18 Acked 1017 Write 2 Done Done Sent false
91642024-02-12T01:38:24.823Z 19 Acked 1018 Write 9 Done Done Sent false
91652024-02-12T01:38:24.823Z 20 Acked 1019 Write 4 Done Done Sent false
91662024-02-12T01:38:24.823Z 21 Acked 1020 Flush 0 Done Done Sent false
91672024-02-12T01:38:24.823Z 22 Acked 1021 Write 7 Done Done Sent false
91682024-02-12T01:38:24.823Z 23 Acked 1022 Read 2 Done Done Sent false
91692024-02-12T01:38:24.824Z 24 Acked 1023 Write 3 Done Done Sent false
91702024-02-12T01:38:24.824Z 25 Acked 1024 Write 1 Done Done Sent false
91712024-02-12T01:38:24.824Z 26 Acked 1025 Read 7 Done Done Sent false
91722024-02-12T01:38:24.824Z 27 Acked 1026 Read 3 Done Done Sent false
91732024-02-12T01:38:24.824Z 28 Acked 1027 Read 3 Done Done Sent false
91742024-02-12T01:38:24.824Z 29 Acked 1028 Write 8 Done Done Sent false
91752024-02-12T01:38:24.824Z 30 Acked 1029 Write 10 Sent Sent Sent false
91762024-02-12T01:38:24.824Z STATES DS:0 DS:1 DS:2 TOTAL
91772024-02-12T01:38:24.824Z New 0 0 0 0
91782024-02-12T01:38:24.824Z Sent 1 1 23 25
91792024-02-12T01:38:24.824Z Done 26 26 4 56
91802024-02-12T01:38:24.824Z Skipped 0 0 0 0
91812024-02-12T01:38:24.824Z Error 0 0 0 0
91822024-02-12T01:38:24.824ZLast Flush: 1020 1020 1002
91832024-02-12T01:38:24.824ZDownstairs last five completed: 1002 1001 1000
91842024-02-12T01:38:24.824ZUpstairs last five completed: 30 29 28 27 26
91852024-02-12T01:38:24.824Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:23.716123477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1463}
91862024-02-12T01:38:24.824ZWrote out file "/tmp/test_repair_verify.data"
91872024-02-12T01:38:24.828Z
91882024-02-12T01:38:24.834ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91892024-02-12T01:38:24.848ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91902024-02-12T01:38:24.848Z 0 000-019 72 72 72 126 126 126 F F F
91912024-02-12T01:38:24.848Z 1 020-039 82 82 80 141 141 137 F F F <---
91922024-02-12T01:38:24.848Z 2 040-059 82 82 80 141 141 137 F F F <---
91932024-02-12T01:38:24.848Z 3 060-079 72 72 72 126 126 126 T T F <---
91942024-02-12T01:38:24.848Z 4 080-099 80 80 80 137 137 137 F F F
91952024-02-12T01:38:24.848Z 5 100-119 80 80 80 137 137 137 F F F
91962024-02-12T01:38:24.848Z 6 120-139 82 82 78 141 141 135 F F F <---
91972024-02-12T01:38:24.848Z 7 140-159 70 70 70 124 124 124 F F F
91982024-02-12T01:38:24.848Z 8 160-179 82 82 80 141 141 137 F F F <---
91992024-02-12T01:38:24.848Z 9 180-199 80 80 80 137 137 137 F F F
92002024-02-12T01:38:24.851Z 10 200-219 80 80 80 137 137 137 F F F
92012024-02-12T01:38:24.851Z 11 220-239 80 80 80 137 137 137 F F F
92022024-02-12T01:38:24.851Z 12 240-259 80 80 80 137 137 137 F F F
92032024-02-12T01:38:24.851Z 13 260-279 78 78 78 135 135 135 T T F <---
92042024-02-12T01:38:24.852Z 14 280-299 82 82 76 140 140 132 F F F <---
92052024-02-12T01:38:24.852Z 15 300-319 76 76 76 132 132 132 F F F
92062024-02-12T01:38:24.852Z 16 320-339 74 74 74 131 131 131 F F F
92072024-02-12T01:38:24.852Z 17 340-359 78 78 78 135 135 135 F F F
92082024-02-12T01:38:24.852Z 18 360-379 80 80 80 137 137 137 F F F
92092024-02-12T01:38:24.852Z 19 380-399 73 73 73 130 130 130 T T F <---
92102024-02-12T01:38:24.852Z 20 400-419 80 80 80 137 137 137 F F F
92112024-02-12T01:38:24.852Z 21 420-439 82 82 80 139 139 137 F F T <---
92122024-02-12T01:38:24.852Z 22 440-459 82 82 74 139 139 131 F F T <---
92132024-02-12T01:38:24.852Z 23 460-479 82 82 82 138 138 138 T T F <---
92142024-02-12T01:38:24.852Z 24 480-499 76 76 76 132 132 132 F F F
92152024-02-12T01:38:24.852Z 25 500-519 78 78 78 134 134 134 F F F
92162024-02-12T01:38:24.852Z 26 520-539 82 82 79 140 140 136 F F F <---
92172024-02-12T01:38:24.852Z 27 540-559 77 77 77 133 133 133 F F F
92182024-02-12T01:38:24.852Z 28 560-579 76 76 76 132 132 132 T T F <---
92192024-02-12T01:38:24.852Z 29 580-599 79 79 79 136 136 136 F F F
92202024-02-12T01:38:24.852ZMax gen: 82, Max flush: 141
92212024-02-12T01:38:24.852ZError: Difference in extent metadata found!
92222024-02-12T01:38:24.852ZA Difference in extent metadata is expected here
92232024-02-12T01:38:24.852ZOn loop 41
92242024-02-12T01:38:24.852Z
92252024-02-12T01:38:24.855ZVerifying data now
92262024-02-12T01:38:30.494ZNo change after verify
92272024-02-12T01:38:30.494ZLoop: 41 Downstairs dump after verify (and repair):
92282024-02-12T01:38:30.509ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92292024-02-12T01:38:30.509Z 0 000-019 72 72 72 126 126 126 F F F
92302024-02-12T01:38:30.509Z 1 020-039 82 82 82 141 141 141 F F F
92312024-02-12T01:38:30.510Z 2 040-059 82 82 82 141 141 141 F F F
92322024-02-12T01:38:30.510Z 3 060-079 83 83 83 142 142 142 F F F
92332024-02-12T01:38:30.510Z 4 080-099 80 80 80 137 137 137 F F F
92342024-02-12T01:38:30.510Z 5 100-119 80 80 80 137 137 137 F F F
92352024-02-12T01:38:30.510Z 6 120-139 82 82 82 141 141 141 F F F
92362024-02-12T01:38:30.510Z 7 140-159 70 70 70 124 124 124 F F F
92372024-02-12T01:38:30.510Z 8 160-179 82 82 82 141 141 141 F F F
92382024-02-12T01:38:30.510Z 9 180-199 80 80 80 137 137 137 F F F
92392024-02-12T01:38:30.513Z 10 200-219 80 80 80 137 137 137 F F F
92402024-02-12T01:38:30.513Z 11 220-239 80 80 80 137 137 137 F F F
92412024-02-12T01:38:30.513Z 12 240-259 80 80 80 137 137 137 F F F
92422024-02-12T01:38:30.513Z 13 260-279 83 83 83 142 142 142 F F F
92432024-02-12T01:38:30.514Z 14 280-299 82 82 82 140 140 140 F F F
92442024-02-12T01:38:30.514Z 15 300-319 76 76 76 132 132 132 F F F
92452024-02-12T01:38:30.514Z 16 320-339 74 74 74 131 131 131 F F F
92462024-02-12T01:38:30.514Z 17 340-359 78 78 78 135 135 135 F F F
92472024-02-12T01:38:30.514Z 18 360-379 80 80 80 137 137 137 F F F
92482024-02-12T01:38:30.514Z 19 380-399 83 83 83 142 142 142 F F F
92492024-02-12T01:38:30.514Z 20 400-419 80 80 80 137 137 137 F F F
92502024-02-12T01:38:30.514Z 21 420-439 82 82 82 139 139 139 F F F
92512024-02-12T01:38:30.514Z 22 440-459 82 82 82 139 139 139 F F F
92522024-02-12T01:38:30.514Z 23 460-479 83 83 83 142 142 142 F F F
92532024-02-12T01:38:30.514Z 24 480-499 76 76 76 132 132 132 F F F
92542024-02-12T01:38:30.514Z 25 500-519 78 78 78 134 134 134 F F F
92552024-02-12T01:38:30.514Z 26 520-539 82 82 82 140 140 140 F F F
92562024-02-12T01:38:30.514Z 27 540-559 77 77 77 133 133 133 F F F
92572024-02-12T01:38:30.514Z 28 560-579 83 83 83 142 142 142 F F F
92582024-02-12T01:38:30.514Z 29 580-599 79 79 79 136 136 136 F F F
92592024-02-12T01:38:30.514ZMax gen: 83, Max flush: 142
92602024-02-12T01:38:30.514Z
92612024-02-12T01:38:30.514ZBegin loop 42
92622024-02-12T01:38:30.514ZDownstairs to restart: 2
92632024-02-12T01:38:30.530Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424230918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
92642024-02-12T01:38:30.530Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424466046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
92652024-02-12T01:38:30.530Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
92662024-02-12T01:38:30.530Z,"time":"2024-02-12T01:38:29.424485805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
92672024-02-12T01:38:30.535Z{"msg":"Crucible 85e5be5a-e132-46c2-94b1-fd8949bbf50c has session id: 745767db-954c-457e-8d29-2234589d40ec","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424529393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92682024-02-12T01:38:30.536Z{"msg":"Upstairs opts: Upstairs UUID: 85e5be5a-e132-46c2-94b1-fd8949bbf50c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424566411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92692024-02-12T01:38:30.536Z{"msg":"Crucible stats registered with UUID: 85e5be5a-e132-46c2-94b1-fd8949bbf50c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.42460348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92702024-02-12T01:38:30.536Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424640498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
92712024-02-12T01:38:30.536Z{"msg":"85e5be5a-e132-46c2-94b1-fd8949bbf50c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424681266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92722024-02-12T01:38:30.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424718184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92732024-02-12T01:38:30.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424756862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92742024-02-12T01:38:30.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424784031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92752024-02-12T01:38:30.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424829649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92762024-02-12T01:38:30.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424856698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92772024-02-12T01:38:30.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424883636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92782024-02-12T01:38:30.536Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424926024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92792024-02-12T01:38:30.536Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424955043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92802024-02-12T01:38:30.536Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.424979452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92812024-02-12T01:38:30.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.42502397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92822024-02-12T01:38:30.536Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:29.425051618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92832024-02-12T01:38:30.536Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425075317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92842024-02-12T01:38:30.536Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:29.425112785Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92852024-02-12T01:38:30.536Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:29.425202971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92862024-02-12T01:38:30.536Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425246749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92872024-02-12T01:38:30.536Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425274498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92882024-02-12T01:38:30.536Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425310446Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92892024-02-12T01:38:30.536Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425746565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92902024-02-12T01:38:30.537Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425779264Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92912024-02-12T01:38:30.537Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425794403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92922024-02-12T01:38:30.537Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.425824822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92932024-02-12T01:38:30.537Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426355116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92942024-02-12T01:38:30.537Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426510779Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92952024-02-12T01:38:30.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426845353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92962024-02-12T01:38:30.537Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426889521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92972024-02-12T01:38:30.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.42691357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92982024-02-12T01:38:30.537Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426945128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
92992024-02-12T01:38:30.537Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.426980376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93002024-02-12T01:38:30.537Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:29.427014975Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93012024-02-12T01:38:31.533Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.426468451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93022024-02-12T01:38:31.533Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.426550827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93032024-02-12T01:38:31.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.427402907Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93042024-02-12T01:38:31.538Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.427448244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93052024-02-12T01:38:31.538Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.427926822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93062024-02-12T01:38:31.538Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428477815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93072024-02-12T01:38:31.538Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428516684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93082024-02-12T01:38:31.538Z{"msg":"[0]R flush_numbers[0..12]: [126, 141, 141, 142, 137, 137, 141, 124, 141, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428534073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93092024-02-12T01:38:31.538Z{"msg":"[0]R generation[0..12]: [72, 82, 82, 83, 80, 80, 82, 70, 82, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428548792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93102024-02-12T01:38:31.538Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428563261Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93112024-02-12T01:38:31.538Z{"msg":"[1]R flush_numbers[0..12]: [126, 141, 141, 142, 137, 137, 141, 124, 141, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428578561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93122024-02-12T01:38:31.538Z{"msg":"[1]R generation[0..12]: [72, 82, 82, 83, 80, 80, 82, 70, 82, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428600889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93132024-02-12T01:38:31.538Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428617389Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93142024-02-12T01:38:31.538Z{"msg":"[2]R flush_numbers[0..12]: [126, 141, 141, 142, 137, 137, 141, 124, 141, 137, 137, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428640038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93152024-02-12T01:38:31.538Z{"msg":"[2]R generation[0..12]: [72, 82, 82, 83, 80, 80, 82, 70, 82, 80, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428654197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93162024-02-12T01:38:31.538Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428674876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93172024-02-12T01:38:31.539Z{"msg":"Wait for a query_work_queue command to finish before sending IO
93182024-02-12T01:38:31.539ZMax found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428692045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93192024-02-12T01:38:31.539Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428712744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93202024-02-12T01:38:31.539Z{"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428725994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93212024-02-12T01:38:31.539Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428745533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93222024-02-12T01:38:31.539Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428759522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93232024-02-12T01:38:31.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428772951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93242024-02-12T01:38:31.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428786461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93252024-02-12T01:38:31.539Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.42879957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93262024-02-12T01:38:31.539Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.42881315Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93272024-02-12T01:38:31.539Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428825439Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93282024-02-12T01:38:31.539Z{"msg":"85e5be5a-e132-46c2-94b1-fd8949bbf50c is now active with session: 745767db-954c-457e-8d29-2234589d40ec","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428846058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93292024-02-12T01:38:31.539Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428858327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471,"session_id":"745767db-954c-457e-8d29-2234589d40ec"}
93302024-02-12T01:38:31.539Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:30.428870647Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
93312024-02-12T01:38:31.539ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93322024-02-12T01:38:31.539ZLoading write count information from file "/tmp/test_repair_verify.data"
93332024-02-12T01:38:31.539ZRun Repair workload
93342024-02-12T01:38:31.539Z01/30 Flush
93352024-02-12T01:38:31.543Z02/30 Write block 79 len 40960 data: 9 11 11 12 9 10 10 9 11 11
93362024-02-12T01:38:31.543Z03/30 Flush
93372024-02-12T01:38:31.543Z04/30 Write block 590 len 12288 data: 7 5 5
93382024-02-12T01:38:31.543Z05/30 Write block 410 len 24576 data: 9 9 9 11 9 10
93392024-02-12T01:38:31.543Z06/30 Read block 127 len 8192
93402024-02-12T01:38:31.546Z07/30 Write block 533 len 20480 data: 11 9 11 10 10
93412024-02-12T01:38:31.546Z08/30 Write block 294 len 20480 data: 5 7 5 4 4
93422024-02-12T01:38:31.546Z09/30 Write block 470 len 24576 data: 7 6 8 8 8 7
93432024-02-12T01:38:31.546Z10/30 Write block 148 len 28672 data: 8 9 10 10 9 7 7
93442024-02-12T01:38:31.549Z11/30 Read block 108 len 4096
93452024-02-12T01:38:31.549Z12/30 Write block 15 len 40960 data: 8 7 6 6 5 4 4 5 5 5
93462024-02-12T01:38:31.549Z13/30 Write block 390 len 12288 data: 12 12 11
93472024-02-12T01:38:31.552Z14/30 Write block 0 len 28672 data: 2 2 3 4 5 6 6
93482024-02-12T01:38:31.552Z15/30 Write block 31 len 40960 data: 11 12 13 12 8 9 7 10 9 9
93492024-02-12T01:38:31.552Z16/30 Write block 356 len 36864 data: 6 5 5 6 7 7 6 7 9
93502024-02-12T01:38:31.552Z17/30 Flush
93512024-02-12T01:38:31.558Z18/30 Write block 158 len 36864 data: 4 4 6 7 7 7 7 9 9
93522024-02-12T01:38:31.561Z19/30 Flush
93532024-02-12T01:38:31.564Z20/30 Write block 49 len 36864 data: 8 8 8 5 3 4 5 5 7
93542024-02-12T01:38:31.564Z21/30 Flush
93552024-02-12T01:38:31.567Z22/30 Write block 381 len 24576 data: 7 6 6 7 9 9
93562024-02-12T01:38:31.567Z23/30 Write block 138 len 32768 data: 8 8 8 10 11 11 11 11
93572024-02-12T01:38:31.567Z24/30 Write block 51 len 12288 data: 9 6 4
93582024-02-12T01:38:31.570Z25/30 Read block 489 len 4096
93592024-02-12T01:38:31.570Z26/30 Read block 67 len 12288
93602024-02-12T01:38:31.570Z27/30 Write block 436 len 28672 data: 4 4 5 5 5 5 4
93612024-02-12T01:38:31.570Z28/30 Read block 536 len 20480
93622024-02-12T01:38:31.574Z29/30 Write block 374 len 24576 data: 9 11 10 9 8 8
93632024-02-12T01:38:31.574Z30/30 Write block 202 len 8192 data: 6 6
93642024-02-12T01:38:31.574Z----------------------------------------------------------------
93652024-02-12T01:38:31.574Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:27
93662024-02-12T01:38:31.574ZStates: Active Active Active
93672024-02-12T01:38:31.574ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93682024-02-12T01:38:31.574Z 4 Acked 1003 Write 3 Done Done Done false
93692024-02-12T01:38:31.574Z 5 Acked 1004 Write 6 Done Done Done false
93702024-02-12T01:38:31.574Z 6 Acked 1005 Read 2 Done Done Sent false
93712024-02-12T01:38:31.574Z 7 Acked 1006 Write 5 Done Done Sent false
93722024-02-12T01:38:31.574Z 8 Acked 1007 Write 5 Done Done Sent false
93732024-02-12T01:38:31.574Z 9 Acked 1008 Write 6 Done Done Sent false
93742024-02-12T01:38:31.574Z 10 Acked 1009 Write 7 Done Done Sent false
93752024-02-12T01:38:31.574Z 11 Acked 1010 Read 1 Done Done Sent false
93762024-02-12T01:38:31.575Z 12 Acked 1011 Write 10 Done Done Sent false
93772024-02-12T01:38:31.575Z 13 Acked 1012 Write 3 Done Done Sent false
93782024-02-12T01:38:31.575Z 14 Acked 1013 Write 7 Done Done Sent false
93792024-02-12T01:38:31.575Z 15 Acked 1014 Write 10 Done Done Sent false
93802024-02-12T01:38:31.575Z 16 Acked 1015 Write 9 Done Done Sent false
93812024-02-12T01:38:31.575Z 17 Acked 1016 Flush 0 Done Done Sent false
93822024-02-12T01:38:31.575Z 18 Acked 1017 Write 9 Done Done Sent false
93832024-02-12T01:38:31.575Z 19 Acked 1018 Flush 0 Done Done Sent false
93842024-02-12T01:38:31.575Z 20 Acked 1019 Write 9 Done Done Sent false
93852024-02-12T01:38:31.575Z 21 Acked 1020 Flush 0 Done Done Sent false
93862024-02-12T01:38:31.575Z 22 Acked 1021 Write 6 Done Done Sent false
93872024-02-12T01:38:31.575Z 23 Acked 1022 Write 8 Done Done Sent false
93882024-02-12T01:38:31.575Z 24 Acked 1023 Write 3 Done Done Sent false
93892024-02-12T01:38:31.575Z 25 Acked 1024 Read 1 Done Done Sent false
93902024-02-12T01:38:31.575Z 26 Acked 1025 Read 3 Done Done Sent false
93912024-02-12T01:38:31.575Z 27 Acked 1026 Write 7 Done Done Sent false
93922024-02-12T01:38:31.575Z 28 Acked 1027 Read 5 Done Done Sent false
93932024-02-12T01:38:31.575Z 29 Acked 1028 Write 6 Sent Sent Sent false
93942024-02-12T01:38:31.575Z 30 Acked 1029 Write 2 Sent Sent Sent false
93952024-02-12T01:38:31.575Z STATES DS:0 DS:1 DS:2 TOTAL
93962024-02-12T01:38:31.575Z New 0 0 0 0
93972024-02-12T01:38:31.575Z Sent 2 2 25 29
93982024-02-12T01:38:31.575Z Done 25 25 2 52
93992024-02-12T01:38:31.575Z Skipped 0 0 0 0
94002024-02-12T01:38:31.575Z Error 0 0 0 0
94012024-02-12T01:38:31.576ZLast Flush: 1020 1020 1002
94022024-02-12T01:38:31.576ZDownstairs last five completed: 1002 1001 1000
94032024-02-12T01:38:31.576ZUpstairs last five completed: 30 29 28 27 26
94042024-02-12T01:38:31.576Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:30.466688952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1471}
94052024-02-12T01:38:31.576ZWrote out file "/tmp/test_repair_verify.data"
94062024-02-12T01:38:31.579Z
94072024-02-12T01:38:31.582ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94082024-02-12T01:38:31.599ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94092024-02-12T01:38:31.599Z 0 000-019 84 84 72 145 145 126 F F F <---
94102024-02-12T01:38:31.599Z 1 020-039 84 84 82 145 145 141 F F F <---
94112024-02-12T01:38:31.600Z 2 040-059 84 84 82 147 147 141 T T F <---
94122024-02-12T01:38:31.600Z 3 060-079 84 84 84 144 144 144 F F F
94132024-02-12T01:38:31.600Z 4 080-099 84 84 84 144 144 144 F F F
94142024-02-12T01:38:31.600Z 5 100-119 80 80 80 137 137 137 F F F
94152024-02-12T01:38:31.600Z 6 120-139 82 82 82 141 141 141 T T F <---
94162024-02-12T01:38:31.600Z 7 140-159 84 84 70 146 146 124 T T F <---
94172024-02-12T01:38:31.600Z 8 160-179 84 84 82 146 146 141 F F F <---
94182024-02-12T01:38:31.600Z 9 180-199 80 80 80 137 137 137 F F F
94192024-02-12T01:38:31.600Z 10 200-219 80 80 80 137 137 137 T T F <---
94202024-02-12T01:38:31.606Z 11 220-239 80 80 80 137 137 137 F F F
94212024-02-12T01:38:31.606Z 12 240-259 80 80 80 137 137 137 F F F
94222024-02-12T01:38:31.606Z 13 260-279 83 83 83 142 142 142 F F F
94232024-02-12T01:38:31.606Z 14 280-299 84 84 82 145 145 140 F F F <---
94242024-02-12T01:38:31.606Z 15 300-319 76 76 76 132 132 132 F F F
94252024-02-12T01:38:31.606Z 16 320-339 74 74 74 131 131 131 F F F
94262024-02-12T01:38:31.606Z 17 340-359 84 84 78 145 145 135 F F F <---
94272024-02-12T01:38:31.606Z 18 360-379 84 84 80 145 145 137 T T F <---
94282024-02-12T01:38:31.606Z 19 380-399 84 84 83 145 145 142 T T F <---
94292024-02-12T01:38:31.606Z 20 400-419 84 84 80 145 145 137 F F T <---
94302024-02-12T01:38:31.606Z 21 420-439 82 82 82 139 139 139 T T F <---
94312024-02-12T01:38:31.606Z 22 440-459 82 82 82 139 139 139 T T F <---
94322024-02-12T01:38:31.606Z 23 460-479 84 84 83 145 145 142 F F F <---
94332024-02-12T01:38:31.606Z 24 480-499 76 76 76 132 132 132 F F F
94342024-02-12T01:38:31.607Z 25 500-519 78 78 78 134 134 134 F F F
94352024-02-12T01:38:31.607Z 26 520-539 84 84 82 145 145 140 F F F <---
94362024-02-12T01:38:31.607Z 27 540-559 77 77 77 133 133 133 F F F
94372024-02-12T01:38:31.607Z 28 560-579 83 83 83 142 142 142 F F F
94382024-02-12T01:38:31.607Z 29 580-599 84 84 79 145 145 136 F F T <---
94392024-02-12T01:38:31.607ZMax gen: 84, Max flush: 147
94402024-02-12T01:38:31.607ZError: Difference in extent metadata found!
94412024-02-12T01:38:31.607ZA Difference in extent metadata is expected here
94422024-02-12T01:38:31.607ZOn loop 42
94432024-02-12T01:38:31.607Z
94442024-02-12T01:38:31.610ZVerifying data now
94452024-02-12T01:38:37.425ZNo change after verify
94462024-02-12T01:38:37.428ZLoop: 42 Downstairs dump after verify (and repair):
94472024-02-12T01:38:37.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94482024-02-12T01:38:37.439Z 0 000-019 84 84 84 145 145 145 F F F
94492024-02-12T01:38:37.439Z 1 020-039 84 84 84 145 145 145 F F F
94502024-02-12T01:38:37.439Z 2 040-059 85 85 85 148 148 148 F F F
94512024-02-12T01:38:37.439Z 3 060-079 84 84 84 144 144 144 F F F
94522024-02-12T01:38:37.440Z 4 080-099 84 84 84 144 144 144 F F F
94532024-02-12T01:38:37.440Z 5 100-119 80 80 80 137 137 137 F F F
94542024-02-12T01:38:37.440Z 6 120-139 85 85 85 148 148 148 F F F
94552024-02-12T01:38:37.440Z 7 140-159 85 85 85 148 148 148 F F F
94562024-02-12T01:38:37.440Z 8 160-179 84 84 84 146 146 146 F F F
94572024-02-12T01:38:37.440Z 9 180-199 80 80 80 137 137 137 F F F
94582024-02-12T01:38:37.440Z 10 200-219 85 85 85 148 148 148 F F F
94592024-02-12T01:38:37.443Z 11 220-239 80 80 80 137 137 137 F F F
94602024-02-12T01:38:37.443Z 12 240-259 80 80 80 137 137 137 F F F
94612024-02-12T01:38:37.443Z 13 260-279 83 83 83 142 142 142 F F F
94622024-02-12T01:38:37.443Z 14 280-299 84 84 84 145 145 145 F F F
94632024-02-12T01:38:37.443Z 15 300-319 76 76 76 132 132 132 F F F
94642024-02-12T01:38:37.443Z 16 320-339 74 74 74 131 131 131 F F F
94652024-02-12T01:38:37.443Z 17 340-359 84 84 84 145 145 145 F F F
94662024-02-12T01:38:37.443Z 18 360-379 85 85 85 148 148 148 F F F
94672024-02-12T01:38:37.443Z 19 380-399 85 85 85 148 148 148 F F F
94682024-02-12T01:38:37.443Z 20 400-419 84 84 84 145 145 145 F F F
94692024-02-12T01:38:37.443Z 21 420-439 85 85 85 148 148 148 F F F
94702024-02-12T01:38:37.443Z 22 440-459 85 85 85 148 148 148 F F F
94712024-02-12T01:38:37.443Z 23 460-479 84 84 84 145 145 145 F F F
94722024-02-12T01:38:37.443Z 24 480-499 76 76 76 132 132 132 F F F
94732024-02-12T01:38:37.443Z 25 500-519 78 78 78 134 134 134 F F F
94742024-02-12T01:38:37.443Z 26 520-539 84 84 84 145 145 145 F F F
94752024-02-12T01:38:37.443Z 27 540-559 77 77 77 133 133 133 F F F
94762024-02-12T01:38:37.444Z 28 560-579 83 83 83 142 142 142 F F F
94772024-02-12T01:38:37.444Z 29 580-599 84 84 84 145 145 145 F F F
94782024-02-12T01:38:37.444ZMax gen: 85, Max flush: 148
94792024-02-12T01:38:37.444Z
94802024-02-12T01:38:37.444ZBegin loop 43
94812024-02-12T01:38:37.444ZDownstairs to restart: 0
94822024-02-12T01:38:37.461Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.353826832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
94832024-02-12T01:38:37.461Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354097969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
94842024-02-12T01:38:37.466Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354156746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
94852024-02-12T01:38:37.466Z{Crucible runtime is spawned
94862024-02-12T01:38:37.466Z"msg":"Crucible 5597cd3a-13d3-4fe9-a804-fa8433de9751 has session id: 89f75f5b-94dc-469d-83dd-f15f9a21a7cb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354241662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94872024-02-12T01:38:37.466Z{"msg":"Upstairs opts: Upstairs UUID: 5597cd3a-13d3-4fe9-a804-fa8433de9751, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354337857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94882024-02-12T01:38:37.466Z{"msg":"Crucible stats registered with UUID: 5597cd3a-13d3-4fe9-a804-fa8433de9751","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354373995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94892024-02-12T01:38:37.466Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354397844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
94902024-02-12T01:38:37.466Z{"msg":"5597cd3a-13d3-4fe9-a804-fa8433de9751 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354424103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94912024-02-12T01:38:37.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354446322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94922024-02-12T01:38:37.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94932024-02-12T01:38:37.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354514359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94942024-02-12T01:38:37.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354532058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94952024-02-12T01:38:37.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354554087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94962024-02-12T01:38:37.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354568116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94972024-02-12T01:38:37.466Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354585665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94982024-02-12T01:38:37.466Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354635193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
94992024-02-12T01:38:37.466Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354683521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95002024-02-12T01:38:37.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354735048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95012024-02-12T01:38:37.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354896821Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95022024-02-12T01:38:37.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.354933559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95032024-02-12T01:38:37.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.355390657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95042024-02-12T01:38:37.467Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.355437285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95052024-02-12T01:38:37.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.355485333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95062024-02-12T01:38:37.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.355594037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95072024-02-12T01:38:37.467Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356095143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95082024-02-12T01:38:37.467Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356148641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95092024-02-12T01:38:37.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356688945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95102024-02-12T01:38:37.467Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356723883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95112024-02-12T01:38:37.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356739293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95122024-02-12T01:38:37.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356767681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95132024-02-12T01:38:37.467Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356782271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95142024-02-12T01:38:37.467Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.35679633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95152024-02-12T01:38:37.467Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356824378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95162024-02-12T01:38:37.467Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.356838688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95172024-02-12T01:38:37.467Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357336924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95182024-02-12T01:38:37.472Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357783993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95192024-02-12T01:38:37.472Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357827651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95202024-02-12T01:38:37.472Z{"msg":"[0]R flush_numbers[0..12]: [145, 145, 148, 144, 144, 137, 148, 148, 146, 137, 148, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.35784599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95212024-02-12T01:38:37.472Z{"msg":"[0]R generation[0..12]: [84, 84, 85, 84, 84, 80, 85, 85, 84, 80, 85, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357867859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95222024-02-12T01:38:37.472Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357882748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95232024-02-12T01:38:37.472Z{"msg":"[1]R flush_numbers[0..12]: [145, 145, 148, 144, 144, 137, 148, 148, 146, 137, 148, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357897697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95242024-02-12T01:38:37.472Z{"msg":"[1]R generation[0..12]: [84, 84, 85, 84, 84, 80, 85, 85, 84, 80, 85, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357918876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95252024-02-12T01:38:37.472Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357933356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95262024-02-12T01:38:37.472Z{"msg":"[2]R flush_numbers[0..12]: [145, 145, 148, 144, 144, 137, 148, 148, 146, 137, 148, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357947615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95272024-02-12T01:38:37.472Z{"msg":"[2]R generation[0..12]: [84, 84, 85, 84, 84, 80, 85, 85, 84, 80, 85, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.357961424Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95282024-02-12T01:38:37.472Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
95292024-02-12T01:38:37.472Z,"time":"2024-02-12T01:38:36.357986563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95302024-02-12T01:38:37.472Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358027121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95312024-02-12T01:38:37.472Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.35804178Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95322024-02-12T01:38:37.472Z{"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.35806141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95332024-02-12T01:38:37.472Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358075119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95342024-02-12T01:38:37.473Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358095108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95352024-02-12T01:38:37.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358107217Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95362024-02-12T01:38:37.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358127006Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95372024-02-12T01:38:37.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358140236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95382024-02-12T01:38:37.473Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358159945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95392024-02-12T01:38:37.473Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358172514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95402024-02-12T01:38:37.473Z{"msg":"5597cd3a-13d3-4fe9-a804-fa8433de9751 is now active with session: 89f75f5b-94dc-469d-83dd-f15f9a21a7cb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358197083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95412024-02-12T01:38:37.473Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358215362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479,"session_id":"89f75f5b-94dc-469d-83dd-f15f9a21a7cb"}
95422024-02-12T01:38:37.473Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:36.358245341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
95432024-02-12T01:38:37.473ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95442024-02-12T01:38:37.473ZLoading write count information from file "/tmp/test_repair_verify.data"
95452024-02-12T01:38:37.473ZRun Repair workload
95462024-02-12T01:38:37.473Z01/30 Write block 389 len 24576 data: 8 13 13 12 10 7
95472024-02-12T01:38:37.473Z02/30 Read block 35 len 24576
95482024-02-12T01:38:37.473Z03/30 Write block 577 len 24576 data: 6 6 6 6 7 6
95492024-02-12T01:38:37.473Z04/30 Write block 439 len 16384 data: 6 6 6 5
95502024-02-12T01:38:37.473Z05/30 Read block 41 len 32768
95512024-02-12T01:38:37.476Z06/30 Write block 169 len 12288 data: 8 8 7
95522024-02-12T01:38:37.476Z07/30 Write block 369 len 28672 data: 8 8 8 9 10 10 12
95532024-02-12T01:38:37.476Z08/30 Read block 263 len 8192
95542024-02-12T01:38:37.477Z09/30 Write block 590 len 20480 data: 8 6 6 3 2
95552024-02-12T01:38:37.477Z10/30 Write block 124 len 12288 data: 11 10 12
95562024-02-12T01:38:37.477Z11/30 Read block 462 len 32768
95572024-02-12T01:38:37.477Z12/30 Write block 210 len 40960 data: 5 6 5 5 7 7 11 8 9 10
95582024-02-12T01:38:37.480Z13/30 Read block 306 len 36864
95592024-02-12T01:38:37.480Z14/30 Read block 277 len 28672
95602024-02-12T01:38:37.480Z15/30 Write block 179 len 24576 data: 11 8 9 5 5 7
95612024-02-12T01:38:37.483Z16/30 Read block 108 len 36864
95622024-02-12T01:38:37.484Z17/30 Write block 584 len 16384 data: 6 7 7 5
95632024-02-12T01:38:37.484Z18/30 Write block 204 len 4096 data: 6
95642024-02-12T01:38:37.484Z19/30 Write block 507 len 24576 data: 4 4 4 6 8 9
95652024-02-12T01:38:37.487Z20/30 Write block 277 len 12288 data: 7 8 10
95662024-02-12T01:38:37.488Z21/30 Read block 107 len 12288
95672024-02-12T01:38:37.488Z22/30 Read block 518 len 28672
95682024-02-12T01:38:37.488Z23/30 Write block 48 len 16384 data: 7 9 9 10
95692024-02-12T01:38:37.488Z24/30 Read block 550 len 24576
95702024-02-12T01:38:37.493Z25/30 Flush
95712024-02-12T01:38:37.498Z26/30 Flush
95722024-02-12T01:38:37.498Z27/30 Write block 263 len 16384 data: 6 4 7 8
95732024-02-12T01:38:37.499Z28/30 Write block 592 len 32768 data: 7 4 3 5 6 5 4 3
95742024-02-12T01:38:37.499Z29/30 Read block 10 len 16384
95752024-02-12T01:38:37.499Z30/30 Read block 53 len 20480
95762024-02-12T01:38:37.499Z----------------------------------------------------------------
95772024-02-12T01:38:37.499Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
95782024-02-12T01:38:37.499ZStates: Active Active Active
95792024-02-12T01:38:37.499ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95802024-02-12T01:38:37.499Z 1 Acked 1000 Write 6 Done Done Done false
95812024-02-12T01:38:37.499Z 2 Acked 1001 Read 6 Done Done Done false
95822024-02-12T01:38:37.499Z 3 Acked 1002 Write 6 Sent Done Done false
95832024-02-12T01:38:37.499Z 4 Acked 1003 Write 4 Sent Done Done false
95842024-02-12T01:38:37.499Z 5 Acked 1004 Read 8 Sent Done Done false
95852024-02-12T01:38:37.499Z 6 Acked 1005 Write 3 Sent Done Done false
95862024-02-12T01:38:37.499Z 7 Acked 1006 Write 7 Sent Done Done false
95872024-02-12T01:38:37.499Z 8 Acked 1007 Read 2 Sent Done Done false
95882024-02-12T01:38:37.499Z 9 Acked 1008 Write 5 Sent Done Done false
95892024-02-12T01:38:37.499Z 10 Acked 1009 Write 3 Sent Done Done false
95902024-02-12T01:38:37.499Z 11 Acked 1010 Read 8 Sent Done Done false
95912024-02-12T01:38:37.499Z 12 Acked 1011 Write 10 Sent Done Done false
95922024-02-12T01:38:37.499Z 13 Acked 1012 Read 9 Sent Done Done false
95932024-02-12T01:38:37.499Z 14 Acked 1013 Read 7 Sent Done Done false
95942024-02-12T01:38:37.499Z 15 Acked 1014 Write 6 Sent Done Done false
95952024-02-12T01:38:37.499Z 16 Acked 1015 Read 9 Sent Done Done false
95962024-02-12T01:38:37.499Z 17 Acked 1016 Write 4 Sent Done Done false
95972024-02-12T01:38:37.500Z 18 Acked 1017 Write 1 Sent Done Done false
95982024-02-12T01:38:37.500Z 19 Acked 1018 Write 6 Sent Done Done false
95992024-02-12T01:38:37.500Z 20 Acked 1019 Write 3 Sent Done Done false
96002024-02-12T01:38:37.500Z 21 Acked 1020 Read 3 Sent Done Done false
96012024-02-12T01:38:37.500Z 22 Acked 1021 Read 7 Sent Done Done false
96022024-02-12T01:38:37.500Z 23 Acked 1022 Write 4 Sent Done Done false
96032024-02-12T01:38:37.500Z 24 Acked 1023 Read 6 Sent Done Done false
96042024-02-12T01:38:37.500Z 25 Acked 1024 Flush 0 Sent Done Done false
96052024-02-12T01:38:37.500Z 26 Acked 1025 Flush 0 Sent Done Done false
96062024-02-12T01:38:37.500Z 27 Acked 1026 Write 4 Sent Done Done false
96072024-02-12T01:38:37.500Z 28 Acked 1027 Write 8 Sent Done Done false
96082024-02-12T01:38:37.500Z 29 Acked 1028 Read 4 Sent Done Done false
96092024-02-12T01:38:37.500Z 30 Acked 1029 Read 5 Sent Done Sent false
96102024-02-12T01:38:37.500Z STATES DS:0 DS:1 DS:2 TOTAL
96112024-02-12T01:38:37.500Z New 0 0 0 0
96122024-02-12T01:38:37.500Z Sent 28 0 1 29
96132024-02-12T01:38:37.500Z Done 2 30 29 61
96142024-02-12T01:38:37.500Z Skipped 0 0 0 0
96152024-02-12T01:38:37.500Z Error 0 0 0 0
96162024-02-12T01:38:37.500ZLast Flush: 0 1025 1025
96172024-02-12T01:38:37.500ZDownstairs last five completed:
96182024-02-12T01:38:37.500ZUpstairs last five completed: 30 29 28 27 26
96192024-02-12T01:38:37.500Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:36.389791626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1479}
96202024-02-12T01:38:37.504ZWrote out file "/tmp/test_repair_verify.data"
96212024-02-12T01:38:37.504Z
96222024-02-12T01:38:37.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96232024-02-12T01:38:37.519ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96242024-02-12T01:38:37.519Z 0 000-019 84 84 84 145 145 145 F F F
96252024-02-12T01:38:37.519Z 1 020-039 84 84 84 145 145 145 F F F
96262024-02-12T01:38:37.519Z 2 040-059 85 86 86 148 149 149 F F F <---
96272024-02-12T01:38:37.519Z 3 060-079 84 84 84 144 144 144 F F F
96282024-02-12T01:38:37.519Z 4 080-099 84 84 84 144 144 144 F F F
96292024-02-12T01:38:37.519Z 5 100-119 80 80 80 137 137 137 F F F
96302024-02-12T01:38:37.520Z 6 120-139 85 86 86 148 149 149 F F F <---
96312024-02-12T01:38:37.520Z 7 140-159 85 85 85 148 148 148 F F F
96322024-02-12T01:38:37.520Z 8 160-179 84 86 86 146 149 149 F F F <---
96332024-02-12T01:38:37.520Z 9 180-199 80 86 86 137 149 149 F F F <---
96342024-02-12T01:38:37.520Z 10 200-219 85 86 86 148 149 149 F F F <---
96352024-02-12T01:38:37.520Z 11 220-239 80 80 80 137 137 137 F F F
96362024-02-12T01:38:37.520Z 12 240-259 80 80 80 137 137 137 F F F
96372024-02-12T01:38:37.520Z 13 260-279 83 86 86 142 149 149 F T T <---
96382024-02-12T01:38:37.523Z 14 280-299 84 84 84 145 145 145 F F F
96392024-02-12T01:38:37.523Z 15 300-319 76 76 76 132 132 132 F F F
96402024-02-12T01:38:37.523Z 16 320-339 74 74 74 131 131 131 F F F
96412024-02-12T01:38:37.523Z 17 340-359 84 84 84 145 145 145 F F F
96422024-02-12T01:38:37.523Z 18 360-379 85 86 86 148 149 149 F F F <---
96432024-02-12T01:38:37.523Z 19 380-399 85 86 86 148 149 149 T F F <---
96442024-02-12T01:38:37.523Z 20 400-419 84 84 84 145 145 145 F F F
96452024-02-12T01:38:37.523Z 21 420-439 85 86 86 148 149 149 F F F <---
96462024-02-12T01:38:37.523Z 22 440-459 85 86 86 148 149 149 F F F <---
96472024-02-12T01:38:37.523Z 23 460-479 84 84 84 145 145 145 F F F
96482024-02-12T01:38:37.523Z 24 480-499 76 76 76 132 132 132 F F F
96492024-02-12T01:38:37.524Z 25 500-519 78 86 86 134 149 149 F F F <---
96502024-02-12T01:38:37.524Z 26 520-539 84 84 84 145 145 145 F F F
96512024-02-12T01:38:37.524Z 27 540-559 77 77 77 133 133 133 F F F
96522024-02-12T01:38:37.524Z 28 560-579 83 86 86 142 149 149 F F F <---
96532024-02-12T01:38:37.524Z 29 580-599 84 86 86 145 149 149 F T T <---
96542024-02-12T01:38:37.524ZMax gen: 86, Max flush: 149
96552024-02-12T01:38:37.524ZError: Difference in extent metadata found!
96562024-02-12T01:38:37.524ZA Difference in extent metadata is expected here
96572024-02-12T01:38:37.524ZOn loop 43
96582024-02-12T01:38:37.524Z
96592024-02-12T01:38:37.527ZVerifying data now
96602024-02-12T01:38:42.158ZNo change after verify
96612024-02-12T01:38:42.158ZLoop: 43 Downstairs dump after verify (and repair):
96622024-02-12T01:38:42.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96632024-02-12T01:38:42.173Z 0 000-019 84 84 84 145 145 145 F F F
96642024-02-12T01:38:42.173Z 1 020-039 84 84 84 145 145 145 F F F
96652024-02-12T01:38:42.173Z 2 040-059 86 86 86 149 149 149 F F F
96662024-02-12T01:38:42.173Z 3 060-079 84 84 84 144 144 144 F F F
96672024-02-12T01:38:42.173Z 4 080-099 84 84 84 144 144 144 F F F
96682024-02-12T01:38:42.173Z 5 100-119 80 80 80 137 137 137 F F F
96692024-02-12T01:38:42.173Z 6 120-139 86 86 86 149 149 149 F F F
96702024-02-12T01:38:42.173Z 7 140-159 85 85 85 148 148 148 F F F
96712024-02-12T01:38:42.173Z 8 160-179 86 86 86 149 149 149 F F F
96722024-02-12T01:38:42.173Z 9 180-199 86 86 86 149 149 149 F F F
96732024-02-12T01:38:42.177Z 10 200-219 86 86 86 149 149 149 F F F
96742024-02-12T01:38:42.177Z 11 220-239 80 80 80 137 137 137 F F F
96752024-02-12T01:38:42.177Z 12 240-259 80 80 80 137 137 137 F F F
96762024-02-12T01:38:42.177Z 13 260-279 87 87 87 150 150 150 F F F
96772024-02-12T01:38:42.177Z 14 280-299 84 84 84 145 145 145 F F F
96782024-02-12T01:38:42.177Z 15 300-319 76 76 76 132 132 132 F F F
96792024-02-12T01:38:42.177Z 16 320-339 74 74 74 131 131 131 F F F
96802024-02-12T01:38:42.177Z 17 340-359 84 84 84 145 145 145 F F F
96812024-02-12T01:38:42.178Z 18 360-379 86 86 86 149 149 149 F F F
96822024-02-12T01:38:42.178Z 19 380-399 86 86 86 149 149 149 F F F
96832024-02-12T01:38:42.178Z 20 400-419 84 84 84 145 145 145 F F F
96842024-02-12T01:38:42.178Z 21 420-439 86 86 86 149 149 149 F F F
96852024-02-12T01:38:42.178Z 22 440-459 86 86 86 149 149 149 F F F
96862024-02-12T01:38:42.178Z 23 460-479 84 84 84 145 145 145 F F F
96872024-02-12T01:38:42.178Z 24 480-499 76 76 76 132 132 132 F F F
96882024-02-12T01:38:42.178Z 25 500-519 86 86 86 149 149 149 F F F
96892024-02-12T01:38:42.178Z 26 520-539 84 84 84 145 145 145 F F F
96902024-02-12T01:38:42.178Z 27 540-559 77 77 77 133 133 133 F F F
96912024-02-12T01:38:42.178Z 28 560-579 86 86 86 149 149 149 F F F
96922024-02-12T01:38:42.178Z 29 580-599 87 87 87 150 150 150 F F F
96932024-02-12T01:38:42.178ZMax gen: 87, Max flush: 150
96942024-02-12T01:38:42.178Z
96952024-02-12T01:38:42.178ZBegin loop 44
96962024-02-12T01:38:42.179ZDownstairs to restart: 2
96972024-02-12T01:38:42.195Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.087629915Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
96982024-02-12T01:38:42.195Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.087902533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
96992024-02-12T01:38:42.195Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.08794956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
97002024-02-12T01:38:42.201Z{"msg":"Crucible 06acee1b-afd8-4e6d-a2c6-69b0024c6c9d has session id: 5a5812e8-e92e-4a2f-9d44-8d6b377f0e23","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.087979349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97012024-02-12T01:38:42.201Z{"msg":"Upstairs opts: Upstairs UUID: 06acee1b-afd8-4e6d-a2c6-69b0024c6c9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088034576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97022024-02-12T01:38:42.201Z{"msg":"Crucible stats registered with UUID: 06acee1b-afd8-4e6d-a2c6-69b0024c6c9d","v":0,"name":"crucible","level":30Crucible runtime is spawned
97032024-02-12T01:38:42.201Z,"time":"2024-02-12T01:38:41.088082284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97042024-02-12T01:38:42.201Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088209198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
97052024-02-12T01:38:42.201Z{"msg":"06acee1b-afd8-4e6d-a2c6-69b0024c6c9d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088274335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97062024-02-12T01:38:42.201Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088298494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97072024-02-12T01:38:42.201Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088342252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97082024-02-12T01:38:42.201Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.08837728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97092024-02-12T01:38:42.201Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088412008Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97102024-02-12T01:38:42.201Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088466106Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97112024-02-12T01:38:42.201Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088521773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97122024-02-12T01:38:42.202Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088559521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97132024-02-12T01:38:42.202Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088641767Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97142024-02-12T01:38:42.202Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088678536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97152024-02-12T01:38:42.202Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088725703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97162024-02-12T01:38:42.202Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:41.088759212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97172024-02-12T01:38:42.202Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:41.08879374Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97182024-02-12T01:38:42.202Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:41.088826009Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97192024-02-12T01:38:42.202Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088850677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97202024-02-12T01:38:42.202Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088891155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97212024-02-12T01:38:42.202Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.088919214Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97222024-02-12T01:38:42.202Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.089026079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97232024-02-12T01:38:42.202Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.089487957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97242024-02-12T01:38:42.202Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.089538935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97252024-02-12T01:38:42.202Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.089580362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97262024-02-12T01:38:42.202Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.089620271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97272024-02-12T01:38:42.203Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090157255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97282024-02-12T01:38:42.203Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090195003Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97292024-02-12T01:38:42.203Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090656391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97302024-02-12T01:38:42.203Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090692729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97312024-02-12T01:38:42.203Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090743097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97322024-02-12T01:38:42.203Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090808774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97332024-02-12T01:38:42.203Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.090850272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97342024-02-12T01:38:42.203Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:41.09089236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97352024-02-12T01:38:43.198Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.090792006Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97362024-02-12T01:38:43.198Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.090894521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97372024-02-12T01:38:43.203Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.091802727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97382024-02-12T01:38:43.203Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.091838096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97392024-02-12T01:38:43.203Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092332552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97402024-02-12T01:38:43.203Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092829398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97412024-02-12T01:38:43.203Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092888896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97422024-02-12T01:38:43.203Z{"msg":"[0]R flush_numbers[0..12]: [145, 145, 149, 144, 144, 137, 149, 148, 149, 149, 149, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092924814Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97432024-02-12T01:38:43.203Z{"msg":"[0]R generation[0..12]: [84, 84, 86, 84, 84, 80, 86, 85, 86, 86, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092955723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97442024-02-12T01:38:43.203Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.092975012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97452024-02-12T01:38:43.203Z{"msg":"[1]R flush_numbers[0..12]: [145, 145, 149, 144, 144, 137, 149, 148, 149, 149, 149, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.09301655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97462024-02-12T01:38:43.203Z{"msg":"[1]R generation[0..12]: [84, 84, 86, 84, 84, 80, 86, 85, 86, 86, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093037159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97472024-02-12T01:38:43.203ZWait for a query_work_queue command to finish before sending IO
97482024-02-12T01:38:43.203Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093070777Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97492024-02-12T01:38:43.203Z{"msg":"[2]R flush_numbers[0..12]: [145, 145, 149, 144, 144, 137, 149, 148, 149, 149, 149, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093090206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97502024-02-12T01:38:43.203Z{"msg":"[2]R generation[0..12]: [84, 84, 86, 84, 84, 80, 86, 85, 86, 86, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093107875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97512024-02-12T01:38:43.203Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093134314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97522024-02-12T01:38:43.203Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093151273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97532024-02-12T01:38:43.203Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093176822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97542024-02-12T01:38:43.203Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093193431Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97552024-02-12T01:38:43.203Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.0932107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97562024-02-12T01:38:43.204Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.0932276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97572024-02-12T01:38:43.204Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093242859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97582024-02-12T01:38:43.204Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093259478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97592024-02-12T01:38:43.204Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093276987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97602024-02-12T01:38:43.204Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093293616Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97612024-02-12T01:38:43.204Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093317805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97622024-02-12T01:38:43.204Z{"msg":"06acee1b-afd8-4e6d-a2c6-69b0024c6c9d is now active with session: 5a5812e8-e92e-4a2f-9d44-8d6b377f0e23","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093335665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97632024-02-12T01:38:43.204Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093350984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487,"session_id":"5a5812e8-e92e-4a2f-9d44-8d6b377f0e23"}
97642024-02-12T01:38:43.204Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:42.093381492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
97652024-02-12T01:38:43.204ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97662024-02-12T01:38:43.204ZLoading write count information from file "/tmp/test_repair_verify.data"
97672024-02-12T01:38:43.204ZRun Repair workload
97682024-02-12T01:38:43.207Z01/30 Write block 240 len 32768 data: 6 4 5 6 4 4 5 6
97692024-02-12T01:38:43.207Z02/30 Read block 564 len 32768
97702024-02-12T01:38:43.207Z03/30 Write block 46 len 16384 data: 5 7 8 10
97712024-02-12T01:38:43.207Z04/30 Write block 577 len 32768 data: 7 7 7 7 8 7 6 7
97722024-02-12T01:38:43.207Z05/30 Read block 124 len 16384
97732024-02-12T01:38:43.210Z06/30 Read block 280 len 8192
97742024-02-12T01:38:43.210Z07/30 Write block 414 len 36864 data: 10 11 9 8 6 4 4 6 5
97752024-02-12T01:38:43.211Z08/30 Write block 446 len 24576 data: 6 8 8 8 8 6
97762024-02-12T01:38:43.211Z09/30 Read block 321 len 28672
97772024-02-12T01:38:43.213Z10/30 Read block 542 len 32768
97782024-02-12T01:38:43.214Z11/30 Read block 80 len 40960
97792024-02-12T01:38:43.214Z12/30 Write block 297 len 16384 data: 5 5 5 5
97802024-02-12T01:38:43.214Z13/30 Read block 182 len 28672
97812024-02-12T01:38:43.231Z14/30 Read block 322 len 24576
97822024-02-12T01:38:43.231Z15/30 Write block 533 len 40960 data: 12 10 12 11 11 10 9 8 5 5
97832024-02-12T01:38:43.231Z16/30 Write block 528 len 12288 data: 5 6 9
97842024-02-12T01:38:43.231Z17/30 Read block 441 len 20480
97852024-02-12T01:38:43.231Z18/30 Write block 305 len 36864 data: 6 8 9 10 7 7 7 6 6
97862024-02-12T01:38:43.231Z19/30 Write block 502 len 32768 data: 7 5 5 5 4 5 5 5
97872024-02-12T01:38:43.231Z20/30 Read block 416 len 28672
97882024-02-12T01:38:43.232Z21/30 Flush
97892024-02-12T01:38:43.232Z22/30 Flush
97902024-02-12T01:38:43.232Z23/30 Write block 141 len 20480 data: 11 12 12 12 12
97912024-02-12T01:38:43.232Z24/30 Write block 167 len 24576 data: 8 8 9 9 8 6
97922024-02-12T01:38:43.232Z25/30 Write block 25 len 8192 data: 5 5
97932024-02-12T01:38:43.232Z26/30 Read block 575 len 28672
97942024-02-12T01:38:43.232Z27/30 Write block 148 len 24576 data: 9 10 11 11 10 8
97952024-02-12T01:38:43.239Z28/30 Write block 510 len 32768 data: 7 9 10 9 8 10 7 6
97962024-02-12T01:38:43.239Z29/30 Write block 346 len 36864 data: 6 6 5 5 7 8 10 8 5
97972024-02-12T01:38:43.239Z30/30 Write block 206 len 32768 data: 7 9 9 8 6 7 6 6
97982024-02-12T01:38:43.239Z----------------------------------------------------------------
97992024-02-12T01:38:43.239Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
98002024-02-12T01:38:43.239ZStates: Active Active Active
98012024-02-12T01:38:43.239ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98022024-02-12T01:38:43.239Z 1 Acked 1000 Write 8 Done Done Done false
98032024-02-12T01:38:43.239Z 2 Acked 1001 Read 8 Done Done Done false
98042024-02-12T01:38:43.239Z 3 Acked 1002 Write 4 Done Done Done false
98052024-02-12T01:38:43.239Z 4 Acked 1003 Write 8 Done Done Done false
98062024-02-12T01:38:43.239Z 5 Acked 1004 Read 4 Done Done Done false
98072024-02-12T01:38:43.239Z 6 Acked 1005 Read 2 Done Done Sent false
98082024-02-12T01:38:43.239Z 7 Acked 1006 Write 9 Done Done Sent false
98092024-02-12T01:38:43.239Z 8 Acked 1007 Write 6 Done Done Sent false
98102024-02-12T01:38:43.239Z 9 Acked 1008 Read 7 Done Done Sent false
98112024-02-12T01:38:43.240Z 10 Acked 1009 Read 8 Done Done Sent false
98122024-02-12T01:38:43.240Z 11 Acked 1010 Read 10 Done Done Sent false
98132024-02-12T01:38:43.240Z 12 Acked 1011 Write 4 Done Done Sent false
98142024-02-12T01:38:43.240Z 13 Acked 1012 Read 7 Done Done Sent false
98152024-02-12T01:38:43.240Z 14 Acked 1013 Read 6 Done Done Sent false
98162024-02-12T01:38:43.240Z 15 Acked 1014 Write 10 Done Done Sent false
98172024-02-12T01:38:43.240Z 16 Acked 1015 Write 3 Done Done Sent false
98182024-02-12T01:38:43.240Z 17 Acked 1016 Read 5 Done Done Sent false
98192024-02-12T01:38:43.240Z 18 Acked 1017 Write 9 Done Done Sent false
98202024-02-12T01:38:43.240Z 19 Acked 1018 Write 8 Done Done Sent false
98212024-02-12T01:38:43.240Z 20 Acked 1019 Read 7 Done Done Sent false
98222024-02-12T01:38:43.240Z 21 Acked 1020 Flush 0 Done Done Sent false
98232024-02-12T01:38:43.240Z 22 Acked 1021 Flush 0 Done Done Sent false
98242024-02-12T01:38:43.240Z 23 Acked 1022 Write 5 Done Done Sent false
98252024-02-12T01:38:43.240Z 24 Acked 1023 Write 6 Done Done Sent false
98262024-02-12T01:38:43.240Z 25 Acked 1024 Write 2 Done Done Sent false
98272024-02-12T01:38:43.240Z 26 Acked 1025 Read 7 Done Done Sent false
98282024-02-12T01:38:43.240Z 27 Acked 1026 Write 6 Done Done Sent false
98292024-02-12T01:38:43.240Z 28 Acked 1027 Write 8 Done Done Sent false
98302024-02-12T01:38:43.240Z 29 Acked 1028 Write 9 Sent Sent Sent false
98312024-02-12T01:38:43.240Z 30 Acked 1029 Write 8 Sent Sent Sent false
98322024-02-12T01:38:43.240Z STATES DS:0 DS:1 DS:2 TOTAL
98332024-02-12T01:38:43.240Z New 0 0 0 0
98342024-02-12T01:38:43.240Z Sent 2 2 25 29
98352024-02-12T01:38:43.240Z Done 28 28 5 61
98362024-02-12T01:38:43.241Z Skipped 0 0 0 0
98372024-02-12T01:38:43.241Z Error 0 0 0 0
98382024-02-12T01:38:43.241ZLast Flush: 1021 1021 0
98392024-02-12T01:38:43.241ZDownstairs last five completed:
98402024-02-12T01:38:43.241ZUpstairs last five completed: 30 29 28 27 26
98412024-02-12T01:38:43.241Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:42.126190917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1487}
98422024-02-12T01:38:43.241ZWrote out file "/tmp/test_repair_verify.data"
98432024-02-12T01:38:43.244Z
98442024-02-12T01:38:43.244ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98452024-02-12T01:38:43.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98462024-02-12T01:38:43.260Z 0 000-019 84 84 84 145 145 145 F F F
98472024-02-12T01:38:43.260Z 1 020-039 84 84 84 145 145 145 T T F <---
98482024-02-12T01:38:43.260Z 2 040-059 88 88 86 151 151 149 F F T <---
98492024-02-12T01:38:43.260Z 3 060-079 84 84 84 144 144 144 F F F
98502024-02-12T01:38:43.260Z 4 080-099 84 84 84 144 144 144 F F F
98512024-02-12T01:38:43.260Z 5 100-119 80 80 80 137 137 137 F F F
98522024-02-12T01:38:43.260Z 6 120-139 86 86 86 149 149 149 F F F
98532024-02-12T01:38:43.260Z 7 140-159 85 85 85 148 148 148 T T F <---
98542024-02-12T01:38:43.260Z 8 160-179 86 86 86 149 149 149 T T F <---
98552024-02-12T01:38:43.260Z 9 180-199 86 86 86 149 149 149 F F F
98562024-02-12T01:38:43.260Z 10 200-219 86 86 86 149 149 149 T T F <---
98572024-02-12T01:38:43.263Z 11 220-239 80 80 80 137 137 137 F F F
98582024-02-12T01:38:43.263Z 12 240-259 88 88 80 151 151 137 F F T <---
98592024-02-12T01:38:43.264Z 13 260-279 87 87 87 150 150 150 F F F
98602024-02-12T01:38:43.264Z 14 280-299 88 88 84 151 151 145 F F F <---
98612024-02-12T01:38:43.264Z 15 300-319 88 88 76 151 151 132 F F F <---
98622024-02-12T01:38:43.264Z 16 320-339 74 74 74 131 131 131 F F F
98632024-02-12T01:38:43.264Z 17 340-359 84 84 84 145 145 145 T T F <---
98642024-02-12T01:38:43.264Z 18 360-379 86 86 86 149 149 149 F F F
98652024-02-12T01:38:43.264Z 19 380-399 86 86 86 149 149 149 F F F
98662024-02-12T01:38:43.264Z 20 400-419 88 88 84 151 151 145 F F F <---
98672024-02-12T01:38:43.264Z 21 420-439 88 88 86 151 151 149 F F F <---
98682024-02-12T01:38:43.264Z 22 440-459 88 88 86 151 151 149 F F F <---
98692024-02-12T01:38:43.264Z 23 460-479 84 84 84 145 145 145 F F F
98702024-02-12T01:38:43.264Z 24 480-499 76 76 76 132 132 132 F F F
98712024-02-12T01:38:43.264Z 25 500-519 88 88 86 151 151 149 T T F <---
98722024-02-12T01:38:43.264Z 26 520-539 88 88 84 151 151 145 F F F <---
98732024-02-12T01:38:43.264Z 27 540-559 88 88 77 151 151 133 F F F <---
98742024-02-12T01:38:43.264Z 28 560-579 88 88 86 151 151 149 F F T <---
98752024-02-12T01:38:43.264Z 29 580-599 88 88 87 151 151 150 F F T <---
98762024-02-12T01:38:43.264ZMax gen: 88, Max flush: 151
98772024-02-12T01:38:43.264ZError: Difference in extent metadata found!
98782024-02-12T01:38:43.264ZA Difference in extent metadata is expected here
98792024-02-12T01:38:43.264ZOn loop 44
98802024-02-12T01:38:43.264Z
98812024-02-12T01:38:43.268ZVerifying data now
98822024-02-12T01:38:49.081ZNo change after verify
98832024-02-12T01:38:49.081ZLoop: 44 Downstairs dump after verify (and repair):
98842024-02-12T01:38:49.097ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98852024-02-12T01:38:49.097Z 0 000-019 84 84 84 145 145 145 F F F
98862024-02-12T01:38:49.097Z 1 020-039 89 89 89 152 152 152 F F F
98872024-02-12T01:38:49.097Z 2 040-059 88 88 88 151 151 151 F F F
98882024-02-12T01:38:49.097Z 3 060-079 84 84 84 144 144 144 F F F
98892024-02-12T01:38:49.097Z 4 080-099 84 84 84 144 144 144 F F F
98902024-02-12T01:38:49.097Z 5 100-119 80 80 80 137 137 137 F F F
98912024-02-12T01:38:49.097Z 6 120-139 86 86 86 149 149 149 F F F
98922024-02-12T01:38:49.097Z 7 140-159 89 89 89 152 152 152 F F F
98932024-02-12T01:38:49.097Z 8 160-179 89 89 89 152 152 152 F F F
98942024-02-12T01:38:49.097Z 9 180-199 86 86 86 149 149 149 F F F
98952024-02-12T01:38:49.097Z 10 200-219 89 89 89 152 152 152 F F F
98962024-02-12T01:38:49.101Z 11 220-239 80 80 80 137 137 137 F F F
98972024-02-12T01:38:49.101Z 12 240-259 88 88 88 151 151 151 F F F
98982024-02-12T01:38:49.101Z 13 260-279 87 87 87 150 150 150 F F F
98992024-02-12T01:38:49.101Z 14 280-299 88 88 88 151 151 151 F F F
99002024-02-12T01:38:49.101Z 15 300-319 88 88 88 151 151 151 F F F
99012024-02-12T01:38:49.101Z 16 320-339 74 74 74 131 131 131 F F F
99022024-02-12T01:38:49.101Z 17 340-359 89 89 89 152 152 152 F F F
99032024-02-12T01:38:49.101Z 18 360-379 86 86 86 149 149 149 F F F
99042024-02-12T01:38:49.101Z 19 380-399 86 86 86 149 149 149 F F F
99052024-02-12T01:38:49.101Z 20 400-419 88 88 88 151 151 151 F F F
99062024-02-12T01:38:49.101Z 21 420-439 88 88 88 151 151 151 F F F
99072024-02-12T01:38:49.102Z 22 440-459 88 88 88 151 151 151 F F F
99082024-02-12T01:38:49.102Z 23 460-479 84 84 84 145 145 145 F F F
99092024-02-12T01:38:49.102Z 24 480-499 76 76 76 132 132 132 F F F
99102024-02-12T01:38:49.102Z 25 500-519 89 89 89 152 152 152 F F F
99112024-02-12T01:38:49.102Z 26 520-539 88 88 88 151 151 151 F F F
99122024-02-12T01:38:49.102Z 27 540-559 88 88 88 151 151 151 F F F
99132024-02-12T01:38:49.102Z 28 560-579 88 88 88 151 151 151 F F F
99142024-02-12T01:38:49.102Z 29 580-599 88 88 88 151 151 151 F F F
99152024-02-12T01:38:49.102ZMax gen: 89, Max flush: 152
99162024-02-12T01:38:49.102Z
99172024-02-12T01:38:49.102ZBegin loop 45
99182024-02-12T01:38:49.102ZDownstairs to restart: 0
99192024-02-12T01:38:49.120Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.011736652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
99202024-02-12T01:38:49.120Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.0119848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
99212024-02-12T01:38:49.120Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012011839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
99222024-02-12T01:38:49.120Z{"msg":"Crucible 80ab4f74-6951-4fe7-ab60-fb1d53208107 has session id: 3097c555-9395-4f2a-91c8-f8d377f7098a","v":0,"name":"crucible","level":30Crucible runtime is spawned
99232024-02-12T01:38:49.120Z,"time":"2024-02-12T01:38:48.012054156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99242024-02-12T01:38:49.126Z{"msg":"Upstairs opts: Upstairs UUID: 80ab4f74-6951-4fe7-ab60-fb1d53208107, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012107074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99252024-02-12T01:38:49.126Z{"msg":"Crucible stats registered with UUID: 80ab4f74-6951-4fe7-ab60-fb1d53208107","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012169211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99262024-02-12T01:38:49.126Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01219339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
99272024-02-12T01:38:49.126Z{"msg":"80ab4f74-6951-4fe7-ab60-fb1d53208107 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012240478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99282024-02-12T01:38:49.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012273436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99292024-02-12T01:38:49.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012300065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99302024-02-12T01:38:49.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012331333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99312024-02-12T01:38:49.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012356152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99322024-02-12T01:38:49.126Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01239753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99332024-02-12T01:38:49.126Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012421209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99342024-02-12T01:38:49.127Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012456367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99352024-02-12T01:38:49.127Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012504305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99362024-02-12T01:38:49.127Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012534964Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99372024-02-12T01:38:49.127Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012561362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99382024-02-12T01:38:49.127Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:48.012708815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99392024-02-12T01:38:49.127Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:48.012772712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99402024-02-12T01:38:49.127Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:48.012811031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99412024-02-12T01:38:49.127Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01282901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99422024-02-12T01:38:49.127Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012863098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99432024-02-12T01:38:49.127Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012889637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99442024-02-12T01:38:49.127Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.012987902Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99452024-02-12T01:38:49.127Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.013412512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99462024-02-12T01:38:49.128Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01345834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99472024-02-12T01:38:49.128Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.013501638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99482024-02-12T01:38:49.128Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.013547725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99492024-02-12T01:38:49.128Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014039102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99502024-02-12T01:38:49.128Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01407239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99512024-02-12T01:38:49.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014572107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99522024-02-12T01:38:49.128Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014609185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99532024-02-12T01:38:49.128Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014634853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99542024-02-12T01:38:49.128Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014660412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99552024-02-12T01:38:49.128Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.014675931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99562024-02-12T01:38:49.128Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:48.01469649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99572024-02-12T01:38:50.123Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.014364827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99582024-02-12T01:38:50.123Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.0145129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99592024-02-12T01:38:50.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.015434826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99602024-02-12T01:38:50.129Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.015477004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99612024-02-12T01:38:50.129Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.015952921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99622024-02-12T01:38:50.130Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016463197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99632024-02-12T01:38:50.130Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016514855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99642024-02-12T01:38:50.130Z{"msg":"[0]R flush_numbers[0..12]: [145, 152, 151, 144, 144, 137, 149, 152, 152, 149, 152, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016540933Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99652024-02-12T01:38:50.130Z{"msg":"[0]R generation[0..12]: [84, 89, 88, 84, 84, 80, 86, 89, 89, 86, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016577032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99662024-02-12T01:38:50.130Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016596761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99672024-02-12T01:38:50.130Z{"msg":"[1]R flush_numbers[0..12]: [145, 152, 151, 144, 144, 137, 149, 152, 152, 149, 152, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016623429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99682024-02-12T01:38:50.130Z{"msg":"[1]R generation[0..12]: [84, 89, 88, 84, 84, 80, 86, 89, 89, 86, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016641109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99692024-02-12T01:38:50.130Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016666978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99702024-02-12T01:38:50.130Z{"msg":"[2]R flush_numbers[0..12]: [145, 152, 151, 144, 144, 137, 149, 152, 152, 149, 152, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016686326Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99712024-02-12T01:38:50.130Z{"msg":"[2]R generation[0..12]: [84, 89, 88, 84, 84, 80, 86, 89, 89, 86, 89, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016711335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99722024-02-12T01:38:50.130ZWait for a query_work_queue command to finish before sending IO
99732024-02-12T01:38:50.130Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016737254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99742024-02-12T01:38:50.130Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016762443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99752024-02-12T01:38:50.131Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016780832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99762024-02-12T01:38:50.131Z{"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016804911Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99772024-02-12T01:38:50.131Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.01682166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99782024-02-12T01:38:50.131Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016846399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99792024-02-12T01:38:50.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016861998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99802024-02-12T01:38:50.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016888107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99812024-02-12T01:38:50.131Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016905986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99822024-02-12T01:38:50.131Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016930375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99832024-02-12T01:38:50.131Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016946204Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99842024-02-12T01:38:50.131Z{"msg":"80ab4f74-6951-4fe7-ab60-fb1d53208107 is now active with session: 3097c555-9395-4f2a-91c8-f8d377f7098a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016970193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99852024-02-12T01:38:50.131Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.016985852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495,"session_id":"3097c555-9395-4f2a-91c8-f8d377f7098a"}
99862024-02-12T01:38:50.131Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:49.017009151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
99872024-02-12T01:38:50.131ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99882024-02-12T01:38:50.131ZLoading write count information from file "/tmp/test_repair_verify.data"
99892024-02-12T01:38:50.131ZRun Repair workload
99902024-02-12T01:38:50.137Z01/30 Write block 9 len 32768 data: 7 6 7 8 7 8 9 8
99912024-02-12T01:38:50.137Z02/30 Write block 273 len 24576 data: 6 7 8 8 8 9
99922024-02-12T01:38:50.137Z03/30 Read block 91 len 36864
99932024-02-12T01:38:50.138Z04/30 Write block 393 len 36864 data: 11 8 7 6 6 6 5 5 4
99942024-02-12T01:38:50.138Z05/30 Write block 372 len 8192 data: 10 11
99952024-02-12T01:38:50.138Z06/30 Read block 469 len 28672
99962024-02-12T01:38:50.138Z07/30 Write block 31 len 4096 data: 12
99972024-02-12T01:38:50.138Z08/30 Read block 205 len 40960
99982024-02-12T01:38:50.138Z09/30 Flush
99992024-02-12T01:38:50.142Z10/30 Write block 424 len 12288 data: 6 5 4
100002024-02-12T01:38:50.142Z11/30 Read block 503 len 40960
100012024-02-12T01:38:50.142Z12/30 Read block 3 len 24576
100022024-02-12T01:38:50.142Z13/30 Read block 252 len 32768
100032024-02-12T01:38:50.142Z14/30 Read block 57 len 20480
100042024-02-12T01:38:50.145Z15/30 Read block 586 len 4096
100052024-02-12T01:38:50.145Z16/30 Write block 142 len 36864 data: 13 13 13 13 10 9 10 11 12
100062024-02-12T01:38:50.145Z17/30 Write block 33 len 24576 data: 14 13 9 10 8 11
100072024-02-12T01:38:50.145Z18/30 Read block 539 len 36864
100082024-02-12T01:38:50.148Z19/30 Read block 314 len 4096
100092024-02-12T01:38:50.148Z20/30 Read block 518 len 28672
100102024-02-12T01:38:50.148Z21/30 Flush
100112024-02-12T01:38:50.151Z22/30 Write block 307 len 16384 data: 10 11 8 8
100122024-02-12T01:38:50.151Z23/30 Read block 126 len 12288
100132024-02-12T01:38:50.154Z24/30 Write block 44 len 32768 data: 9 6 6 8 9 11 10 11
100142024-02-12T01:38:50.154Z25/30 Write block 237 len 32768 data: 8 9 8 7 5 6 7 5
100152024-02-12T01:38:50.155Z26/30 Read block 312 len 24576
100162024-02-12T01:38:50.155Z27/30 Read block 26 len 20480
100172024-02-12T01:38:50.159Z28/30 Read block 348 len 12288
100182024-02-12T01:38:50.159Z29/30 Read block 46 len 16384
100192024-02-12T01:38:50.159Z30/30 Read block 555 len 28672
100202024-02-12T01:38:50.159Z----------------------------------------------------------------
100212024-02-12T01:38:50.159Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
100222024-02-12T01:38:50.159ZStates: Active Active Active
100232024-02-12T01:38:50.159ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
100242024-02-12T01:38:50.159Z 1 Acked 1000 Write 8 Sent Done Done false
100252024-02-12T01:38:50.159Z 2 Acked 1001 Write 6 Sent Done Done false
100262024-02-12T01:38:50.159Z 3 Acked 1002 Read 9 Sent Done Done false
100272024-02-12T01:38:50.159Z 4 Acked 1003 Write 9 Sent Done Done false
100282024-02-12T01:38:50.159Z 5 Acked 1004 Write 2 Sent Done Done false
100292024-02-12T01:38:50.160Z 6 Acked 1005 Read 7 Sent Done Done false
100302024-02-12T01:38:50.160Z 7 Acked 1006 Write 1 Sent Done Done false
100312024-02-12T01:38:50.160Z 8 Acked 1007 Read 10 Sent Done Done false
100322024-02-12T01:38:50.160Z 9 Acked 1008 Flush 0 Sent Done Done false
100332024-02-12T01:38:50.160Z 10 Acked 1009 Write 3 Sent Done Done false
100342024-02-12T01:38:50.160Z 11 Acked 1010 Read 10 Sent Done Done false
100352024-02-12T01:38:50.160Z 12 Acked 1011 Read 6 Sent Done Done false
100362024-02-12T01:38:50.160Z 13 Acked 1012 Read 8 Sent Done Done false
100372024-02-12T01:38:50.160Z 14 Acked 1013 Read 5 Sent Done Done false
100382024-02-12T01:38:50.160Z 15 Acked 1014 Read 1 Sent Done Done false
100392024-02-12T01:38:50.160Z 16 Acked 1015 Write 9 Sent Done Done false
100402024-02-12T01:38:50.160Z 17 Acked 1016 Write 6 Sent Done Done false
100412024-02-12T01:38:50.161Z 18 Acked 1017 Read 9 Sent Done Done false
100422024-02-12T01:38:50.161Z 19 Acked 1018 Read 1 Sent Done Done false
100432024-02-12T01:38:50.161Z 20 Acked 1019 Read 7 Sent Done Done false
100442024-02-12T01:38:50.161Z 21 Acked 1020 Flush 0 Sent Done Done false
100452024-02-12T01:38:50.161Z 22 Acked 1021 Write 4 Sent Done Done false
100462024-02-12T01:38:50.161Z 23 Acked 1022 Read 3 Sent Done Done false
100472024-02-12T01:38:50.161Z 24 Acked 1023 Write 8 Sent Done Done false
100482024-02-12T01:38:50.161Z 25 Acked 1024 Write 8 Sent Done Done false
100492024-02-12T01:38:50.161Z 26 Acked 1025 Read 6 Sent Done Done false
100502024-02-12T01:38:50.161Z 27 Acked 1026 Read 5 Sent Done Done false
100512024-02-12T01:38:50.161Z 28 Acked 1027 Read 3 Sent Done Done false
100522024-02-12T01:38:50.161Z 29 Acked 1028 Read 4 Sent Done Done false
100532024-02-12T01:38:50.162Z 30 Acked 1029 Read 7 Sent Done Done false
100542024-02-12T01:38:50.162Z STATES DS:0 DS:1 DS:2 TOTAL
100552024-02-12T01:38:50.162Z New 0 0 0 0
100562024-02-12T01:38:50.162Z Sent 30 0 0 30
100572024-02-12T01:38:50.162Z Done 0 30 30 60
100582024-02-12T01:38:50.162Z Skipped 0 0 0 0
100592024-02-12T01:38:50.162Z Error 0 0 0 0
100602024-02-12T01:38:50.162ZLast Flush: 0 1020 1020
100612024-02-12T01:38:50.162ZDownstairs last five completed:
100622024-02-12T01:38:50.162ZUpstairs last five completed: 30 29 28 27 26
100632024-02-12T01:38:50.162Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:49.049439934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1495}
100642024-02-12T01:38:50.162ZWrote out file "/tmp/test_repair_verify.data"
100652024-02-12T01:38:50.167Z
100662024-02-12T01:38:50.167ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100672024-02-12T01:38:50.182ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100682024-02-12T01:38:50.182Z 0 000-019 84 90 90 145 153 153 F F F <---
100692024-02-12T01:38:50.182Z 1 020-039 89 90 90 152 154 154 F F F <---
100702024-02-12T01:38:50.182Z 2 040-059 88 88 88 151 151 151 F T T <---
100712024-02-12T01:38:50.182Z 3 060-079 84 84 84 144 144 144 F F F
100722024-02-12T01:38:50.182Z 4 080-099 84 84 84 144 144 144 F F F
100732024-02-12T01:38:50.186Z 5 100-119 80 80 80 137 137 137 F F F
100742024-02-12T01:38:50.186Z 6 120-139 86 86 86 149 149 149 F F F
100752024-02-12T01:38:50.186Z 7 140-159 89 90 90 152 154 154 F F F <---
100762024-02-12T01:38:50.187Z 8 160-179 89 89 89 152 152 152 F F F
100772024-02-12T01:38:50.187Z 9 180-199 86 86 86 149 149 149 F F F
100782024-02-12T01:38:50.187Z 10 200-219 89 89 89 152 152 152 F F F
100792024-02-12T01:38:50.187Z 11 220-239 80 80 80 137 137 137 F T T <---
100802024-02-12T01:38:50.187Z 12 240-259 88 88 88 151 151 151 F T T <---
100812024-02-12T01:38:50.187Z 13 260-279 87 90 90 150 153 153 F F F <---
100822024-02-12T01:38:50.187Z 14 280-299 88 88 88 151 151 151 F F F
100832024-02-12T01:38:50.187Z 15 300-319 88 88 88 151 151 151 F T T <---
100842024-02-12T01:38:50.187Z 16 320-339 74 74 74 131 131 131 F F F
100852024-02-12T01:38:50.187Z 17 340-359 89 89 89 152 152 152 F F F
100862024-02-12T01:38:50.187Z 18 360-379 86 90 90 149 153 153 F F F <---
100872024-02-12T01:38:50.187Z 19 380-399 86 90 90 149 153 153 F F F <---
100882024-02-12T01:38:50.187Z 20 400-419 88 90 90 151 153 153 F F F <---
100892024-02-12T01:38:50.187Z 21 420-439 88 90 90 151 154 154 F F F <---
100902024-02-12T01:38:50.187Z 22 440-459 88 88 88 151 151 151 F F F
100912024-02-12T01:38:50.187Z 23 460-479 84 84 84 145 145 145 F F F
100922024-02-12T01:38:50.187Z 24 480-499 76 76 76 132 132 132 F F F
100932024-02-12T01:38:50.187Z 25 500-519 89 89 89 152 152 152 F F F
100942024-02-12T01:38:50.188Z 26 520-539 88 88 88 151 151 151 F F F
100952024-02-12T01:38:50.188Z 27 540-559 88 88 88 151 151 151 F F F
100962024-02-12T01:38:50.188Z 28 560-579 88 88 88 151 151 151 F F F
100972024-02-12T01:38:50.188Z 29 580-599 88 88 88 151 151 151 F F F
100982024-02-12T01:38:50.188ZMax gen: 90, Max flush: 154
100992024-02-12T01:38:50.188ZError: Difference in extent metadata found!
101002024-02-12T01:38:50.188ZA Difference in extent metadata is expected here
101012024-02-12T01:38:50.188ZOn loop 45
101022024-02-12T01:38:50.188Z
101032024-02-12T01:38:50.192ZVerifying data now
101042024-02-12T01:38:54.794ZNo change after verify
101052024-02-12T01:38:54.794ZLoop: 45 Downstairs dump after verify (and repair):
101062024-02-12T01:38:54.810ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101072024-02-12T01:38:54.810Z 0 000-019 90 90 90 153 153 153 F F F
101082024-02-12T01:38:54.810Z 1 020-039 90 90 90 154 154 154 F F F
101092024-02-12T01:38:54.810Z 2 040-059 91 91 91 155 155 155 F F F
101102024-02-12T01:38:54.810Z 3 060-079 84 84 84 144 144 144 F F F
101112024-02-12T01:38:54.810Z 4 080-099 84 84 84 144 144 144 F F F
101122024-02-12T01:38:54.810Z 5 100-119 80 80 80 137 137 137 F F F
101132024-02-12T01:38:54.810Z 6 120-139 86 86 86 149 149 149 F F F
101142024-02-12T01:38:54.810Z 7 140-159 90 90 90 154 154 154 F F F
101152024-02-12T01:38:54.810Z 8 160-179 89 89 89 152 152 152 F F F
101162024-02-12T01:38:54.810Z 9 180-199 86 86 86 149 149 149 F F F
101172024-02-12T01:38:54.810Z 10 200-219 89 89 89 152 152 152 F F F
101182024-02-12T01:38:54.810Z 11 220-239 91 91 91 155 155 155 F F F
101192024-02-12T01:38:54.810Z 12 240-259 91 91 91 155 155 155 F F F
101202024-02-12T01:38:54.813Z 13 260-279 90 90 90 153 153 153 F F F
101212024-02-12T01:38:54.814Z 14 280-299 88 88 88 151 151 151 F F F
101222024-02-12T01:38:54.814Z 15 300-319 91 91 91 155 155 155 F F F
101232024-02-12T01:38:54.814Z 16 320-339 74 74 74 131 131 131 F F F
101242024-02-12T01:38:54.814Z 17 340-359 89 89 89 152 152 152 F F F
101252024-02-12T01:38:54.814Z 18 360-379 90 90 90 153 153 153 F F F
101262024-02-12T01:38:54.814Z 19 380-399 90 90 90 153 153 153 F F F
101272024-02-12T01:38:54.814Z 20 400-419 90 90 90 153 153 153 F F F
101282024-02-12T01:38:54.814Z 21 420-439 90 90 90 154 154 154 F F F
101292024-02-12T01:38:54.814Z 22 440-459 88 88 88 151 151 151 F F F
101302024-02-12T01:38:54.814Z 23 460-479 84 84 84 145 145 145 F F F
101312024-02-12T01:38:54.814Z 24 480-499 76 76 76 132 132 132 F F F
101322024-02-12T01:38:54.814Z 25 500-519 89 89 89 152 152 152 F F F
101332024-02-12T01:38:54.814Z 26 520-539 88 88 88 151 151 151 F F F
101342024-02-12T01:38:54.814Z 27 540-559 88 88 88 151 151 151 F F F
101352024-02-12T01:38:54.814Z 28 560-579 88 88 88 151 151 151 F F F
101362024-02-12T01:38:54.814Z 29 580-599 88 88 88 151 151 151 F F F
101372024-02-12T01:38:54.814ZMax gen: 91, Max flush: 155
101382024-02-12T01:38:54.814Z
101392024-02-12T01:38:54.814ZBegin loop 46
101402024-02-12T01:38:54.814ZDownstairs to restart: 1
101412024-02-12T01:38:54.838Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
101422024-02-12T01:38:54.838Z,"time":"2024-02-12T01:38:53.726371041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
101432024-02-12T01:38:54.843Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726641308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
101442024-02-12T01:38:54.843Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726674056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
101452024-02-12T01:38:54.843Z{"msg":"Crucible f256878a-07d1-4102-a7af-0e78c4a18ad9 has session id: ef6bea28-b0d1-4c90-b9d9-c03a62ad984b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726687546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101462024-02-12T01:38:54.843Z{"msg":"Upstairs opts: Upstairs UUID: f256878a-07d1-4102-a7af-0e78c4a18ad9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726716624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101472024-02-12T01:38:54.843Z{"msg":"Crucible stats registered with UUID: f256878a-07d1-4102-a7af-0e78c4a18ad9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726739133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101482024-02-12T01:38:54.843Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726774671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
101492024-02-12T01:38:54.843Z{"msg":"f256878a-07d1-4102-a7af-0e78c4a18ad9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726796181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101502024-02-12T01:38:54.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726817889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101512024-02-12T01:38:54.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726865237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101522024-02-12T01:38:54.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726890726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101532024-02-12T01:38:54.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726914355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101542024-02-12T01:38:54.844Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726950953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101552024-02-12T01:38:54.844Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.726974822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101562024-02-12T01:38:54.844Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.72700745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101572024-02-12T01:38:54.844Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727041469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101582024-02-12T01:38:54.844Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727077307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101592024-02-12T01:38:54.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727104186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101602024-02-12T01:38:54.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727128994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101612024-02-12T01:38:54.844Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:53.727178542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101622024-02-12T01:38:54.844Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:53.727207621Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101632024-02-12T01:38:54.844Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:53.727326795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101642024-02-12T01:38:54.844Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727360343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101652024-02-12T01:38:54.844Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727410171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101662024-02-12T01:38:54.844Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.727665519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101672024-02-12T01:38:54.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728165645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101682024-02-12T01:38:54.844Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728210923Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101692024-02-12T01:38:54.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728235802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101702024-02-12T01:38:54.844Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728259141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101712024-02-12T01:38:54.844Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728777946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101722024-02-12T01:38:54.844Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.728845263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101732024-02-12T01:38:54.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729401186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101742024-02-12T01:38:54.844Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729436144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101752024-02-12T01:38:54.844Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729452314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101762024-02-12T01:38:54.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729465223Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101772024-02-12T01:38:54.845Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729486772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101782024-02-12T01:38:54.845Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:53.729500891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101792024-02-12T01:38:55.838Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.729270233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101802024-02-12T01:38:55.838Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.729422026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101812024-02-12T01:38:55.843Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.730283644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101822024-02-12T01:38:55.843Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.730319563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101832024-02-12T01:38:55.843Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.73079064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101842024-02-12T01:38:55.843Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731299476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101852024-02-12T01:38:55.843Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731355763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101862024-02-12T01:38:55.843Z{"msg":"[0]R flush_numbers[0..12]: [153, 154, 155, 144, 144, 137, 149, 154, 152, 149, 152, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731397941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101872024-02-12T01:38:55.844Z{"msg":"[0]R generation[0..12]: [90, 90, 91, 84, 84, 80, 86, 90, 89, 86, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.73141853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101882024-02-12T01:38:55.844Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731446449Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101892024-02-12T01:38:55.844Z{"msg":"[1]R flush_numbers[0..12]: [153, 154, 155, 144, 144, 137, 149, 154, 152, 149, 152, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731466048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101902024-02-12T01:38:55.844Z{"msg":"[1]R generation[0..12]: [90, 90, 91, 84, 84, 80, 86, 90, 89, 86, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731493127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101912024-02-12T01:38:55.844Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
101922024-02-12T01:38:55.844Z,"time":"2024-02-12T01:38:54.731511486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101932024-02-12T01:38:55.844Z{"msg":"[2]R flush_numbers[0..12]: [153, 154, 155, 144, 144, 137, 149, 154, 152, 149, 152, 155]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731533385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101942024-02-12T01:38:55.844Z{"msg":"[2]R generation[0..12]: [90, 90, 91, 84, 84, 80, 86, 90, 89, 86, 89, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731558713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101952024-02-12T01:38:55.844Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731576443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101962024-02-12T01:38:55.844Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731600762Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101972024-02-12T01:38:55.844Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731618181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101982024-02-12T01:38:55.844Z{"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731642089Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
101992024-02-12T01:38:55.844Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731659069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102002024-02-12T01:38:55.844Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731684347Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102012024-02-12T01:38:55.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731699587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102022024-02-12T01:38:55.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731724376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102032024-02-12T01:38:55.844Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731741465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102042024-02-12T01:38:55.844Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731765824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102052024-02-12T01:38:55.844Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731781443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102062024-02-12T01:38:55.844Z{"msg":"f256878a-07d1-4102-a7af-0e78c4a18ad9 is now active with session: ef6bea28-b0d1-4c90-b9d9-c03a62ad984b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731798232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102072024-02-12T01:38:55.844Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.731820961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503,"session_id":"ef6bea28-b0d1-4c90-b9d9-c03a62ad984b"}
102082024-02-12T01:38:55.844Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:38:54.7318365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
102092024-02-12T01:38:55.844ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
102102024-02-12T01:38:55.844ZLoading write count information from file "/tmp/test_repair_verify.data"
102112024-02-12T01:38:55.845ZRun Repair workload
102122024-02-12T01:38:55.845Z01/30 Write block 227 len 4096 data: 6
102132024-02-12T01:38:55.848Z02/30 Write block 237 len 40960 data: 9 10 9 8 6 7 8 6 5 6
102142024-02-12T01:38:55.848Z03/30 Write block 58 len 8192 data: 11 8
102152024-02-12T01:38:55.848Z04/30 Write block 177 len 20480 data: 10 11 12 9 10
102162024-02-12T01:38:55.848Z05/30 Read block 433 len 16384
102172024-02-12T01:38:55.848Z06/30 Read block 118 len 28672
102182024-02-12T01:38:55.848Z07/30 Read block 370 len 28672
102192024-02-12T01:38:55.851Z08/30 Read block 590 len 24576
102202024-02-12T01:38:55.851Z09/30 Flush
102212024-02-12T01:38:55.854Z10/30 Write block 379 len 16384 data: 9 7 8 7
102222024-02-12T01:38:55.854Z11/30 Read block 21 len 40960
102232024-02-12T01:38:55.854Z12/30 Write block 291 len 40960 data: 8 6 6 6 8 6 6 6 6 6
102242024-02-12T01:38:55.857Z13/30 Flush
102252024-02-12T01:38:55.860Z14/30 Flush
102262024-02-12T01:38:55.860Z15/30 Write block 192 len 24576 data: 7 6 4 3 4 6
102272024-02-12T01:38:55.860Z16/30 Write block 339 len 20480 data: 5 4 3 3 5
102282024-02-12T01:38:55.860Z17/30 Write block 80 len 36864 data: 12 12 13 10 11 11 10 12 12
102292024-02-12T01:38:55.866Z18/30 Write block 481 len 28672 data: 7 7 8 8 8 9 9
102302024-02-12T01:38:55.866Z19/30 Read block 311 len 32768
102312024-02-12T01:38:55.866Z20/30 Write block 569 len 4096 data: 11
102322024-02-12T01:38:55.869Z21/30 Write block 490 len 40960 data: 8 5 4 4 5 8 7 6 7 7
102332024-02-12T01:38:55.869Z22/30 Read block 591 len 36864
102342024-02-12T01:38:55.869Z23/30 Write block 574 len 20480 data: 12 10 7 8 8
102352024-02-12T01:38:55.869Z24/30 Read block 434 len 40960
102362024-02-12T01:38:55.869Z25/30 Read block 132 len 12288
102372024-02-12T01:38:55.869Z26/30 Write block 534 len 4096 data: 11
102382024-02-12T01:38:55.870Z27/30 Write block 510 len 8192 data: 8 10
102392024-02-12T01:38:55.873Z28/30 Write block 78 len 8192 data: 7 10
102402024-02-12T01:38:55.873Z29/30 Read block 367 len 8192
102412024-02-12T01:38:55.873Z30/30 Write block 406 len 16384 data: 8 6 7 8
102422024-02-12T01:38:55.873Z----------------------------------------------------------------
102432024-02-12T01:38:55.873Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
102442024-02-12T01:38:55.873ZStates: Active Active Active
102452024-02-12T01:38:55.873ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102462024-02-12T01:38:55.873Z 1 Acked 1000 Write 1 Done Done Done false
102472024-02-12T01:38:55.873Z 2 Acked 1001 Write 10 Done Done Done false
102482024-02-12T01:38:55.873Z 3 Acked 1002 Write 2 Done Done Done false
102492024-02-12T01:38:55.874Z 4 Acked 1003 Write 5 Done Sent Done false
102502024-02-12T01:38:55.874Z 5 Acked 1004 Read 4 Done Sent Done false
102512024-02-12T01:38:55.874Z 6 Acked 1005 Read 7 Done Sent Done false
102522024-02-12T01:38:55.874Z 7 Acked 1006 Read 7 Done Sent Done false
102532024-02-12T01:38:55.874Z 8 Acked 1007 Read 6 Done Sent Done false
102542024-02-12T01:38:55.874Z 9 Acked 1008 Flush 0 Done Sent Done false
102552024-02-12T01:38:55.874Z 10 Acked 1009 Write 4 Done Sent Done false
102562024-02-12T01:38:55.874Z 11 Acked 1010 Read 10 Done Sent Done false
102572024-02-12T01:38:55.874Z 12 Acked 1011 Write 10 Done Sent Done false
102582024-02-12T01:38:55.874Z 13 Acked 1012 Flush 0 Done Sent Done false
102592024-02-12T01:38:55.874Z 14 Acked 1013 Flush 0 Done Sent Done false
102602024-02-12T01:38:55.874Z 15 Acked 1014 Write 6 Done Sent Done false
102612024-02-12T01:38:55.874Z 16 Acked 1015 Write 5 Done Sent Done false
102622024-02-12T01:38:55.874Z 17 Acked 1016 Write 9 Done Sent Done false
102632024-02-12T01:38:55.874Z 18 Acked 1017 Write 7 Done Sent Done false
102642024-02-12T01:38:55.874Z 19 Acked 1018 Read 8 Done Sent Done false
102652024-02-12T01:38:55.874Z 20 Acked 1019 Write 1 Done Sent Done false
102662024-02-12T01:38:55.874Z 21 Acked 1020 Write 10 Done Sent Done false
102672024-02-12T01:38:55.874Z 22 Acked 1021 Read 9 Done Sent Done false
102682024-02-12T01:38:55.874Z 23 Acked 1022 Write 5 Done Sent Done false
102692024-02-12T01:38:55.874Z 24 Acked 1023 Read 10 Done Sent Done false
102702024-02-12T01:38:55.874Z 25 Acked 1024 Read 3 Done Sent Done false
102712024-02-12T01:38:55.874Z 26 Acked 1025 Write 1 Done Sent Done false
102722024-02-12T01:38:55.874Z 27 Acked 1026 Write 2 Done Sent Done false
102732024-02-12T01:38:55.875Z 28 Acked 1027 Write 2 Done Sent Done false
102742024-02-12T01:38:55.875Z 29 Acked 1028 Read 2 Done Sent Done false
102752024-02-12T01:38:55.875Z 30 Acked 1029 Write 4 Sent Sent Sent false
102762024-02-12T01:38:55.875Z STATES DS:0 DS:1 DS:2 TOTAL
102772024-02-12T01:38:55.875Z New 0 0 0 0
102782024-02-12T01:38:55.875Z Sent 1 27 1 29
102792024-02-12T01:38:55.875Z Done 29 3 29 61
102802024-02-12T01:38:55.875Z Skipped 0 0 0 0
102812024-02-12T01:38:55.875Z Error 0 0 0 0
102822024-02-12T01:38:55.875ZLast Flush: 1013 0 1013
102832024-02-12T01:38:55.875ZDownstairs last five completed:
102842024-02-12T01:38:55.875ZUpstairs last five completed: 30 29 28 27 26
102852024-02-12T01:38:55.875Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:38:54.763192974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1503}
102862024-02-12T01:38:55.875ZWrote out file "/tmp/test_repair_verify.data"
102872024-02-12T01:38:55.878Z
102882024-02-12T01:38:55.881ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102892024-02-12T01:38:55.899ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102902024-02-12T01:38:55.900Z 0 000-019 90 90 90 153 153 153 F F F
102912024-02-12T01:38:55.900Z 1 020-039 90 90 90 154 154 154 F F F
102922024-02-12T01:38:55.900Z 2 040-059 92 91 92 156 155 156 F T F <---
102932024-02-12T01:38:55.900Z 3 060-079 84 84 84 144 144 144 T F T <---
102942024-02-12T01:38:55.900Z 4 080-099 84 84 84 144 144 144 T F T <---
102952024-02-12T01:38:55.900Z 5 100-119 80 80 80 137 137 137 F F F
102962024-02-12T01:38:55.903Z 6 120-139 86 86 86 149 149 149 F F F
102972024-02-12T01:38:55.903Z 7 140-159 90 90 90 154 154 154 F F F
102982024-02-12T01:38:55.903Z 8 160-179 92 89 92 156 152 156 F F F <---
102992024-02-12T01:38:55.903Z 9 180-199 92 86 92 156 149 156 T F T <---
103002024-02-12T01:38:55.903Z 10 200-219 89 89 89 152 152 152 F F F
103012024-02-12T01:38:55.903Z 11 220-239 92 91 92 156 155 156 F T F <---
103022024-02-12T01:38:55.903Z 12 240-259 92 91 92 156 155 156 F T F <---
103032024-02-12T01:38:55.903Z 13 260-279 90 90 90 153 153 153 F F F
103042024-02-12T01:38:55.903Z 14 280-299 92 88 92 157 151 157 F F F <---
103052024-02-12T01:38:55.903Z 15 300-319 92 91 92 157 155 157 F F F <---
103062024-02-12T01:38:55.903Z 16 320-339 74 74 74 131 131 131 T F T <---
103072024-02-12T01:38:55.903Z 17 340-359 89 89 89 152 152 152 T F T <---
103082024-02-12T01:38:55.903Z 18 360-379 92 90 92 157 153 157 F F F <---
103092024-02-12T01:38:55.903Z 19 380-399 92 90 92 157 153 157 F F F <---
103102024-02-12T01:38:55.903Z 20 400-419 90 90 90 153 153 153 T F T <---
103112024-02-12T01:38:55.904Z 21 420-439 90 90 90 154 154 154 F F F
103122024-02-12T01:38:55.904Z 22 440-459 88 88 88 151 151 151 F F F
103132024-02-12T01:38:55.904Z 23 460-479 84 84 84 145 145 145 F F F
103142024-02-12T01:38:55.904Z 24 480-499 76 76 76 132 132 132 T F T <---
103152024-02-12T01:38:55.904Z 25 500-519 89 89 89 152 152 152 T F T <---
103162024-02-12T01:38:55.904Z 26 520-539 88 88 88 151 151 151 T F T <---
103172024-02-12T01:38:55.904Z 27 540-559 88 88 88 151 151 151 F F F
103182024-02-12T01:38:55.904Z 28 560-579 88 88 88 151 151 151 T F T <---
103192024-02-12T01:38:55.904Z 29 580-599 88 88 88 151 151 151 F F F
103202024-02-12T01:38:55.904ZMax gen: 92, Max flush: 157
103212024-02-12T01:38:55.904ZError: Difference in extent metadata found!
103222024-02-12T01:38:55.904ZA Difference in extent metadata is expected here
103232024-02-12T01:38:55.904ZOn loop 46
103242024-02-12T01:38:55.904Z
103252024-02-12T01:38:55.907ZVerifying data now
103262024-02-12T01:39:01.759ZNo change after verify
103272024-02-12T01:39:01.759ZLoop: 46 Downstairs dump after verify (and repair):
103282024-02-12T01:39:01.774ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
103292024-02-12T01:39:01.774Z 0 000-019 90 90 90 153 153 153 F F F
103302024-02-12T01:39:01.774Z 1 020-039 90 90 90 154 154 154 F F F
103312024-02-12T01:39:01.774Z 2 040-059 92 92 92 156 156 156 F F F
103322024-02-12T01:39:01.774Z 3 060-079 93 93 93 158 158 158 F F F
103332024-02-12T01:39:01.774Z 4 080-099 93 93 93 158 158 158 F F F
103342024-02-12T01:39:01.774Z 5 100-119 80 80 80 137 137 137 F F F
103352024-02-12T01:39:01.774Z 6 120-139 86 86 86 149 149 149 F F F
103362024-02-12T01:39:01.774Z 7 140-159 90 90 90 154 154 154 F F F
103372024-02-12T01:39:01.774Z 8 160-179 92 92 92 156 156 156 F F F
103382024-02-12T01:39:01.774Z 9 180-199 93 93 93 158 158 158 F F F
103392024-02-12T01:39:01.774Z 10 200-219 89 89 89 152 152 152 F F F
103402024-02-12T01:39:01.774Z 11 220-239 92 92 92 156 156 156 F F F
103412024-02-12T01:39:01.774Z 12 240-259 92 92 92 156 156 156 F F F
103422024-02-12T01:39:01.774Z 13 260-279 90 90 90 153 153 153 F F F
103432024-02-12T01:39:01.778Z 14 280-299 92 92 92 157 157 157 F F F
103442024-02-12T01:39:01.778Z 15 300-319 92 92 92 157 157 157 F F F
103452024-02-12T01:39:01.778Z 16 320-339 93 93 93 158 158 158 F F F
103462024-02-12T01:39:01.778Z 17 340-359 93 93 93 158 158 158 F F F
103472024-02-12T01:39:01.778Z 18 360-379 92 92 92 157 157 157 F F F
103482024-02-12T01:39:01.778Z 19 380-399 92 92 92 157 157 157 F F F
103492024-02-12T01:39:01.778Z 20 400-419 93 93 93 158 158 158 F F F
103502024-02-12T01:39:01.778Z 21 420-439 90 90 90 154 154 154 F F F
103512024-02-12T01:39:01.778Z 22 440-459 88 88 88 151 151 151 F F F
103522024-02-12T01:39:01.778Z 23 460-479 84 84 84 145 145 145 F F F
103532024-02-12T01:39:01.778Z 24 480-499 93 93 93 158 158 158 F F F
103542024-02-12T01:39:01.778Z 25 500-519 93 93 93 158 158 158 F F F
103552024-02-12T01:39:01.778Z 26 520-539 93 93 93 158 158 158 F F F
103562024-02-12T01:39:01.778Z 27 540-559 88 88 88 151 151 151 F F F
103572024-02-12T01:39:01.778Z 28 560-579 93 93 93 158 158 158 F F F
103582024-02-12T01:39:01.778Z 29 580-599 88 88 88 151 151 151 F F F
103592024-02-12T01:39:01.778ZMax gen: 93, Max flush: 158
103602024-02-12T01:39:01.778Z
103612024-02-12T01:39:01.778ZBegin loop 47
103622024-02-12T01:39:01.778ZDownstairs to restart: 1
103632024-02-12T01:39:01.794Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.684837358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
103642024-02-12T01:39:01.795Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
103652024-02-12T01:39:01.795Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685134524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
103662024-02-12T01:39:01.800Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685173072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
103672024-02-12T01:39:01.801Z{"msg":"Crucible 287f3cbd-91c1-491a-9ad2-09b45a845857 has session id: 3ace3daf-940a-4872-9de3-1688448aa0be","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685205571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103682024-02-12T01:39:01.801Z{"msg":"Upstairs opts: Upstairs UUID: 287f3cbd-91c1-491a-9ad2-09b45a845857, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685243359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103692024-02-12T01:39:01.801Z{"msg":"Crucible stats registered with UUID: 287f3cbd-91c1-491a-9ad2-09b45a845857","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685260348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103702024-02-12T01:39:01.801Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685278847Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
103712024-02-12T01:39:01.801Z{"msg":"287f3cbd-91c1-491a-9ad2-09b45a845857 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685409831Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103722024-02-12T01:39:01.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685467268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103732024-02-12T01:39:01.801Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685505756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103742024-02-12T01:39:01.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685550484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103752024-02-12T01:39:01.801Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685588362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103762024-02-12T01:39:01.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685623731Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103772024-02-12T01:39:01.801Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685733845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103782024-02-12T01:39:01.801Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685772514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103792024-02-12T01:39:01.801Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685823051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103802024-02-12T01:39:01.801Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68585703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103812024-02-12T01:39:01.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685895908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103822024-02-12T01:39:01.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685916107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103832024-02-12T01:39:01.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.685944145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103842024-02-12T01:39:01.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.686447001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103852024-02-12T01:39:01.801Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68648396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103862024-02-12T01:39:01.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.686499649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103872024-02-12T01:39:01.801Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.686525418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103882024-02-12T01:39:01.801Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687033423Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103892024-02-12T01:39:01.802Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687074141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103902024-02-12T01:39:01.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687615656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103912024-02-12T01:39:01.802Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687679563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103922024-02-12T01:39:01.802Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687723221Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103932024-02-12T01:39:01.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68773914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103942024-02-12T01:39:01.802Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687760819Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103952024-02-12T01:39:01.802Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.687774798Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103962024-02-12T01:39:01.802Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.688076464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103972024-02-12T01:39:01.802Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.688108992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103982024-02-12T01:39:01.802Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.6885702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
103992024-02-12T01:39:01.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.688977391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104002024-02-12T01:39:01.807Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689023699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104012024-02-12T01:39:01.807Z{"msg":"[0]R flush_numbers[0..12]: [153, 154, 156, 158, 158, 137, 149, 154, 156, 158, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689043658Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104022024-02-12T01:39:01.807Z{"msg":"[0]R generation[0..12]: [90, 90, 92, 93, 93, 80, 86, 90, 92, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689066046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104032024-02-12T01:39:01.807Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689081026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104042024-02-12T01:39:01.807Z{"msg":"[1]R flush_numbers[0..12]: [153, 154, 156, 158, 158, 137, 149, 154, 156, 158, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689096635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104052024-02-12T01:39:01.807Z{"msg":"[1]R generation[0..12]: [90, 90, 92, 93, 93, 80, 86, 90, 92, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689117754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104062024-02-12T01:39:01.807Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689132433Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104072024-02-12T01:39:01.807Z{"msg":"[2]R flush_numbers[0..12]: [153, 154, 156, 158, 158, 137, 149, 154, 156, 158, 152, 156]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
104082024-02-12T01:39:01.807Z,"time":"2024-02-12T01:39:00.689147102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104092024-02-12T01:39:01.807Z{"msg":"[2]R generation[0..12]: [90, 90, 92, 93, 93, 80, 86, 90, 92, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689161842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104102024-02-12T01:39:01.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689175921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104112024-02-12T01:39:01.808Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68919606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104122024-02-12T01:39:01.808Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68921055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104132024-02-12T01:39:01.808Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689235098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104142024-02-12T01:39:01.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689249438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104152024-02-12T01:39:01.808Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689262757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104162024-02-12T01:39:01.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689275467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104172024-02-12T01:39:01.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689288766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104182024-02-12T01:39:01.808Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689313875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104192024-02-12T01:39:01.808Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689334474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104202024-02-12T01:39:01.808Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689354643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104212024-02-12T01:39:01.808Z{"msg":"287f3cbd-91c1-491a-9ad2-09b45a845857 is now active with session: 3ace3daf-940a-4872-9de3-1688448aa0be","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689368202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104222024-02-12T01:39:01.808Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.689392041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511,"session_id":"3ace3daf-940a-4872-9de3-1688448aa0be"}
104232024-02-12T01:39:01.808Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:00.68941093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
104242024-02-12T01:39:01.808ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
104252024-02-12T01:39:01.808ZLoading write count information from file "/tmp/test_repair_verify.data"
104262024-02-12T01:39:01.808ZRun Repair workload
104272024-02-12T01:39:01.808Z01/30 Write block 437 len 12288 data: 5 6 7
104282024-02-12T01:39:01.808Z02/30 Read block 200 len 20480
104292024-02-12T01:39:01.808Z03/30 Flush
104302024-02-12T01:39:01.808Z04/30 Write block 440 len 20480 data: 7 7 6 4 5
104312024-02-12T01:39:01.808Z05/30 Write block 148 len 16384 data: 11 12 13 12
104322024-02-12T01:39:01.808Z06/30 Read block 569 len 40960
104332024-02-12T01:39:01.812Z07/30 Write block 295 len 32768 data: 9 7 7 7 7 7 5 3
104342024-02-12T01:39:01.812Z08/30 Write block 367 len 16384 data: 10 8 9 9
104352024-02-12T01:39:01.812Z09/30 Write block 102 len 40960 data: 5 5 8 8 7 7 6 5 5 5
104362024-02-12T01:39:01.812Z10/30 Write block 109 len 40960 data: 6 6 6 5 7 9 7 8 10 11
104372024-02-12T01:39:01.812Z11/30 Flush
104382024-02-12T01:39:01.816Z12/30 Write block 317 len 24576 data: 5 6 7 6 7 8
104392024-02-12T01:39:01.816Z13/30 Flush
104402024-02-12T01:39:01.819Z14/30 Write block 502 len 16384 data: 8 6 6 6
104412024-02-12T01:39:01.819Z15/30 Write block 87 len 28672 data: 13 13 11 9 9 7 6
104422024-02-12T01:39:01.819Z16/30 Read block 270 len 12288
104432024-02-12T01:39:01.823Z17/30 Write block 304 len 8192 data: 6 7
104442024-02-12T01:39:01.823Z18/30 Flush
104452024-02-12T01:39:01.823Z19/30 Write block 309 len 8192 data: 9 9
104462024-02-12T01:39:01.826Z20/30 Write block 105 len 16384 data: 9 8 8 7
104472024-02-12T01:39:01.826Z21/30 Write block 567 len 28672 data: 8 10 12 11 9 11 13
104482024-02-12T01:39:01.826Z22/30 Read block 151 len 40960
104492024-02-12T01:39:01.829Z23/30 Write block 432 len 32768 data: 3 3 4 4 5 6 7 8
104502024-02-12T01:39:01.829Z24/30 Read block 28 len 4096
104512024-02-12T01:39:01.829Z25/30 Read block 14 len 20480
104522024-02-12T01:39:01.829Z26/30 Read block 313 len 32768
104532024-02-12T01:39:01.833Z27/30 Read block 440 len 28672
104542024-02-12T01:39:01.833Z28/30 Write block 160 len 24576 data: 7 8 8 8 8 10
104552024-02-12T01:39:01.833Z29/30 Write block 515 len 28672 data: 11 8 7 5 6 7 7
104562024-02-12T01:39:01.833Z30/30 Write block 458 len 4096 data: 8
104572024-02-12T01:39:01.833Z----------------------------------------------------------------
104582024-02-12T01:39:01.833Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
104592024-02-12T01:39:01.833ZStates: Active Active Active
104602024-02-12T01:39:01.833ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104612024-02-12T01:39:01.833Z 1 Acked 1000 Write 3 Done Done Done false
104622024-02-12T01:39:01.833Z 2 Acked 1001 Read 5 Done Done Done false
104632024-02-12T01:39:01.833Z 3 Acked 1002 Flush 0 Done Sent Done false
104642024-02-12T01:39:01.833Z 4 Acked 1003 Write 5 Done Sent Done false
104652024-02-12T01:39:01.833Z 5 Acked 1004 Write 4 Done Sent Done false
104662024-02-12T01:39:01.833Z 6 Acked 1005 Read 10 Done Sent Done false
104672024-02-12T01:39:01.834Z 7 Acked 1006 Write 8 Done Sent Done false
104682024-02-12T01:39:01.834Z 8 Acked 1007 Write 4 Done Sent Done false
104692024-02-12T01:39:01.834Z 9 Acked 1008 Write 10 Done Sent Done false
104702024-02-12T01:39:01.834Z 10 Acked 1009 Write 10 Done Sent Done false
104712024-02-12T01:39:01.834Z 11 Acked 1010 Flush 0 Done Sent Done false
104722024-02-12T01:39:01.834Z 12 Acked 1011 Write 6 Done Sent Done false
104732024-02-12T01:39:01.834Z 13 Acked 1012 Flush 0 Done Sent Done false
104742024-02-12T01:39:01.834Z 14 Acked 1013 Write 4 Done Sent Done false
104752024-02-12T01:39:01.834Z 15 Acked 1014 Write 7 Done Sent Done false
104762024-02-12T01:39:01.834Z 16 Acked 1015 Read 3 Done Sent Done false
104772024-02-12T01:39:01.834Z 17 Acked 1016 Write 2 Done Sent Done false
104782024-02-12T01:39:01.834Z 18 Acked 1017 Flush 0 Done Sent Done false
104792024-02-12T01:39:01.834Z 19 Acked 1018 Write 2 Done Sent Done false
104802024-02-12T01:39:01.834Z 20 Acked 1019 Write 4 Done Sent Done false
104812024-02-12T01:39:01.834Z 21 Acked 1020 Write 7 Done Sent Done false
104822024-02-12T01:39:01.834Z 22 Acked 1021 Read 10 Done Sent Done false
104832024-02-12T01:39:01.834Z 23 Acked 1022 Write 8 Done Sent Done false
104842024-02-12T01:39:01.834Z 24 Acked 1023 Read 1 Done Sent Done false
104852024-02-12T01:39:01.834Z 25 Acked 1024 Read 5 Done Sent Done false
104862024-02-12T01:39:01.834Z 26 Acked 1025 Read 8 Done Sent Done false
104872024-02-12T01:39:01.834Z 27 Acked 1026 Read 7 Done Sent Done false
104882024-02-12T01:39:01.834Z 28 Acked 1027 Write 6 Done Sent Done false
104892024-02-12T01:39:01.834Z 29 Acked 1028 Write 7 Sent Sent Sent false
104902024-02-12T01:39:01.834Z 30 Acked 1029 Write 1 Sent Sent Sent false
104912024-02-12T01:39:01.835Z STATES DS:0 DS:1 DS:2 TOTAL
104922024-02-12T01:39:01.835Z New 0 0 0 0
104932024-02-12T01:39:01.835Z Sent 2 28 2 32
104942024-02-12T01:39:01.835Z Done 28 2 28 58
104952024-02-12T01:39:01.835Z Skipped 0 0 0 0
104962024-02-12T01:39:01.835Z Error 0 0 0 0
104972024-02-12T01:39:01.835ZLast Flush: 1017 0 1017
104982024-02-12T01:39:01.835ZDownstairs last five completed:
104992024-02-12T01:39:01.835ZUpstairs last five completed: 30 29 28 27 26
105002024-02-12T01:39:01.839Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:00.723407218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1511}
105012024-02-12T01:39:01.839ZWrote out file "/tmp/test_repair_verify.data"
105022024-02-12T01:39:01.839Z
105032024-02-12T01:39:01.842ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
105042024-02-12T01:39:01.863ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105052024-02-12T01:39:01.863Z 0 000-019 90 90 90 153 153 153 F F F
105062024-02-12T01:39:01.863Z 1 020-039 90 90 90 154 154 154 F F F
105072024-02-12T01:39:01.863Z 2 040-059 92 92 92 156 156 156 F F F
105082024-02-12T01:39:01.863Z 3 060-079 93 93 93 158 158 158 F F F
105092024-02-12T01:39:01.863Z 4 080-099 94 93 94 162 158 162 F F F <---
105102024-02-12T01:39:01.863Z 5 100-119 94 80 94 160 137 160 T F T <---
105112024-02-12T01:39:01.863Z 6 120-139 86 86 86 149 149 149 F F F
105122024-02-12T01:39:01.863Z 7 140-159 94 90 94 160 154 160 F F F <---
105132024-02-12T01:39:01.867Z 8 160-179 92 92 92 156 156 156 T F T <---
105142024-02-12T01:39:01.867Z 9 180-199 93 93 93 158 158 158 F F F
105152024-02-12T01:39:01.867Z 10 200-219 89 89 89 152 152 152 F F F
105162024-02-12T01:39:01.867Z 11 220-239 92 92 92 156 156 156 F F F
105172024-02-12T01:39:01.867Z 12 240-259 92 92 92 156 156 156 F F F
105182024-02-12T01:39:01.867Z 13 260-279 90 90 90 153 153 153 F F F
105192024-02-12T01:39:01.867Z 14 280-299 94 92 94 160 157 160 F F F <---
105202024-02-12T01:39:01.867Z 15 300-319 94 92 94 162 157 162 T F T <---
105212024-02-12T01:39:01.867Z 16 320-339 94 93 94 161 158 161 F F F <---
105222024-02-12T01:39:01.867Z 17 340-359 93 93 93 158 158 158 F F F
105232024-02-12T01:39:01.867Z 18 360-379 94 92 94 160 157 160 F F F <---
105242024-02-12T01:39:01.867Z 19 380-399 92 92 92 157 157 157 F F F
105252024-02-12T01:39:01.868Z 20 400-419 93 93 93 158 158 158 F F F
105262024-02-12T01:39:01.868Z 21 420-439 94 90 94 159 154 159 T T T <---
105272024-02-12T01:39:01.868Z 22 440-459 94 88 94 160 151 160 T F T <---
105282024-02-12T01:39:01.868Z 23 460-479 84 84 84 145 145 145 F F F
105292024-02-12T01:39:01.868Z 24 480-499 93 93 93 158 158 158 F F F
105302024-02-12T01:39:01.868Z 25 500-519 94 93 94 162 158 162 T F T <---
105312024-02-12T01:39:01.868Z 26 520-539 93 93 93 158 158 158 T F T <---
105322024-02-12T01:39:01.868Z 27 540-559 88 88 88 151 151 151 F F F
105332024-02-12T01:39:01.868Z 28 560-579 93 93 93 158 158 158 T F T <---
105342024-02-12T01:39:01.868Z 29 580-599 88 88 88 151 151 151 F F F
105352024-02-12T01:39:01.868ZMax gen: 94, Max flush: 162
105362024-02-12T01:39:01.868ZError: Difference in extent metadata found!
105372024-02-12T01:39:01.868ZA Difference in extent metadata is expected here
105382024-02-12T01:39:01.868ZOn loop 47
105392024-02-12T01:39:01.868Z
105402024-02-12T01:39:01.868ZVerifying data now
105412024-02-12T01:39:06.520ZNo change after verify
105422024-02-12T01:39:06.520ZLoop: 47 Downstairs dump after verify (and repair):
105432024-02-12T01:39:06.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
105442024-02-12T01:39:06.535Z 0 000-019 90 90 90 153 153 153 F F F
105452024-02-12T01:39:06.535Z 1 020-039 90 90 90 154 154 154 F F F
105462024-02-12T01:39:06.535Z 2 040-059 92 92 92 156 156 156 F F F
105472024-02-12T01:39:06.535Z 3 060-079 93 93 93 158 158 158 F F F
105482024-02-12T01:39:06.535Z 4 080-099 94 94 94 162 162 162 F F F
105492024-02-12T01:39:06.535Z 5 100-119 95 95 95 163 163 163 F F F
105502024-02-12T01:39:06.535Z 6 120-139 86 86 86 149 149 149 F F F
105512024-02-12T01:39:06.535Z 7 140-159 94 94 94 160 160 160 F F F
105522024-02-12T01:39:06.535Z 8 160-179 95 95 95 163 163 163 F F F
105532024-02-12T01:39:06.535Z 9 180-199 93 93 93 158 158 158 F F F
105542024-02-12T01:39:06.538Z 10 200-219 89 89 89 152 152 152 F F F
105552024-02-12T01:39:06.539Z 11 220-239 92 92 92 156 156 156 F F F
105562024-02-12T01:39:06.539Z 12 240-259 92 92 92 156 156 156 F F F
105572024-02-12T01:39:06.539Z 13 260-279 90 90 90 153 153 153 F F F
105582024-02-12T01:39:06.539Z 14 280-299 94 94 94 160 160 160 F F F
105592024-02-12T01:39:06.539Z 15 300-319 95 95 95 163 163 163 F F F
105602024-02-12T01:39:06.539Z 16 320-339 94 94 94 161 161 161 F F F
105612024-02-12T01:39:06.539Z 17 340-359 93 93 93 158 158 158 F F F
105622024-02-12T01:39:06.539Z 18 360-379 94 94 94 160 160 160 F F F
105632024-02-12T01:39:06.539Z 19 380-399 92 92 92 157 157 157 F F F
105642024-02-12T01:39:06.539Z 20 400-419 93 93 93 158 158 158 F F F
105652024-02-12T01:39:06.539Z 21 420-439 95 95 95 163 163 163 F F F
105662024-02-12T01:39:06.539Z 22 440-459 95 95 95 163 163 163 F F F
105672024-02-12T01:39:06.539Z 23 460-479 84 84 84 145 145 145 F F F
105682024-02-12T01:39:06.539Z 24 480-499 93 93 93 158 158 158 F F F
105692024-02-12T01:39:06.539Z 25 500-519 95 95 95 163 163 163 F F F
105702024-02-12T01:39:06.539Z 26 520-539 95 95 95 163 163 163 F F F
105712024-02-12T01:39:06.539Z 27 540-559 88 88 88 151 151 151 F F F
105722024-02-12T01:39:06.539Z 28 560-579 95 95 95 163 163 163 F F F
105732024-02-12T01:39:06.539Z 29 580-599 88 88 88 151 151 151 F F F
105742024-02-12T01:39:06.539ZMax gen: 95, Max flush: 163
105752024-02-12T01:39:06.539Z
105762024-02-12T01:39:06.539ZBegin loop 48
105772024-02-12T01:39:06.539ZDownstairs to restart: 2
105782024-02-12T01:39:06.557Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.44676278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
105792024-02-12T01:39:06.557Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447021887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
105802024-02-12T01:39:06.557Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447054256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
105812024-02-12T01:39:06.563Z{"msg":"Crucible 889fd2e2-9b55-4cda-a0a4-ecd5a7b066fb has session id: 6288002b-ed6a-4203-91a4-78a1e8cefb8e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447088234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105822024-02-12T01:39:06.563ZCrucible runtime is spawned
105832024-02-12T01:39:06.563Z{"msg":"Upstairs opts: Upstairs UUID: 889fd2e2-9b55-4cda-a0a4-ecd5a7b066fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447134072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105842024-02-12T01:39:06.563Z{"msg":"Crucible stats registered with UUID: 889fd2e2-9b55-4cda-a0a4-ecd5a7b066fb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.4471677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105852024-02-12T01:39:06.563Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.44718252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
105862024-02-12T01:39:06.563Z{"msg":"889fd2e2-9b55-4cda-a0a4-ecd5a7b066fb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447255466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105872024-02-12T01:39:06.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447315904Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105882024-02-12T01:39:06.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447353522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105892024-02-12T01:39:06.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447380281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105902024-02-12T01:39:06.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447405509Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105912024-02-12T01:39:06.563Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447443707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105922024-02-12T01:39:06.563Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447478246Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105932024-02-12T01:39:06.563Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447502595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105942024-02-12T01:39:06.563Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447542333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105952024-02-12T01:39:06.563Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447567861Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105962024-02-12T01:39:06.563Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.44760672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105972024-02-12T01:39:06.563Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:05.447647838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105982024-02-12T01:39:06.563Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447711455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
105992024-02-12T01:39:06.564Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:05.447759002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106002024-02-12T01:39:06.564Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:05.4478158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106012024-02-12T01:39:06.564Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447840659Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106022024-02-12T01:39:06.564Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447887906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106032024-02-12T01:39:06.564Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.447980862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106042024-02-12T01:39:06.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.44843834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106052024-02-12T01:39:06.564Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.448479978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106062024-02-12T01:39:06.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.448500477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106072024-02-12T01:39:06.564Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.448535255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106082024-02-12T01:39:06.564Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449049351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106092024-02-12T01:39:06.564Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449188034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106102024-02-12T01:39:06.564Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449514429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106112024-02-12T01:39:06.564Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449556786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106122024-02-12T01:39:06.564Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449573116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106132024-02-12T01:39:06.564Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449681451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106142024-02-12T01:39:06.564Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449709519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106152024-02-12T01:39:06.564Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:05.449724139Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106162024-02-12T01:39:07.559Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.448373004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106172024-02-12T01:39:07.563Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.448626152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106182024-02-12T01:39:07.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.449564527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106192024-02-12T01:39:07.564Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.449595276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106202024-02-12T01:39:07.564Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450056354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106212024-02-12T01:39:07.564Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.45055122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106222024-02-12T01:39:07.564Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450578979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106232024-02-12T01:39:07.564Z{"msg":"[0]R flush_numbers[0..12]: [153, 154, 156, 158, 162, 163, 149, 160, 163, 158, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450594038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106242024-02-12T01:39:07.564Z{"msg":"[0]R generation[0..12]: [90, 90, 92, 93, 94, 95, 86, 94, 95, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450614417Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106252024-02-12T01:39:07.564Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450650405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106262024-02-12T01:39:07.564Z{"msg":"[1]R flush_numbers[0..12]: [153, 154, 156, 158, 162, 163, 149, 160, 163, 158, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450666475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106272024-02-12T01:39:07.564Z{"msg":"[1]R generation[0..12]: [90, 90, 92, 93, 94, 95, 86, 94, 95, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450683834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106282024-02-12T01:39:07.564Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450714662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106292024-02-12T01:39:07.564Z{"msg":"[2]R flush_numbers[0..12]: [153, 154, 156, 158, 162, 163, 149, 160, 163, 158, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450730612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106302024-02-12T01:39:07.564Z{"msg":"[2]R generation[0..12]: [90, 90, 92, 93, 94, 95, 86, 94, 95, 93, 89, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.45076313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"Wait for a query_work_queue command to finish before sending IO
106312024-02-12T01:39:07.564Zdownstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106322024-02-12T01:39:07.564Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450784669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106332024-02-12T01:39:07.564Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450814348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106342024-02-12T01:39:07.564Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450830057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106352024-02-12T01:39:07.564Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450862195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106362024-02-12T01:39:07.564Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450877665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106372024-02-12T01:39:07.564Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450908773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106382024-02-12T01:39:07.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450923392Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106392024-02-12T01:39:07.564Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.450954521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106402024-02-12T01:39:07.565Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.45097481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106412024-02-12T01:39:07.565Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.451002639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106422024-02-12T01:39:07.565Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.451016478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106432024-02-12T01:39:07.565Z{"msg":"889fd2e2-9b55-4cda-a0a4-ecd5a7b066fb is now active with session: 6288002b-ed6a-4203-91a4-78a1e8cefb8e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.451029637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106442024-02-12T01:39:07.565Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.451044487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519,"session_id":"6288002b-ed6a-4203-91a4-78a1e8cefb8e"}
106452024-02-12T01:39:07.565Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:06.451072425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
106462024-02-12T01:39:07.565ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106472024-02-12T01:39:07.565ZLoading write count information from file "/tmp/test_repair_verify.data"
106482024-02-12T01:39:07.565ZRun Repair workload
106492024-02-12T01:39:07.565Z01/30 Flush
106502024-02-12T01:39:07.568Z02/30 Write block 62 len 40960 data: 9 8 8 9 8 9 8 10 9 11
106512024-02-12T01:39:07.568Z03/30 Read block 6 len 16384
106522024-02-12T01:39:07.568Z04/30 Write block 473 len 8192 data: 9 9
106532024-02-12T01:39:07.568Z05/30 Read block 51 len 24576
106542024-02-12T01:39:07.568Z06/30 Write block 169 len 24576 data: 10 10 9 7 8 7
106552024-02-12T01:39:07.568Z07/30 Read block 489 len 28672
106562024-02-12T01:39:07.571Z08/30 Read block 443 len 32768
106572024-02-12T01:39:07.571Z09/30 Read block 343 len 12288
106582024-02-12T01:39:07.571Z10/30 Read block 128 len 12288
106592024-02-12T01:39:07.574Z11/30 Read block 151 len 32768
106602024-02-12T01:39:07.574Z12/30 Read block 380 len 28672
106612024-02-12T01:39:07.574Z13/30 Read block 28 len 36864
106622024-02-12T01:39:07.580Z14/30 Read block 519 len 32768
106632024-02-12T01:39:07.580Z15/30 Flush
106642024-02-12T01:39:07.583Z16/30 Write block 402 len 36864 data: 6 5 7 8 9 7 8 9 10
106652024-02-12T01:39:07.583Z17/30 Write block 594 len 8192 data: 4 6
106662024-02-12T01:39:07.583Z18/30 Read block 78 len 20480
106672024-02-12T01:39:07.583Z19/30 Read block 342 len 4096
106682024-02-12T01:39:07.583Z20/30 Read block 319 len 28672
106692024-02-12T01:39:07.583Z21/30 Write block 29 len 40960 data: 11 12 13 13 15 14 10 11 9 12
106702024-02-12T01:39:07.583Z22/30 Write block 223 len 12288 data: 8 7 6
106712024-02-12T01:39:07.586Z23/30 Write block 442 len 20480 data: 7 5 6 7 7
106722024-02-12T01:39:07.586Z24/30 Read block 340 len 8192
106732024-02-12T01:39:07.586Z25/30 Flush
106742024-02-12T01:39:07.589Z26/30 Write block 188 len 20480 data: 10 10 8 7 8
106752024-02-12T01:39:07.589Z27/30 Write block 182 len 8192 data: 6 6
106762024-02-12T01:39:07.589Z28/30 Write block 202 len 28672 data: 7 7 7 5 8 10 10
106772024-02-12T01:39:07.593Z29/30 Read block 415 len 36864
106782024-02-12T01:39:07.593Z30/30 Read block 106 len 40960
106792024-02-12T01:39:07.593Z----------------------------------------------------------------
106802024-02-12T01:39:07.593Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:29
106812024-02-12T01:39:07.593ZStates: Active Active Active
106822024-02-12T01:39:07.593ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106832024-02-12T01:39:07.593Z 2 Acked 1001 Write 10 Done Done Done false
106842024-02-12T01:39:07.593Z 3 Acked 1002 Read 4 Done Done Done false
106852024-02-12T01:39:07.593Z 4 Acked 1003 Write 2 Done Done Done false
106862024-02-12T01:39:07.593Z 5 Acked 1004 Read 6 Done Done Done false
106872024-02-12T01:39:07.593Z 6 Acked 1005 Write 6 Done Done Done false
106882024-02-12T01:39:07.593Z 7 Acked 1006 Read 7 Done Done Sent false
106892024-02-12T01:39:07.593Z 8 Acked 1007 Read 8 Done Done Sent false
106902024-02-12T01:39:07.593Z 9 Acked 1008 Read 3 Done Done Sent false
106912024-02-12T01:39:07.593Z 10 Acked 1009 Read 3 Done Done Sent false
106922024-02-12T01:39:07.594Z 11 Acked 1010 Read 8 Done Done Sent false
106932024-02-12T01:39:07.594Z 12 Acked 1011 Read 7 Done Done Sent false
106942024-02-12T01:39:07.594Z 13 Acked 1012 Read 9 Done Done Sent false
106952024-02-12T01:39:07.594Z 14 Acked 1013 Read 8 Done Done Sent false
106962024-02-12T01:39:07.594Z 15 Acked 1014 Flush 0 Done Done Sent false
106972024-02-12T01:39:07.594Z 16 Acked 1015 Write 9 Done Done Sent false
106982024-02-12T01:39:07.594Z 17 Acked 1016 Write 2 Done Done Sent false
106992024-02-12T01:39:07.594Z 18 Acked 1017 Read 5 Done Done Sent false
107002024-02-12T01:39:07.594Z 19 Acked 1018 Read 1 Done Done Sent false
107012024-02-12T01:39:07.594Z 20 Acked 1019 Read 7 Done Done Sent false
107022024-02-12T01:39:07.594Z 21 Acked 1020 Write 10 Done Done Sent false
107032024-02-12T01:39:07.594Z 22 Acked 1021 Write 3 Done Done Sent false
107042024-02-12T01:39:07.594Z 23 Acked 1022 Write 5 Done Done Sent false
107052024-02-12T01:39:07.594Z 24 Acked 1023 Read 2 Done Done Sent false
107062024-02-12T01:39:07.594Z 25 Acked 1024 Flush 0 Done Done Sent false
107072024-02-12T01:39:07.594Z 26 Acked 1025 Write 5 Done Done Sent false
107082024-02-12T01:39:07.594Z 27 Acked 1026 Write 2 Done Done Sent false
107092024-02-12T01:39:07.594Z 28 Acked 1027 Write 7 Done Done Sent false
107102024-02-12T01:39:07.594Z 29 Acked 1028 Read 9 Done Done Sent false
107112024-02-12T01:39:07.594Z 30 Acked 1029 Read 10 Done Sent Sent false
107122024-02-12T01:39:07.594Z STATES DS:0 DS:1 DS:2 TOTAL
107132024-02-12T01:39:07.594Z New 0 0 0 0
107142024-02-12T01:39:07.594Z Sent 0 1 24 25
107152024-02-12T01:39:07.595Z Done 29 28 5 62
107162024-02-12T01:39:07.595Z Skipped 0 0 0 0
107172024-02-12T01:39:07.595Z Error 0 0 0 0
107182024-02-12T01:39:07.595ZLast Flush: 1024 1024 1000
107192024-02-12T01:39:07.595ZDownstairs last five completed: 1000
107202024-02-12T01:39:07.595ZUpstairs last five completed: 30 29 28 27 26
107212024-02-12T01:39:07.595Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:06.482560023Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1519}
107222024-02-12T01:39:07.600ZWrote out file "/tmp/test_repair_verify.data"
107232024-02-12T01:39:07.600Z
107242024-02-12T01:39:07.603ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
107252024-02-12T01:39:07.618ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107262024-02-12T01:39:07.618Z 0 000-019 90 90 90 153 153 153 F F F
107272024-02-12T01:39:07.618Z 1 020-039 96 96 90 166 166 154 F F F <---
107282024-02-12T01:39:07.618Z 2 040-059 92 92 92 156 156 156 F F F
107292024-02-12T01:39:07.618Z 3 060-079 96 96 93 165 165 158 F F T <---
107302024-02-12T01:39:07.618Z 4 080-099 94 94 94 162 162 162 F F F
107312024-02-12T01:39:07.618Z 5 100-119 95 95 95 163 163 163 F F F
107322024-02-12T01:39:07.618Z 6 120-139 86 86 86 149 149 149 F F F
107332024-02-12T01:39:07.618Z 7 140-159 94 94 94 160 160 160 F F F
107342024-02-12T01:39:07.618Z 8 160-179 96 96 95 165 165 163 F F T <---
107352024-02-12T01:39:07.618Z 9 180-199 93 93 93 158 158 158 T T F <---
107362024-02-12T01:39:07.618Z 10 200-219 89 89 89 152 152 152 T T F <---
107372024-02-12T01:39:07.621Z 11 220-239 96 96 92 166 166 156 F F F <---
107382024-02-12T01:39:07.621Z 12 240-259 92 92 92 156 156 156 F F F
107392024-02-12T01:39:07.621Z 13 260-279 90 90 90 153 153 153 F F F
107402024-02-12T01:39:07.621Z 14 280-299 94 94 94 160 160 160 F F F
107412024-02-12T01:39:07.621Z 15 300-319 95 95 95 163 163 163 F F F
107422024-02-12T01:39:07.621Z 16 320-339 94 94 94 161 161 161 F F F
107432024-02-12T01:39:07.621Z 17 340-359 93 93 93 158 158 158 F F F
107442024-02-12T01:39:07.621Z 18 360-379 94 94 94 160 160 160 F F F
107452024-02-12T01:39:07.621Z 19 380-399 92 92 92 157 157 157 F F F
107462024-02-12T01:39:07.621Z 20 400-419 96 96 93 166 166 158 F F F <---
107472024-02-12T01:39:07.622Z 21 420-439 95 95 95 163 163 163 F F F
107482024-02-12T01:39:07.622Z 22 440-459 96 96 95 166 166 163 F F F <---
107492024-02-12T01:39:07.622Z 23 460-479 96 96 84 165 165 145 F F T <---
107502024-02-12T01:39:07.622Z 24 480-499 93 93 93 158 158 158 F F F
107512024-02-12T01:39:07.622Z 25 500-519 95 95 95 163 163 163 F F F
107522024-02-12T01:39:07.622Z 26 520-539 95 95 95 163 163 163 F F F
107532024-02-12T01:39:07.622Z 27 540-559 88 88 88 151 151 151 F F F
107542024-02-12T01:39:07.622Z 28 560-579 95 95 95 163 163 163 F F F
107552024-02-12T01:39:07.622Z 29 580-599 96 96 88 166 166 151 F F F <---
107562024-02-12T01:39:07.622ZMax gen: 96, Max flush: 166
107572024-02-12T01:39:07.622ZError: Difference in extent metadata found!
107582024-02-12T01:39:07.622ZA Difference in extent metadata is expected here
107592024-02-12T01:39:07.622ZOn loop 48
107602024-02-12T01:39:07.622Z
107612024-02-12T01:39:07.625ZVerifying data now
107622024-02-12T01:39:13.144ZNo change after verify
107632024-02-12T01:39:13.144ZLoop: 48 Downstairs dump after verify (and repair):
107642024-02-12T01:39:13.159ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107652024-02-12T01:39:13.159Z 0 000-019 90 90 90 153 153 153 F F F
107662024-02-12T01:39:13.159Z 1 020-039 96 96 96 166 166 166 F F F
107672024-02-12T01:39:13.159Z 2 040-059 92 92 92 156 156 156 F F F
107682024-02-12T01:39:13.159Z 3 060-079 96 96 96 165 165 165 F F F
107692024-02-12T01:39:13.159Z 4 080-099 94 94 94 162 162 162 F F F
107702024-02-12T01:39:13.159Z 5 100-119 95 95 95 163 163 163 F F F
107712024-02-12T01:39:13.159Z 6 120-139 86 86 86 149 149 149 F F F
107722024-02-12T01:39:13.159Z 7 140-159 94 94 94 160 160 160 F F F
107732024-02-12T01:39:13.159Z 8 160-179 96 96 96 165 165 165 F F F
107742024-02-12T01:39:13.159Z 9 180-199 97 97 97 167 167 167 F F F
107752024-02-12T01:39:13.159Z 10 200-219 97 97 97 167 167 167 F F F
107762024-02-12T01:39:13.163Z 11 220-239 96 96 96 166 166 166 F F F
107772024-02-12T01:39:13.163Z 12 240-259 92 92 92 156 156 156 F F F
107782024-02-12T01:39:13.163Z 13 260-279 90 90 90 153 153 153 F F F
107792024-02-12T01:39:13.163Z 14 280-299 94 94 94 160 160 160 F F F
107802024-02-12T01:39:13.163Z 15 300-319 95 95 95 163 163 163 F F F
107812024-02-12T01:39:13.163Z 16 320-339 94 94 94 161 161 161 F F F
107822024-02-12T01:39:13.163Z 17 340-359 93 93 93 158 158 158 F F F
107832024-02-12T01:39:13.163Z 18 360-379 94 94 94 160 160 160 F F F
107842024-02-12T01:39:13.163Z 19 380-399 92 92 92 157 157 157 F F F
107852024-02-12T01:39:13.163Z 20 400-419 96 96 96 166 166 166 F F F
107862024-02-12T01:39:13.163Z 21 420-439 95 95 95 163 163 163 F F F
107872024-02-12T01:39:13.163Z 22 440-459 96 96 96 166 166 166 F F F
107882024-02-12T01:39:13.163Z 23 460-479 96 96 96 165 165 165 F F F
107892024-02-12T01:39:13.163Z 24 480-499 93 93 93 158 158 158 F F F
107902024-02-12T01:39:13.163Z 25 500-519 95 95 95 163 163 163 F F F
107912024-02-12T01:39:13.163Z 26 520-539 95 95 95 163 163 163 F F F
107922024-02-12T01:39:13.163Z 27 540-559 88 88 88 151 151 151 F F F
107932024-02-12T01:39:13.163Z 28 560-579 95 95 95 163 163 163 F F F
107942024-02-12T01:39:13.163Z 29 580-599 96 96 96 166 166 166 F F F
107952024-02-12T01:39:13.163ZMax gen: 97, Max flush: 167
107962024-02-12T01:39:13.163Z
107972024-02-12T01:39:13.163ZBegin loop 49
107982024-02-12T01:39:13.163ZDownstairs to restart: 2
107992024-02-12T01:39:13.180Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.069709496Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
108002024-02-12T01:39:13.180Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.069981063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
108012024-02-12T01:39:13.180Z{Crucible runtime is spawned
108022024-02-12T01:39:13.188Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070053879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
108032024-02-12T01:39:13.188Z{"msg":"Crucible 77f2a20f-fc73-435c-9676-fcb5e0079e82 has session id: f6d51b1d-cccb-4820-abe9-cb00d3aae668","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070096957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108042024-02-12T01:39:13.188Z{"msg":"Upstairs opts: Upstairs UUID: 77f2a20f-fc73-435c-9676-fcb5e0079e82, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070156055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108052024-02-12T01:39:13.188Z{"msg":"Crucible stats registered with UUID: 77f2a20f-fc73-435c-9676-fcb5e0079e82","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070198413Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108062024-02-12T01:39:13.188Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070225701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
108072024-02-12T01:39:13.188Z{"msg":"77f2a20f-fc73-435c-9676-fcb5e0079e82 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.07024239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108082024-02-12T01:39:13.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070272739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108092024-02-12T01:39:13.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070291918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108102024-02-12T01:39:13.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070328076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108112024-02-12T01:39:13.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070372224Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108122024-02-12T01:39:13.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070393813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108132024-02-12T01:39:13.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070435191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108142024-02-12T01:39:13.188Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070473639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108152024-02-12T01:39:13.188Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070511187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108162024-02-12T01:39:13.188Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070560025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108172024-02-12T01:39:13.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070606133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108182024-02-12T01:39:13.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070646271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108192024-02-12T01:39:13.189Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:12.070781855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108202024-02-12T01:39:13.189Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:12.070821983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108212024-02-12T01:39:13.189Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:12.070920028Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108222024-02-12T01:39:13.189Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.070995495Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108232024-02-12T01:39:13.189Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071020273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108242024-02-12T01:39:13.189Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071044132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108252024-02-12T01:39:13.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071363957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108262024-02-12T01:39:13.189Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071421264Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108272024-02-12T01:39:13.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071452233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108282024-02-12T01:39:13.189Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071483191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108292024-02-12T01:39:13.189Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.071867163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108302024-02-12T01:39:13.189Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072025135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108312024-02-12T01:39:13.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072374879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108322024-02-12T01:39:13.189Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072427016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108332024-02-12T01:39:13.189Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072485013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108342024-02-12T01:39:13.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.07255631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108352024-02-12T01:39:13.189Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072599638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108362024-02-12T01:39:13.189Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:12.072617527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108372024-02-12T01:39:14.183Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.072516753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108382024-02-12T01:39:14.184Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.072604349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108392024-02-12T01:39:14.188Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.073461548Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108402024-02-12T01:39:14.188Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.073508655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108412024-02-12T01:39:14.188Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.073988203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108422024-02-12T01:39:14.189Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074570475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108432024-02-12T01:39:14.189Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074619112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108442024-02-12T01:39:14.189Z{"msg":"[0]R flush_numbers[0..12]: [153, 166, 156, 165, 162, 163, 149, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074659391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108452024-02-12T01:39:14.189Z{"msg":"[0]R generation[0..12]: [90, 96, 92, 96, 94, 95, 86, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.07467693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108462024-02-12T01:39:14.189Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074693789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108472024-02-12T01:39:14.189Z{"msg":"[1]R flush_numbers[0..12]: [153, 166, 156, 165, 162, 163, 149, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074708768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108482024-02-12T01:39:14.189Z{"msg":"[1]R generation[0..12]: [90, 96, 92, 96, 94, 95, 86, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074730277Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108492024-02-12T01:39:14.189Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074745546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108502024-02-12T01:39:14.189Z{"msg":"Wait for a query_work_queue command to finish before sending IO
108512024-02-12T01:39:14.189Z[2]R flush_numbers[0..12]: [153, 166, 156, 165, 162, 163, 149, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074773745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108522024-02-12T01:39:14.189Z{"msg":"[2]R generation[0..12]: [90, 96, 92, 96, 94, 95, 86, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074791034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108532024-02-12T01:39:14.189Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074813773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108542024-02-12T01:39:14.189Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074827563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108552024-02-12T01:39:14.189Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074842292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108562024-02-12T01:39:14.189Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074855311Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108572024-02-12T01:39:14.189Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.07487756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108582024-02-12T01:39:14.189Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074891409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108592024-02-12T01:39:14.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074915508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108602024-02-12T01:39:14.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074929067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108612024-02-12T01:39:14.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074942897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108622024-02-12T01:39:14.189Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074956116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108632024-02-12T01:39:14.189Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074969386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108642024-02-12T01:39:14.189Z{"msg":"77f2a20f-fc73-435c-9676-fcb5e0079e82 is now active with session: f6d51b1d-cccb-4820-abe9-cb00d3aae668","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.074989205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108652024-02-12T01:39:14.190Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.075001744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527,"session_id":"f6d51b1d-cccb-4820-abe9-cb00d3aae668"}
108662024-02-12T01:39:14.190Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:13.075014383Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
108672024-02-12T01:39:14.190ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108682024-02-12T01:39:14.190ZLoading write count information from file "/tmp/test_repair_verify.data"
108692024-02-12T01:39:14.190ZRun Repair workload
108702024-02-12T01:39:14.190Z01/30 Write block 530 len 8192 data: 10 10
108712024-02-12T01:39:14.193Z02/30 Flush
108722024-02-12T01:39:14.193Z03/30 Flush
108732024-02-12T01:39:14.193Z04/30 Write block 517 len 28672 data: 8 6 7 8 8 8 9
108742024-02-12T01:39:14.193Z05/30 Read block 179 len 36864
108752024-02-12T01:39:14.196Z06/30 Read block 71 len 12288
108762024-02-12T01:39:14.196Z07/30 Read block 299 len 12288
108772024-02-12T01:39:14.196Z08/30 Write block 127 len 4096 data: 11
108782024-02-12T01:39:14.196Z09/30 Read block 394 len 32768
108792024-02-12T01:39:14.199Z10/30 Write block 278 len 24576 data: 10 11 10 8 10 8
108802024-02-12T01:39:14.199Z11/30 Read block 151 len 40960
108812024-02-12T01:39:14.199Z12/30 Write block 80 len 24576 data: 13 13 14 11 12 12
108822024-02-12T01:39:14.201Z13/30 Read block 457 len 20480
108832024-02-12T01:39:14.201Z14/30 Write block 536 len 28672 data: 12 12 11 10 9 6 6
108842024-02-12T01:39:14.202Z15/30 Read block 119 len 28672
108852024-02-12T01:39:14.207Z16/30 Read block 72 len 40960
108862024-02-12T01:39:14.207Z17/30 Flush
108872024-02-12T01:39:14.210Z18/30 Flush
108882024-02-12T01:39:14.210Z19/30 Write block 240 len 36864 data: 9 7 8 9 7 6 7 7 6
108892024-02-12T01:39:14.210Z20/30 Read block 440 len 36864
108902024-02-12T01:39:14.210Z21/30 Read block 142 len 32768
108912024-02-12T01:39:14.213Z22/30 Flush
108922024-02-12T01:39:14.213Z23/30 Write block 393 len 8192 data: 12 9
108932024-02-12T01:39:14.213Z24/30 Read block 543 len 8192
108942024-02-12T01:39:14.216Z25/30 Write block 367 len 40960 data: 11 9 10 10 9 11 12 11 13 11
108952024-02-12T01:39:14.216Z26/30 Read block 323 len 8192
108962024-02-12T01:39:14.216Z27/30 Read block 427 len 12288
108972024-02-12T01:39:14.218Z28/30 Read block 361 len 28672
108982024-02-12T01:39:14.218Z29/30 Read block 93 len 32768
108992024-02-12T01:39:14.222Z30/30 Write block 6 len 36864 data: 7 6 7 8 7 8 9 8 9
109002024-02-12T01:39:14.222Z----------------------------------------------------------------
109012024-02-12T01:39:14.222Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
109022024-02-12T01:39:14.222ZStates: Active Active Active
109032024-02-12T01:39:14.222ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109042024-02-12T01:39:14.222Z 1 Acked 1000 Write 2 Done Done Sent false
109052024-02-12T01:39:14.222Z 2 Acked 1001 Flush 0 Done Done Sent false
109062024-02-12T01:39:14.222Z 3 Acked 1002 Flush 0 Done Done Sent false
109072024-02-12T01:39:14.222Z 4 Acked 1003 Write 7 Done Done Sent false
109082024-02-12T01:39:14.222Z 5 Acked 1004 Read 9 Done Done Sent false
109092024-02-12T01:39:14.222Z 6 Acked 1005 Read 3 Done Done Sent false
109102024-02-12T01:39:14.222Z 7 Acked 1006 Read 3 Done Done Sent false
109112024-02-12T01:39:14.222Z 8 Acked 1007 Write 1 Done Done Sent false
109122024-02-12T01:39:14.222Z 9 Acked 1008 Read 8 Done Done Sent false
109132024-02-12T01:39:14.222Z 10 Acked 1009 Write 6 Done Done Sent false
109142024-02-12T01:39:14.223Z 11 Acked 1010 Read 10 Done Done Sent false
109152024-02-12T01:39:14.223Z 12 Acked 1011 Write 6 Done Done Sent false
109162024-02-12T01:39:14.223Z 13 Acked 1012 Read 5 Done Done Sent false
109172024-02-12T01:39:14.223Z 14 Acked 1013 Write 7 Done Done Sent false
109182024-02-12T01:39:14.223Z 15 Acked 1014 Read 7 Done Done Sent false
109192024-02-12T01:39:14.223Z 16 Acked 1015 Read 10 Done Done Sent false
109202024-02-12T01:39:14.223Z 17 Acked 1016 Flush 0 Done Done Sent false
109212024-02-12T01:39:14.223Z 18 Acked 1017 Flush 0 Done Done Sent false
109222024-02-12T01:39:14.223Z 19 Acked 1018 Write 9 Done Done Sent false
109232024-02-12T01:39:14.223Z 20 Acked 1019 Read 9 Done Done Sent false
109242024-02-12T01:39:14.223Z 21 Acked 1020 Read 8 Done Done Sent false
109252024-02-12T01:39:14.223Z 22 Acked 1021 Flush 0 Done Done Sent false
109262024-02-12T01:39:14.223Z 23 Acked 1022 Write 2 Done Done Sent false
109272024-02-12T01:39:14.223Z 24 Acked 1023 Read 2 Done Done Sent false
109282024-02-12T01:39:14.223Z 25 Acked 1024 Write 10 Done Done Sent false
109292024-02-12T01:39:14.223Z 26 Acked 1025 Read 2 Done Done Sent false
109302024-02-12T01:39:14.223Z 27 Acked 1026 Read 3 Done Done Sent false
109312024-02-12T01:39:14.223Z 28 Acked 1027 Read 7 Done Done Sent false
109322024-02-12T01:39:14.223Z 29 Acked 1028 Read 8 Done Done Sent false
109332024-02-12T01:39:14.223Z 30 Acked 1029 Write 9 Sent Sent Sent false
109342024-02-12T01:39:14.223Z STATES DS:0 DS:1 DS:2 TOTAL
109352024-02-12T01:39:14.223Z New 0 0 0 0
109362024-02-12T01:39:14.223Z Sent 1 1 30 32
109372024-02-12T01:39:14.223Z Done 29 29 0 58
109382024-02-12T01:39:14.223Z Skipped 0 0 0 0
109392024-02-12T01:39:14.223Z Error 0 0 0 0
109402024-02-12T01:39:14.224ZLast Flush: 1021 1021 0
109412024-02-12T01:39:14.224ZDownstairs last five completed:
109422024-02-12T01:39:14.224ZUpstairs last five completed: 30 29 28 27 26
109432024-02-12T01:39:14.224Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:13.109112777Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1527}
109442024-02-12T01:39:14.224ZWrote out file "/tmp/test_repair_verify.data"
109452024-02-12T01:39:14.227Z
109462024-02-12T01:39:14.229ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109472024-02-12T01:39:14.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109482024-02-12T01:39:14.244Z 0 000-019 90 90 90 153 153 153 T T F <---
109492024-02-12T01:39:14.244Z 1 020-039 96 96 96 166 166 166 F F F
109502024-02-12T01:39:14.244Z 2 040-059 92 92 92 156 156 156 F F F
109512024-02-12T01:39:14.244Z 3 060-079 96 96 96 165 165 165 F F F
109522024-02-12T01:39:14.244Z 4 080-099 98 98 94 170 170 162 F F F <---
109532024-02-12T01:39:14.244Z 5 100-119 95 95 95 163 163 163 F F F
109542024-02-12T01:39:14.244Z 6 120-139 98 98 86 170 170 149 F F F <---
109552024-02-12T01:39:14.244Z 7 140-159 94 94 94 160 160 160 F F F
109562024-02-12T01:39:14.247Z 8 160-179 96 96 96 165 165 165 F F F
109572024-02-12T01:39:14.247Z 9 180-199 97 97 97 167 167 167 F F F
109582024-02-12T01:39:14.247Z 10 200-219 97 97 97 167 167 167 F F F
109592024-02-12T01:39:14.247Z 11 220-239 96 96 96 166 166 166 F F F
109602024-02-12T01:39:14.247Z 12 240-259 98 98 92 172 172 156 F F F <---
109612024-02-12T01:39:14.248Z 13 260-279 98 98 90 170 170 153 F F F <---
109622024-02-12T01:39:14.248Z 14 280-299 98 98 94 170 170 160 F F F <---
109632024-02-12T01:39:14.248Z 15 300-319 95 95 95 163 163 163 F F F
109642024-02-12T01:39:14.248Z 16 320-339 94 94 94 161 161 161 F F F
109652024-02-12T01:39:14.248Z 17 340-359 93 93 93 158 158 158 F F F
109662024-02-12T01:39:14.248Z 18 360-379 94 94 94 160 160 160 T T F <---
109672024-02-12T01:39:14.248Z 19 380-399 92 92 92 157 157 157 T T F <---
109682024-02-12T01:39:14.248Z 20 400-419 96 96 96 166 166 166 F F F
109692024-02-12T01:39:14.248Z 21 420-439 95 95 95 163 163 163 F F F
109702024-02-12T01:39:14.248Z 22 440-459 96 96 96 166 166 166 F F F
109712024-02-12T01:39:14.248Z 23 460-479 96 96 96 165 165 165 F F F
109722024-02-12T01:39:14.248Z 24 480-499 93 93 93 158 158 158 F F F
109732024-02-12T01:39:14.248Z 25 500-519 98 98 95 170 170 163 F F F <---
109742024-02-12T01:39:14.248Z 26 520-539 98 98 95 170 170 163 F F F <---
109752024-02-12T01:39:14.248Z 27 540-559 98 98 88 170 170 151 F F F <---
109762024-02-12T01:39:14.248Z 28 560-579 95 95 95 163 163 163 F F F
109772024-02-12T01:39:14.248Z 29 580-599 96 96 96 166 166 166 F F F
109782024-02-12T01:39:14.248ZMax gen: 98, Max flush: 172
109792024-02-12T01:39:14.248ZError: Difference in extent metadata found!
109802024-02-12T01:39:14.248ZA Difference in extent metadata is expected here
109812024-02-12T01:39:14.248ZOn loop 49
109822024-02-12T01:39:14.248Z
109832024-02-12T01:39:14.252ZVerifying data now
109842024-02-12T01:39:18.810ZNo change after verify
109852024-02-12T01:39:18.810ZLoop: 49 Downstairs dump after verify (and repair):
109862024-02-12T01:39:18.828ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109872024-02-12T01:39:18.828Z 0 000-019 99 99 99 173 173 173 F F F
109882024-02-12T01:39:18.828Z 1 020-039 96 96 96 166 166 166 F F F
109892024-02-12T01:39:18.828Z 2 040-059 92 92 92 156 156 156 F F F
109902024-02-12T01:39:18.829Z 3 060-079 96 96 96 165 165 165 F F F
109912024-02-12T01:39:18.829Z 4 080-099 98 98 98 170 170 170 F F F
109922024-02-12T01:39:18.829Z 5 100-119 95 95 95 163 163 163 F F F
109932024-02-12T01:39:18.829Z 6 120-139 98 98 98 170 170 170 F F F
109942024-02-12T01:39:18.829Z 7 140-159 94 94 94 160 160 160 F F F
109952024-02-12T01:39:18.829Z 8 160-179 96 96 96 165 165 165 F F F
109962024-02-12T01:39:18.829Z 9 180-199 97 97 97 167 167 167 F F F
109972024-02-12T01:39:18.829Z 10 200-219 97 97 97 167 167 167 F F F
109982024-02-12T01:39:18.834Z 11 220-239 96 96 96 166 166 166 F F F
109992024-02-12T01:39:18.834Z 12 240-259 98 98 98 172 172 172 F F F
110002024-02-12T01:39:18.834Z 13 260-279 98 98 98 170 170 170 F F F
110012024-02-12T01:39:18.834Z 14 280-299 98 98 98 170 170 170 F F F
110022024-02-12T01:39:18.834Z 15 300-319 95 95 95 163 163 163 F F F
110032024-02-12T01:39:18.834Z 16 320-339 94 94 94 161 161 161 F F F
110042024-02-12T01:39:18.834Z 17 340-359 93 93 93 158 158 158 F F F
110052024-02-12T01:39:18.834Z 18 360-379 99 99 99 173 173 173 F F F
110062024-02-12T01:39:18.834Z 19 380-399 99 99 99 173 173 173 F F F
110072024-02-12T01:39:18.834Z 20 400-419 96 96 96 166 166 166 F F F
110082024-02-12T01:39:18.834Z 21 420-439 95 95 95 163 163 163 F F F
110092024-02-12T01:39:18.834Z 22 440-459 96 96 96 166 166 166 F F F
110102024-02-12T01:39:18.834Z 23 460-479 96 96 96 165 165 165 F F F
110112024-02-12T01:39:18.834Z 24 480-499 93 93 93 158 158 158 F F F
110122024-02-12T01:39:18.834Z 25 500-519 98 98 98 170 170 170 F F F
110132024-02-12T01:39:18.834Z 26 520-539 98 98 98 170 170 170 F F F
110142024-02-12T01:39:18.834Z 27 540-559 98 98 98 170 170 170 F F F
110152024-02-12T01:39:18.834Z 28 560-579 95 95 95 163 163 163 F F F
110162024-02-12T01:39:18.834Z 29 580-599 96 96 96 166 166 166 F F F
110172024-02-12T01:39:18.834ZMax gen: 99, Max flush: 173
110182024-02-12T01:39:18.834Z
110192024-02-12T01:39:18.834ZBegin loop 50
110202024-02-12T01:39:18.835ZDownstairs to restart: 0
110212024-02-12T01:39:18.848Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.736680509Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
110222024-02-12T01:39:18.848Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.736912108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
110232024-02-12T01:39:18.852Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.736946366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
110242024-02-12T01:39:18.852Z{"msg":"Crucible 89efbc48-ca73-4350-8124-b7008e59c612 has session id: d4206efc-fbe3-4c41-8421-f2068643d967","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.736971255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110252024-02-12T01:39:18.852Z{"msg":"Upstairs opts: Upstairs UUID: 89efbc48-ca73-4350-8124-b7008e59c612, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737008473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110262024-02-12T01:39:18.852Z{"msg":"Crucible stats registered with UUID: 89efbc48-ca73-4350-8124-b7008e59c612","v":0,"name":"crucible","level":Crucible runtime is spawned
110272024-02-12T01:39:18.852Z30,"time":"2024-02-12T01:39:17.737044802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110282024-02-12T01:39:18.852Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737128788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
110292024-02-12T01:39:18.852Z{"msg":"89efbc48-ca73-4350-8124-b7008e59c612 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737207214Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110302024-02-12T01:39:18.852Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737242532Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110312024-02-12T01:39:18.852Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737260231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110322024-02-12T01:39:18.852Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737275191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110332024-02-12T01:39:18.852Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.73729093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110342024-02-12T01:39:18.852Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737318899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110352024-02-12T01:39:18.853Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737334468Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110362024-02-12T01:39:18.853Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737363897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110372024-02-12T01:39:18.853Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737394385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110382024-02-12T01:39:18.853Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737428723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110392024-02-12T01:39:18.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737650603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110402024-02-12T01:39:18.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.73771308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110412024-02-12T01:39:18.853Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.737741678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110422024-02-12T01:39:18.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.738368538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110432024-02-12T01:39:18.853Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.738404847Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110442024-02-12T01:39:18.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.738421466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110452024-02-12T01:39:18.853Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.738435335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110462024-02-12T01:39:18.853Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739034937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110472024-02-12T01:39:18.853Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739071725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110482024-02-12T01:39:18.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739582051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110492024-02-12T01:39:18.853Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739623519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110502024-02-12T01:39:18.853Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739665097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110512024-02-12T01:39:18.853Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739700735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110522024-02-12T01:39:18.853Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739722974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110532024-02-12T01:39:18.853Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739752223Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110542024-02-12T01:39:18.853Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.739768682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110552024-02-12T01:39:18.853Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.73979879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110562024-02-12T01:39:18.858Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740286757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110572024-02-12T01:39:18.858Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740789853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110582024-02-12T01:39:18.858Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740838631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110592024-02-12T01:39:18.858Z{"msg":"[0]R flush_numbers[0..12]: [173, 166, 156, 165, 170, 163, 170, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740876179Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110602024-02-12T01:39:18.858Z{"msg":"[0]R generation[0..12]: [99, 96, 92, 96, 98, 95, 98, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740911327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110612024-02-12T01:39:18.858Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.740948166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110622024-02-12T01:39:18.858Z{"msg":"[1]R flush_numbers[0..12]: [173, 166, 156, 165, 170, 163, 170, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
110632024-02-12T01:39:18.858Z,"time":"2024-02-12T01:39:17.740985434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110642024-02-12T01:39:18.858Z{"msg":"[1]R generation[0..12]: [99, 96, 92, 96, 98, 95, 98, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741016962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110652024-02-12T01:39:18.858Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.7410635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110662024-02-12T01:39:18.858Z{"msg":"[2]R flush_numbers[0..12]: [173, 166, 156, 165, 170, 163, 170, 160, 165, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741092039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110672024-02-12T01:39:18.859Z{"msg":"[2]R generation[0..12]: [99, 96, 92, 96, 98, 95, 98, 94, 96, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741128437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110682024-02-12T01:39:18.859Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741167195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110692024-02-12T01:39:18.859Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741200263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110702024-02-12T01:39:18.859Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741225162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110712024-02-12T01:39:18.859Z{"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.74126701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110722024-02-12T01:39:18.859Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741295809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110732024-02-12T01:39:18.859Z":"downstairs","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110742024-02-12T01:39:18.859Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741326577Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110752024-02-12T01:39:18.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741359836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110762024-02-12T01:39:18.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741392174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110772024-02-12T01:39:18.859Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741407693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110782024-02-12T01:39:18.859Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741428423Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110792024-02-12T01:39:18.859Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741441222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110802024-02-12T01:39:18.859Z{"msg":"89efbc48-ca73-4350-8124-b7008e59c612 is now active with session: d4206efc-fbe3-4c41-8421-f2068643d967","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741460921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110812024-02-12T01:39:18.859Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.74147319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535,"session_id":"d4206efc-fbe3-4c41-8421-f2068643d967"}
110822024-02-12T01:39:18.859Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:17.741492289Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
110832024-02-12T01:39:18.859ZLoading write count information from file "/tmp/test_repair_verify.data"
110842024-02-12T01:39:18.859ZRun Repair workload
110852024-02-12T01:39:18.859Z01/30 Flush
110862024-02-12T01:39:18.859Z02/30 Write block 108 len 16384 data: 8 7 7 7
110872024-02-12T01:39:18.859Z03/30 Read block 316 len 8192
110882024-02-12T01:39:18.859Z04/30 Read block 562 len 40960
110892024-02-12T01:39:18.864Z05/30 Read block 86 len 36864
110902024-02-12T01:39:18.864Z06/30 Read block 58 len 20480
110912024-02-12T01:39:18.864Z07/30 Read block 96 len 16384
110922024-02-12T01:39:18.864Z08/30 Write block 137 len 36864 data: 7 9 9 9 12 14 14 14 14
110932024-02-12T01:39:18.864Z09/30 Write block 401 len 40960 data: 5 7 6 8 9 10 8 9 10 11
110942024-02-12T01:39:18.864Z10/30 Write block 260 len 8192 data: 7 5
110952024-02-12T01:39:18.864Z11/30 Flush
110962024-02-12T01:39:18.867Z12/30 Write block 158 len 12288 data: 5 5 8
110972024-02-12T01:39:18.867Z13/30 Flush
110982024-02-12T01:39:18.870Z14/30 Write block 124 len 40960 data: 12 11 13 12 10 9 8 9 9 6
110992024-02-12T01:39:18.870Z15/30 Flush
111002024-02-12T01:39:18.873Z16/30 Write block 474 len 28672 data: 10 8 5 6 6 9 9
111012024-02-12T01:39:18.873Z17/30 Write block 488 len 32768 data: 10 7 9 6 5 5 6 9
111022024-02-12T01:39:18.876Z18/30 Read block 360 len 28672
111032024-02-12T01:39:18.876Z19/30 Read block 93 len 20480
111042024-02-12T01:39:18.876Z20/30 Write block 195 len 4096 data: 4
111052024-02-12T01:39:18.878Z21/30 Write block 318 len 28672 data: 7 8 7 8 9 7 6
111062024-02-12T01:39:18.878Z22/30 Write block 453 len 8192 data: 5 7
111072024-02-12T01:39:18.879Z23/30 Read block 304 len 32768
111082024-02-12T01:39:18.881Z24/30 Write block 180 len 20480 data: 10 11 7 7 8
111092024-02-12T01:39:18.881Z25/30 Write block 406 len 12288 data: 11 9 10
111102024-02-12T01:39:18.881Z26/30 Write block 57 len 8192 data: 8 12
111112024-02-12T01:39:18.881Z27/30 Write block 377 len 40960 data: 10 9 10 8 9 8 7 8 10 10
111122024-02-12T01:39:18.881Z28/30 Read block 241 len 32768
111132024-02-12T01:39:19.136Z29/30 Write block 242 len 16384 data: 9 10 8 7
111142024-02-12T01:39:19.136Z30/30 Read block 337 len 12288
111152024-02-12T01:39:19.136Z----------------------------------------------------------------
111162024-02-12T01:39:19.136Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:29
111172024-02-12T01:39:19.136ZStates: Active Active Active
111182024-02-12T01:39:19.136ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111192024-02-12T01:39:19.136Z 2 Acked 1001 Write 4 Done Done Done false
111202024-02-12T01:39:19.136Z 3 Acked 1002 Read 2 Done Done Done false
111212024-02-12T01:39:19.136Z 4 Acked 1003 Read 10 Sent Done Done false
111222024-02-12T01:39:19.137Z 5 Acked 1004 Read 9 Sent Done Done false
111232024-02-12T01:39:19.137Z 6 Acked 1005 Read 5 Sent Done Done false
111242024-02-12T01:39:19.137Z 7 Acked 1006 Read 4 Sent Done Done false
111252024-02-12T01:39:19.137Z 8 Acked 1007 Write 9 Sent Done Done false
111262024-02-12T01:39:19.137Z 9 Acked 1008 Write 10 Sent Done Done false
111272024-02-12T01:39:19.137Z 10 Acked 1009 Write 2 Sent Done Done false
111282024-02-12T01:39:19.137Z 11 Acked 1010 Flush 0 Sent Done Done false
111292024-02-12T01:39:19.137Z 12 Acked 1011 Write 3 Sent Done Done false
111302024-02-12T01:39:19.137Z 13 Acked 1012 Flush 0 Sent Done Done false
111312024-02-12T01:39:19.137Z 14 Acked 1013 Write 10 Sent Done Done false
111322024-02-12T01:39:19.137Z 15 Acked 1014 Flush 0 Sent Done Done false
111332024-02-12T01:39:19.137Z 16 Acked 1015 Write 7 Sent Done Done false
111342024-02-12T01:39:19.137Z 17 Acked 1016 Write 8 Sent Done Done false
111352024-02-12T01:39:19.137Z 18 Acked 1017 Read 7 Sent Done Done false
111362024-02-12T01:39:19.137Z 19 Acked 1018 Read 5 Sent Done Done false
111372024-02-12T01:39:19.137Z 20 Acked 1019 Write 1 Sent Done Done false
111382024-02-12T01:39:19.137Z 21 Acked 1020 Write 7 Sent Done Done false
111392024-02-12T01:39:19.137Z 22 Acked 1021 Write 2 Sent Done Done false
111402024-02-12T01:39:19.137Z 23 Acked 1022 Read 8 Sent Done Done false
111412024-02-12T01:39:19.137Z 24 Acked 1023 Write 5 Sent Done Done false
111422024-02-12T01:39:19.137Z 25 Acked 1024 Write 3 Sent Done Done false
111432024-02-12T01:39:19.137Z 26 Acked 1025 Write 2 Sent Done Done false
111442024-02-12T01:39:19.137Z 27 Acked 1026 Write 10 Sent Done Done false
111452024-02-12T01:39:19.137Z 28 Acked 1027 Read 8 Sent Done Done false
111462024-02-12T01:39:19.137Z 29 Acked 1028 Write 4 Sent Done Done false
111472024-02-12T01:39:19.138Z 30 Acked 1029 Read 3 Sent Done Sent false
111482024-02-12T01:39:19.138Z STATES DS:0 DS:1 DS:2 TOTAL
111492024-02-12T01:39:19.138Z New 0 0 0 0
111502024-02-12T01:39:19.138Z Sent 27 0 1 28
111512024-02-12T01:39:19.138Z Done 2 29 28 59
111522024-02-12T01:39:19.138Z Skipped 0 0 0 0
111532024-02-12T01:39:19.138Z Error 0 0 0 0
111542024-02-12T01:39:19.138ZLast Flush: 1000 1014 1014
111552024-02-12T01:39:19.138ZDownstairs last five completed: 1000
111562024-02-12T01:39:19.138ZUpstairs last five completed: 30 29 28 27 26
111572024-02-12T01:39:19.138Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:17.773493953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1535}
111582024-02-12T01:39:19.141ZWrote out file "/tmp/test_repair_verify.data"
111592024-02-12T01:39:19.141Z
111602024-02-12T01:39:19.141ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111612024-02-12T01:39:19.142ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111622024-02-12T01:39:19.142Z 0 000-019 99 99 99 173 173 173 F F F
111632024-02-12T01:39:19.142Z 1 020-039 96 96 96 166 166 166 F F F
111642024-02-12T01:39:19.142Z 2 040-059 92 92 92 156 156 156 F T T <---
111652024-02-12T01:39:19.142Z 3 060-079 96 96 96 165 165 165 F F F
111662024-02-12T01:39:19.142Z 4 080-099 98 98 98 170 170 170 F F F
111672024-02-12T01:39:19.142Z 5 100-119 95 100 100 163 175 175 T F F <---
111682024-02-12T01:39:19.142Z 6 120-139 98 100 100 170 177 177 F F F <---
111692024-02-12T01:39:19.142Z 7 140-159 94 100 100 160 176 176 F F F <---
111702024-02-12T01:39:19.142Z 8 160-179 96 100 100 165 176 176 F F F <---
111712024-02-12T01:39:19.142Z 9 180-199 97 97 97 167 167 167 F T T <---
111722024-02-12T01:39:19.142Z 10 200-219 97 97 97 167 167 167 F F F
111732024-02-12T01:39:19.142Z 11 220-239 96 96 96 166 166 166 F F F
111742024-02-12T01:39:19.142Z 12 240-259 98 98 98 172 172 172 F T T <---
111752024-02-12T01:39:19.142Z 13 260-279 98 100 100 170 175 175 F F F <---
111762024-02-12T01:39:19.142Z 14 280-299 98 98 98 170 170 170 F F F
111772024-02-12T01:39:19.142Z 15 300-319 95 95 95 163 163 163 F T T <---
111782024-02-12T01:39:19.142Z 16 320-339 94 94 94 161 161 161 F T T <---
111792024-02-12T01:39:19.142Z 17 340-359 93 93 93 158 158 158 F F F
111802024-02-12T01:39:19.142Z 18 360-379 99 99 99 173 173 173 F T T <---
111812024-02-12T01:39:19.142Z 19 380-399 99 99 99 173 173 173 F T T <---
111822024-02-12T01:39:19.142Z 20 400-419 96 100 100 166 175 175 F T T <---
111832024-02-12T01:39:19.142Z 21 420-439 95 95 95 163 163 163 F F F
111842024-02-12T01:39:19.142Z 22 440-459 96 96 96 166 166 166 F T T <---
111852024-02-12T01:39:19.143Z 23 460-479 96 96 96 165 165 165 F T T <---
111862024-02-12T01:39:19.143Z 24 480-499 93 93 93 158 158 158 F T T <---
111872024-02-12T01:39:19.143Z 25 500-519 98 98 98 170 170 170 F F F
111882024-02-12T01:39:19.143Z 26 520-539 98 98 98 170 170 170 F F F
111892024-02-12T01:39:19.143Z 27 540-559 98 98 98 170 170 170 F F F
111902024-02-12T01:39:19.143Z 28 560-579 95 95 95 163 163 163 F F F
111912024-02-12T01:39:19.143Z 29 580-599 96 96 96 166 166 166 F F F
111922024-02-12T01:39:19.143ZMax gen: 100, Max flush: 177
111932024-02-12T01:39:19.143ZError: Difference in extent metadata found!
111942024-02-12T01:39:19.143ZA Difference in extent metadata is expected here
111952024-02-12T01:39:19.143ZOn loop 50
111962024-02-12T01:39:19.143Z
111972024-02-12T01:39:19.143ZVerifying data now
111982024-02-12T01:39:23.713ZNo change after verify
111992024-02-12T01:39:23.713ZLoop: 50 Downstairs dump after verify (and repair):
112002024-02-12T01:39:23.733ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112012024-02-12T01:39:23.733Z 0 000-019 99 99 99 173 173 173 F F F
112022024-02-12T01:39:23.733Z 1 020-039 96 96 96 166 166 166 F F F
112032024-02-12T01:39:23.733Z 2 040-059 101 101 101 178 178 178 F F F
112042024-02-12T01:39:23.733Z 3 060-079 96 96 96 165 165 165 F F F
112052024-02-12T01:39:23.733Z 4 080-099 98 98 98 170 170 170 F F F
112062024-02-12T01:39:23.733Z 5 100-119 100 100 100 175 175 175 F F F
112072024-02-12T01:39:23.733Z 6 120-139 100 100 100 177 177 177 F F F
112082024-02-12T01:39:23.733Z 7 140-159 100 100 100 176 176 176 F F F
112092024-02-12T01:39:23.733Z 8 160-179 100 100 100 176 176 176 F F F
112102024-02-12T01:39:23.733Z 9 180-199 101 101 101 178 178 178 F F F
112112024-02-12T01:39:23.734Z 10 200-219 97 97 97 167 167 167 F F F
112122024-02-12T01:39:23.734Z 11 220-239 96 96 96 166 166 166 F F F
112132024-02-12T01:39:23.737Z 12 240-259 101 101 101 178 178 178 F F F
112142024-02-12T01:39:23.737Z 13 260-279 100 100 100 175 175 175 F F F
112152024-02-12T01:39:23.737Z 14 280-299 98 98 98 170 170 170 F F F
112162024-02-12T01:39:23.737Z 15 300-319 101 101 101 178 178 178 F F F
112172024-02-12T01:39:23.737Z 16 320-339 101 101 101 178 178 178 F F F
112182024-02-12T01:39:23.737Z 17 340-359 93 93 93 158 158 158 F F F
112192024-02-12T01:39:23.737Z 18 360-379 101 101 101 178 178 178 F F F
112202024-02-12T01:39:23.737Z 19 380-399 101 101 101 178 178 178 F F F
112212024-02-12T01:39:23.737Z 20 400-419 101 101 101 178 178 178 F F F
112222024-02-12T01:39:23.737Z 21 420-439 95 95 95 163 163 163 F F F
112232024-02-12T01:39:23.737Z 22 440-459 101 101 101 178 178 178 F F F
112242024-02-12T01:39:23.738Z 23 460-479 101 101 101 178 178 178 F F F
112252024-02-12T01:39:23.738Z 24 480-499 101 101 101 178 178 178 F F F
112262024-02-12T01:39:23.738Z 25 500-519 98 98 98 170 170 170 F F F
112272024-02-12T01:39:23.738Z 26 520-539 98 98 98 170 170 170 F F F
112282024-02-12T01:39:23.738Z 27 540-559 98 98 98 170 170 170 F F F
112292024-02-12T01:39:23.738Z 28 560-579 95 95 95 163 163 163 F F F
112302024-02-12T01:39:23.738Z 29 580-599 96 96 96 166 166 166 F F F
112312024-02-12T01:39:23.738ZMax gen: 101, Max flush: 178
112322024-02-12T01:39:23.738Z
112332024-02-12T01:39:23.738ZBegin loop 51
112342024-02-12T01:39:23.738ZDownstairs to restart: 2
112352024-02-12T01:39:23.753Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64199471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
112362024-02-12T01:39:23.758Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642233029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
112372024-02-12T01:39:23.758Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642275387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
112382024-02-12T01:39:23.758Z{"msg":"Crucible ec2d9119-fe6b-4272-a618-174ab618589c has session id: 3d8febfc-151a-49be-92dc-b40066c1c0ad","v":0,"name":"crucible","level":30Crucible runtime is spawned
112392024-02-12T01:39:23.758Z,"time":"2024-02-12T01:39:22.642305006Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112402024-02-12T01:39:23.758Z{"msg":"Upstairs opts: Upstairs UUID: ec2d9119-fe6b-4272-a618-174ab618589c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642357953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112412024-02-12T01:39:23.758Z{"msg":"Crucible stats registered with UUID: ec2d9119-fe6b-4272-a618-174ab618589c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642398951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112422024-02-12T01:39:23.758Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64242052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
112432024-02-12T01:39:23.758Z{"msg":"ec2d9119-fe6b-4272-a618-174ab618589c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642586842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112442024-02-12T01:39:23.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642621631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112452024-02-12T01:39:23.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642684497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112462024-02-12T01:39:23.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642725355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112472024-02-12T01:39:23.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642757864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112482024-02-12T01:39:23.758Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642777053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112492024-02-12T01:39:23.758Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642801932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112502024-02-12T01:39:23.758Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64283998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112512024-02-12T01:39:23.758Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.642895887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112522024-02-12T01:39:23.758Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643019481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112532024-02-12T01:39:23.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64305318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112542024-02-12T01:39:23.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643087738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112552024-02-12T01:39:23.759Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:22.643118937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112562024-02-12T01:39:23.759Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:22.643160885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112572024-02-12T01:39:23.759Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:22.643213972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112582024-02-12T01:39:23.759Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64325331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112592024-02-12T01:39:23.759Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643288629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112602024-02-12T01:39:23.759Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643331697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112612024-02-12T01:39:23.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64367941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112622024-02-12T01:39:23.759Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643715108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112632024-02-12T01:39:23.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643730478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112642024-02-12T01:39:23.759Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.643756156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112652024-02-12T01:39:23.759Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644221254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112662024-02-12T01:39:23.759Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644332019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112672024-02-12T01:39:23.759Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64473993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112682024-02-12T01:39:23.759Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644778468Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112692024-02-12T01:39:23.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644833945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112702024-02-12T01:39:23.759Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644877253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112712024-02-12T01:39:23.759Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.644911481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112722024-02-12T01:39:23.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:22.64494395Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112732024-02-12T01:39:24.758Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.645479115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112742024-02-12T01:39:24.758Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.645637977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112752024-02-12T01:39:24.762Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.646750544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112762024-02-12T01:39:24.762Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.646790302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112772024-02-12T01:39:24.762Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.64726626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112782024-02-12T01:39:24.763Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.647844772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112792024-02-12T01:39:24.763Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.64789112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112802024-02-12T01:39:24.763Z{"msg":"[0]R flush_numbers[0..12]: [173, 166, 178, 165, 170, 175, 177, 176, 176, 178, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.647917439Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112812024-02-12T01:39:24.763Z{"msg":"[0]R generation[0..12]: [99, 96, 101, 96, 98, 100, 100, 100, 100, 101, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.647952627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112822024-02-12T01:39:24.763Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.647978466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112832024-02-12T01:39:24.763Z{"msg":"[1]R flush_numbers[0..12]: [173, 166, 178, 165, 170, 175, 177, 176, 176, 178, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648013824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112842024-02-12T01:39:24.763Z{"msg":"[1]R generation[0..12]: [99, 96, 101, 96, 98, 100, 100, 100, 100, 101, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648048442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112852024-02-12T01:39:24.763Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648082231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112862024-02-12T01:39:24.763Z{"msg":"[2]R flush_numbers[0..12]: [173, 166, 178, 165, 170, 175, 177, 176, 176, 178, 167, 166]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
112872024-02-12T01:39:24.763Z,"time":"2024-02-12T01:39:23.648132258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112882024-02-12T01:39:24.763Z{"msg":"[2]R generation[0..12]: [99, 96, 101, 96, 98, 100, 100, 100, 100, 101, 97, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648173357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112892024-02-12T01:39:24.763Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648212375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112902024-02-12T01:39:24.763Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648240163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112912024-02-12T01:39:24.763Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648277272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112922024-02-12T01:39:24.763Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.64830343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112932024-02-12T01:39:24.763Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648328279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112942024-02-12T01:39:24.763Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648363237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112952024-02-12T01:39:24.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648398396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112962024-02-12T01:39:24.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648436604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112972024-02-12T01:39:24.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648463363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112982024-02-12T01:39:24.763Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648498761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
112992024-02-12T01:39:24.763Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648532109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
113002024-02-12T01:39:24.763Z{"msg":"ec2d9119-fe6b-4272-a618-174ab618589c is now active with session: 3d8febfc-151a-49be-92dc-b40066c1c0ad","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113012024-02-12T01:39:24.763Zlevel":30,"time":"2024-02-12T01:39:23.648571078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
113022024-02-12T01:39:24.764Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648605836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543,"session_id":"3d8febfc-151a-49be-92dc-b40066c1c0ad"}
113032024-02-12T01:39:24.764Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:23.648628405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
113042024-02-12T01:39:24.764ZLoading write count information from file "/tmp/test_repair_verify.data"
113052024-02-12T01:39:24.764ZRun Repair workload
113062024-02-12T01:39:24.767Z01/30 Write block 479 len 40960 data: 10 10 8 8 9 9 9 10 10 11
113072024-02-12T01:39:24.767Z02/30 Write block 424 len 12288 data: 7 6 5
113082024-02-12T01:39:24.767Z03/30 Read block 495 len 24576
113092024-02-12T01:39:24.767Z04/30 Write block 231 len 40960 data: 5 6 6 6 5 7 10 11 10 10
113102024-02-12T01:39:24.767Z05/30 Read block 442 len 8192
113112024-02-12T01:39:24.770Z06/30 Read block 149 len 4096
113122024-02-12T01:39:24.770Z07/30 Flush
113132024-02-12T01:39:24.773Z08/30 Write block 313 len 32768 data: 7 4 5 4 6 8 9 8
113142024-02-12T01:39:24.773Z09/30 Write block 162 len 40960 data: 9 9 9 11 10 9 9 11 11 10
113152024-02-12T01:39:24.776Z10/30 Write block 394 len 36864 data: 10 8 7 7 7 6 6 6 8
113162024-02-12T01:39:24.776Z11/30 Flush
113172024-02-12T01:39:24.779Z12/30 Write block 282 len 4096 data: 11
113182024-02-12T01:39:24.779Z13/30 Write block 561 len 16384 data: 7 6 7 6
113192024-02-12T01:39:24.779Z14/30 Write block 343 len 4096 data: 6
113202024-02-12T01:39:24.779Z15/30 Read block 508 len 12288
113212024-02-12T01:39:24.779Z16/30 Read block 78 len 4096
113222024-02-12T01:39:24.781Z17/30 Write block 384 len 24576 data: 9 11 11 10 10 9
113232024-02-12T01:39:24.782Z18/30 Read block 462 len 12288
113242024-02-12T01:39:24.782Z19/30 Read block 103 len 24576
113252024-02-12T01:39:24.787Z20/30 Read block 214 len 40960
113262024-02-12T01:39:24.787Z21/30 Read block 447 len 12288
113272024-02-12T01:39:24.788Z22/30 Write block 162 len 36864 data: 10 10 10 12 11 10 10 12 12
113282024-02-12T01:39:24.788Z23/30 Flush
113292024-02-12T01:39:24.790Z24/30 Write block 251 len 40960 data: 6 5 6 5 5 4 5 5 5 8
113302024-02-12T01:39:24.790Z25/30 Write block 228 len 12288 data: 7 6 4
113312024-02-12T01:39:24.790Z26/30 Write block 234 len 40960 data: 7 6 8 11 12 11 11 8 10 11
113322024-02-12T01:39:24.790Z27/30 Write block 94 len 12288 data: 4 4 6
113332024-02-12T01:39:24.791Z28/30 Read block 277 len 40960
113342024-02-12T01:39:24.794Z29/30 Read block 174 len 16384
113352024-02-12T01:39:24.794Z30/30 Read block 182 len 8192
113362024-02-12T01:39:24.794Z----------------------------------------------------------------
113372024-02-12T01:39:24.794Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
113382024-02-12T01:39:24.794ZStates: Active Active Active
113392024-02-12T01:39:24.794ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113402024-02-12T01:39:24.794Z 1 Acked 1000 Write 10 Done Done Done false
113412024-02-12T01:39:24.794Z 2 Acked 1001 Write 3 Done Done Done false
113422024-02-12T01:39:24.794Z 3 Acked 1002 Read 6 Done Done Done false
113432024-02-12T01:39:24.794Z 4 Acked 1003 Write 10 Done Done Done false
113442024-02-12T01:39:24.794Z 5 Acked 1004 Read 2 Done Done Sent false
113452024-02-12T01:39:24.794Z 6 Acked 1005 Read 1 Done Done Sent false
113462024-02-12T01:39:24.794Z 7 Acked 1006 Flush 0 Done Done Sent false
113472024-02-12T01:39:24.794Z 8 Acked 1007 Write 8 Done Done Sent false
113482024-02-12T01:39:24.794Z 9 Acked 1008 Write 10 Done Done Sent false
113492024-02-12T01:39:24.794Z 10 Acked 1009 Write 9 Done Done Sent false
113502024-02-12T01:39:24.795Z 11 Acked 1010 Flush 0 Done Done Sent false
113512024-02-12T01:39:24.795Z 12 Acked 1011 Write 1 Done Done Sent false
113522024-02-12T01:39:24.795Z 13 Acked 1012 Write 4 Done Done Sent false
113532024-02-12T01:39:24.795Z 14 Acked 1013 Write 1 Done Done Sent false
113542024-02-12T01:39:24.795Z 15 Acked 1014 Read 3 Done Done Sent false
113552024-02-12T01:39:24.795Z 16 Acked 1015 Read 1 Done Done Sent false
113562024-02-12T01:39:24.795Z 17 Acked 1016 Write 6 Done Done Sent false
113572024-02-12T01:39:24.795Z 18 Acked 1017 Read 3 Done Done Sent false
113582024-02-12T01:39:24.795Z 19 Acked 1018 Read 6 Done Done Sent false
113592024-02-12T01:39:24.795Z 20 Acked 1019 Read 10 Done Done Sent false
113602024-02-12T01:39:24.795Z 21 Acked 1020 Read 3 Done Done Sent false
113612024-02-12T01:39:24.795Z 22 Acked 1021 Write 9 Done Done Sent false
113622024-02-12T01:39:24.795Z 23 Acked 1022 Flush 0 Done Done Sent false
113632024-02-12T01:39:24.795Z 24 Acked 1023 Write 10 Done Done Sent false
113642024-02-12T01:39:24.795Z 25 Acked 1024 Write 3 Done Done Sent false
113652024-02-12T01:39:24.795Z 26 Acked 1025 Write 10 Done Done Sent false
113662024-02-12T01:39:24.795Z 27 Acked 1026 Write 3 Done Done Sent false
113672024-02-12T01:39:24.795Z 28 Acked 1027 Read 10 Done Done Sent false
113682024-02-12T01:39:24.795Z 29 Acked 1028 Read 4 Done Done Sent false
113692024-02-12T01:39:24.795Z 30 Acked 1029 Read 2 Done Sent Sent false
113702024-02-12T01:39:24.795Z STATES DS:0 DS:1 DS:2 TOTAL
113712024-02-12T01:39:24.795Z New 0 0 0 0
113722024-02-12T01:39:24.795Z Sent 0 1 26 27
113732024-02-12T01:39:24.796Z Done 30 29 4 63
113742024-02-12T01:39:24.796Z Skipped 0 0 0 0
113752024-02-12T01:39:24.796Z Error 0 0 0 0
113762024-02-12T01:39:24.796ZLast Flush: 1022 1022 0
113772024-02-12T01:39:24.796ZDownstairs last five completed:
113782024-02-12T01:39:24.796ZUpstairs last five completed: 30 29 28 27 26
113792024-02-12T01:39:24.796Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:23.681927906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1543}
113802024-02-12T01:39:24.799ZWrote out file "/tmp/test_repair_verify.data"
113812024-02-12T01:39:24.799Z
113822024-02-12T01:39:24.802ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113832024-02-12T01:39:24.821ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113842024-02-12T01:39:24.821Z 0 000-019 99 99 99 173 173 173 F F F
113852024-02-12T01:39:24.821Z 1 020-039 96 96 96 166 166 166 F F F
113862024-02-12T01:39:24.821Z 2 040-059 101 101 101 178 178 178 F F F
113872024-02-12T01:39:24.821Z 3 060-079 96 96 96 165 165 165 F F F
113882024-02-12T01:39:24.821Z 4 080-099 98 98 98 170 170 170 T T F <---
113892024-02-12T01:39:24.821Z 5 100-119 100 100 100 175 175 175 F F F
113902024-02-12T01:39:24.821Z 6 120-139 100 100 100 177 177 177 F F F
113912024-02-12T01:39:24.821Z 7 140-159 100 100 100 176 176 176 F F F
113922024-02-12T01:39:24.821Z 8 160-179 102 102 100 181 181 176 F F F <---
113932024-02-12T01:39:24.821Z 9 180-199 101 101 101 178 178 178 F F F
113942024-02-12T01:39:24.821Z 10 200-219 97 97 97 167 167 167 F F F
113952024-02-12T01:39:24.821Z 11 220-239 102 102 96 179 179 166 T T T <---
113962024-02-12T01:39:24.821Z 12 240-259 102 102 101 179 179 178 T T T <---
113972024-02-12T01:39:24.821Z 13 260-279 100 100 100 175 175 175 T T F <---
113982024-02-12T01:39:24.821Z 14 280-299 102 102 98 181 181 170 F F F <---
113992024-02-12T01:39:24.822Z 15 300-319 102 102 101 180 180 178 F F F <---
114002024-02-12T01:39:24.822Z 16 320-339 102 102 101 180 180 178 F F F <---
114012024-02-12T01:39:24.822Z 17 340-359 102 102 93 181 181 158 F F F <---
114022024-02-12T01:39:24.822Z 18 360-379 101 101 101 178 178 178 F F F
114032024-02-12T01:39:24.822Z 19 380-399 102 102 101 181 181 178 F F F <---
114042024-02-12T01:39:24.822Z 20 400-419 102 102 101 180 180 178 F F F <---
114052024-02-12T01:39:24.825Z 21 420-439 102 102 95 179 179 163 F F T <---
114062024-02-12T01:39:24.825Z 22 440-459 101 101 101 178 178 178 F F F
114072024-02-12T01:39:24.825Z 23 460-479 102 102 101 179 179 178 F F T <---
114082024-02-12T01:39:24.825Z 24 480-499 102 102 101 179 179 178 F F T <---
114092024-02-12T01:39:24.825Z 25 500-519 98 98 98 170 170 170 F F F
114102024-02-12T01:39:24.825Z 26 520-539 98 98 98 170 170 170 F F F
114112024-02-12T01:39:24.825Z 27 540-559 98 98 98 170 170 170 F F F
114122024-02-12T01:39:24.825Z 28 560-579 102 102 95 181 181 163 F F F <---
114132024-02-12T01:39:24.825Z 29 580-599 96 96 96 166 166 166 F F F
114142024-02-12T01:39:24.825ZMax gen: 102, Max flush: 181
114152024-02-12T01:39:24.825ZError: Difference in extent metadata found!
114162024-02-12T01:39:24.825ZA Difference in extent metadata is expected here
114172024-02-12T01:39:24.825ZOn loop 51
114182024-02-12T01:39:24.825Z
114192024-02-12T01:39:24.828ZVerifying data now
114202024-02-12T01:39:30.544ZNo change after verify
114212024-02-12T01:39:30.544ZLoop: 51 Downstairs dump after verify (and repair):
114222024-02-12T01:39:30.559ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114232024-02-12T01:39:30.559Z 0 000-019 99 99 99 173 173 173 F F F
114242024-02-12T01:39:30.559Z 1 020-039 96 96 96 166 166 166 F F F
114252024-02-12T01:39:30.559Z 2 040-059 101 101 101 178 178 178 F F F
114262024-02-12T01:39:30.559Z 3 060-079 96 96 96 165 165 165 F F F
114272024-02-12T01:39:30.559Z 4 080-099 103 103 103 182 182 182 F F F
114282024-02-12T01:39:30.559Z 5 100-119 100 100 100 175 175 175 F F F
114292024-02-12T01:39:30.559Z 6 120-139 100 100 100 177 177 177 F F F
114302024-02-12T01:39:30.559Z 7 140-159 100 100 100 176 176 176 F F F
114312024-02-12T01:39:30.560Z 8 160-179 102 102 102 181 181 181 F F F
114322024-02-12T01:39:30.560Z 9 180-199 101 101 101 178 178 178 F F F
114332024-02-12T01:39:30.560Z 10 200-219 97 97 97 167 167 167 F F F
114342024-02-12T01:39:30.563Z 11 220-239 103 103 103 182 182 182 F F F
114352024-02-12T01:39:30.563Z 12 240-259 103 103 103 182 182 182 F F F
114362024-02-12T01:39:30.563Z 13 260-279 103 103 103 182 182 182 F F F
114372024-02-12T01:39:30.563Z 14 280-299 102 102 102 181 181 181 F F F
114382024-02-12T01:39:30.563Z 15 300-319 102 102 102 180 180 180 F F F
114392024-02-12T01:39:30.563Z 16 320-339 102 102 102 180 180 180 F F F
114402024-02-12T01:39:30.563Z 17 340-359 102 102 102 181 181 181 F F F
114412024-02-12T01:39:30.563Z 18 360-379 101 101 101 178 178 178 F F F
114422024-02-12T01:39:30.563Z 19 380-399 102 102 102 181 181 181 F F F
114432024-02-12T01:39:30.563Z 20 400-419 102 102 102 180 180 180 F F F
114442024-02-12T01:39:30.563Z 21 420-439 102 102 102 179 179 179 F F F
114452024-02-12T01:39:30.563Z 22 440-459 101 101 101 178 178 178 F F F
114462024-02-12T01:39:30.563Z 23 460-479 102 102 102 179 179 179 F F F
114472024-02-12T01:39:30.563Z 24 480-499 102 102 102 179 179 179 F F F
114482024-02-12T01:39:30.563Z 25 500-519 98 98 98 170 170 170 F F F
114492024-02-12T01:39:30.563Z 26 520-539 98 98 98 170 170 170 F F F
114502024-02-12T01:39:30.563Z 27 540-559 98 98 98 170 170 170 F F F
114512024-02-12T01:39:30.563Z 28 560-579 102 102 102 181 181 181 F F F
114522024-02-12T01:39:30.563Z 29 580-599 96 96 96 166 166 166 F F F
114532024-02-12T01:39:30.563ZMax gen: 103, Max flush: 182
114542024-02-12T01:39:30.564Z
114552024-02-12T01:39:30.564ZBegin loop 52
114562024-02-12T01:39:30.564ZDownstairs to restart: 0
114572024-02-12T01:39:30.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469464849Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
114582024-02-12T01:39:30.582Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469703957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
114592024-02-12T01:39:30.582Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469728656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
114602024-02-12T01:39:30.582Z{"msg":"Crucible fddeeb1e-48d5-46cc-9d2b-ccf33a969328 has session id: d40df301-2e69-472b-b77b-55e333b56487","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469760895Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114612024-02-12T01:39:30.587Z{"msg":"Upstairs opts: Upstairs UUID: fddeeb1e-48d5-46cc-9d2b-ccf33a969328, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0Crucible runtime is spawned
114622024-02-12T01:39:30.587Z,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469805912Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114632024-02-12T01:39:30.587Z{"msg":"Crucible stats registered with UUID: fddeeb1e-48d5-46cc-9d2b-ccf33a969328","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469829301Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114642024-02-12T01:39:30.587Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469868159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
114652024-02-12T01:39:30.587Z{"msg":"fddeeb1e-48d5-46cc-9d2b-ccf33a969328 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.469970074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114662024-02-12T01:39:30.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470030942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114672024-02-12T01:39:30.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.47006229Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114682024-02-12T01:39:30.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470087909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114692024-02-12T01:39:30.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470113598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114702024-02-12T01:39:30.587Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470138746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114712024-02-12T01:39:30.587Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470162195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114722024-02-12T01:39:30.587Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470184704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114732024-02-12T01:39:30.587Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470239062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114742024-02-12T01:39:30.587Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.47027502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114752024-02-12T01:39:30.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470300849Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114762024-02-12T01:39:30.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470382445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114772024-02-12T01:39:30.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.470459141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114782024-02-12T01:39:30.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471056643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114792024-02-12T01:39:30.587Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471099491Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114802024-02-12T01:39:30.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471156108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114812024-02-12T01:39:30.588Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471209626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114822024-02-12T01:39:30.588Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471702532Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114832024-02-12T01:39:30.588Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.471827156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114842024-02-12T01:39:30.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472249106Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114852024-02-12T01:39:30.588Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472300163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114862024-02-12T01:39:30.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472322492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114872024-02-12T01:39:30.588Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.47236892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114882024-02-12T01:39:30.588Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472401189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114892024-02-12T01:39:30.588Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472421628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114902024-02-12T01:39:30.588Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472448086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114912024-02-12T01:39:30.588Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472465666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114922024-02-12T01:39:30.588Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.472923704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114932024-02-12T01:39:30.592Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473344504Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114942024-02-12T01:39:30.592Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473387342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114952024-02-12T01:39:30.592Z{"msg":"[0]R flush_numbers[0..12]: [173, 166, 178, 165, 182, 175, 177, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.47341168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114962024-02-12T01:39:30.592Z{"msg":"[0]R generation[0..12]: [99, 96, 101, 96, 103, 100, 100, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473440749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114972024-02-12T01:39:30.592Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473457138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114982024-02-12T01:39:30.592Z{"msg":"[1]R flush_numbers[0..12]: [173, 166, 178, 165, 182, 175, 177, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473478247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
114992024-02-12T01:39:30.592Z{"msg":"[1]R generation[0..12]: [99, 96, 101, 96, 103, 100, 100, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473492917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115002024-02-12T01:39:30.592Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473513665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115012024-02-12T01:39:30.592Z{"msg":"[2]R flush_numbers[0..12]: [173, 166, 178, 165, 182, 175, 177, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473527925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115022024-02-12T01:39:30.592Z{"msg":"[2]R generation[0..12]: [99, 96, 101, 96, 103, 100, 100, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473547994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115032024-02-12T01:39:30.592Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473562363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO
115042024-02-12T01:39:30.592Z"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115052024-02-12T01:39:30.592Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473590152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115062024-02-12T01:39:30.592Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473603591Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115072024-02-12T01:39:30.592Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115082024-02-12T01:39:30.592Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.47363665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115092024-02-12T01:39:30.593Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473649959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115102024-02-12T01:39:30.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473662778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115112024-02-12T01:39:30.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473675958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115122024-02-12T01:39:30.593Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473696937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115132024-02-12T01:39:30.593Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473710566Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115142024-02-12T01:39:30.593Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473733215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115152024-02-12T01:39:30.593Z{"msg":"fddeeb1e-48d5-46cc-9d2b-ccf33a969328 is now active with session: d40df301-2e69-472b-b77b-55e333b56487","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473749584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115162024-02-12T01:39:30.593Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473762524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551,"session_id":"d40df301-2e69-472b-b77b-55e333b56487"}
115172024-02-12T01:39:30.593Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:29.473782613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
115182024-02-12T01:39:30.593ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115192024-02-12T01:39:30.593ZLoading write count information from file "/tmp/test_repair_verify.data"
115202024-02-12T01:39:30.593ZRun Repair workload
115212024-02-12T01:39:30.593Z01/30 Write block 52 len 24576 data: 7 5 5 6 6 9
115222024-02-12T01:39:30.593Z02/30 Read block 42 len 8192
115232024-02-12T01:39:30.593Z03/30 Write block 66 len 20480 data: 9 10 9 11 10
115242024-02-12T01:39:30.593Z04/30 Flush
115252024-02-12T01:39:30.596Z05/30 Write block 566 len 40960 data: 8 9 11 13 12 10 12 14 13 11
115262024-02-12T01:39:30.596Z06/30 Write block 576 len 16384 data: 8 9 9 8
115272024-02-12T01:39:30.596Z07/30 Read block 93 len 20480
115282024-02-12T01:39:30.596Z08/30 Flush
115292024-02-12T01:39:30.599Z09/30 Write block 138 len 8192 data: 10 10
115302024-02-12T01:39:30.599Z10/30 Read block 317 len 24576
115312024-02-12T01:39:30.599Z11/30 Write block 462 len 16384 data: 5 5 6 6
115322024-02-12T01:39:30.600Z12/30 Flush
115332024-02-12T01:39:30.602Z13/30 Write block 67 len 8192 data: 11 10
115342024-02-12T01:39:30.602Z14/30 Read block 32 len 8192
115352024-02-12T01:39:30.602Z15/30 Read block 283 len 8192
115362024-02-12T01:39:30.602Z16/30 Read block 298 len 40960
115372024-02-12T01:39:30.605Z17/30 Write block 277 len 4096 data: 9
115382024-02-12T01:39:30.605Z18/30 Read block 307 len 32768
115392024-02-12T01:39:30.605Z19/30 Read block 225 len 4096
115402024-02-12T01:39:30.608Z20/30 Write block 439 len 24576 data: 9 8 8 8 6 7
115412024-02-12T01:39:30.608Z21/30 Read block 1 len 24576
115422024-02-12T01:39:30.608Z22/30 Read block 184 len 20480
115432024-02-12T01:39:30.611Z23/30 Read block 217 len 24576
115442024-02-12T01:39:30.611Z24/30 Read block 540 len 28672
115452024-02-12T01:39:30.611Z25/30 Read block 495 len 24576
115462024-02-12T01:39:30.611Z26/30 Flush
115472024-02-12T01:39:30.615Z27/30 Write block 366 len 36864 data: 8 12 10 11 11 10 12 13 12
115482024-02-12T01:39:30.622Z28/30 Read block 424 len 8192
115492024-02-12T01:39:30.622Z29/30 Write block 400 len 16384 data: 7 7 9 7
115502024-02-12T01:39:30.622Z30/30 Write block 339 len 8192 data: 6 5
115512024-02-12T01:39:30.622Z----------------------------------------------------------------
115522024-02-12T01:39:30.622Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:22
115532024-02-12T01:39:30.622ZStates: Active Active Active
115542024-02-12T01:39:30.622ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
115552024-02-12T01:39:30.622Z 9 Acked 1008 Write 2 Done Done Done false
115562024-02-12T01:39:30.622Z 10 Acked 1009 Read 6 Sent Done Done false
115572024-02-12T01:39:30.623Z 11 Acked 1010 Write 4 Sent Done Done false
115582024-02-12T01:39:30.623Z 12 Acked 1011 Flush 0 Sent Done Done false
115592024-02-12T01:39:30.623Z 13 Acked 1012 Write 2 Sent Done Done false
115602024-02-12T01:39:30.623Z 14 Acked 1013 Read 2 Sent Done Done false
115612024-02-12T01:39:30.623Z 15 Acked 1014 Read 2 Sent Done Done false
115622024-02-12T01:39:30.623Z 16 Acked 1015 Read 10 Sent Done Done false
115632024-02-12T01:39:30.623Z 17 Acked 1016 Write 1 Sent Done Done false
115642024-02-12T01:39:30.623Z 18 Acked 1017 Read 8 Sent Done Done false
115652024-02-12T01:39:30.623Z 19 Acked 1018 Read 1 Sent Done Done false
115662024-02-12T01:39:30.623Z 20 Acked 1019 Write 6 Sent Done Done false
115672024-02-12T01:39:30.623Z 21 Acked 1020 Read 6 Sent Done Done false
115682024-02-12T01:39:30.623Z 22 Acked 1021 Read 5 Sent Done Done false
115692024-02-12T01:39:30.623Z 23 Acked 1022 Read 6 Sent Done Done false
115702024-02-12T01:39:30.623Z 24 Acked 1023 Read 7 Sent Done Done false
115712024-02-12T01:39:30.623Z 25 Acked 1024 Read 6 Sent Done Done false
115722024-02-12T01:39:30.623Z 26 Acked 1025 Flush 0 Sent Done Done false
115732024-02-12T01:39:30.623Z 27 Acked 1026 Write 9 Sent Done Done false
115742024-02-12T01:39:30.623Z 28 Acked 1027 Read 2 Sent Done Done false
115752024-02-12T01:39:30.623Z 29 Acked 1028 Write 4 Sent Done Sent false
115762024-02-12T01:39:30.623Z 30 Acked 1029 Write 2 Sent Sent Sent false
115772024-02-12T01:39:30.623Z STATES DS:0 DS:1 DS:2 TOTAL
115782024-02-12T01:39:30.623Z New 0 0 0 0
115792024-02-12T01:39:30.623Z Sent 21 1 2 24
115802024-02-12T01:39:30.623Z Done 1 21 20 42
115812024-02-12T01:39:30.623Z Skipped 0 0 0 0
115822024-02-12T01:39:30.623Z Error 0 0 0 0
115832024-02-12T01:39:30.624ZLast Flush: 1007 1025 1025
115842024-02-12T01:39:30.624ZDownstairs last five completed: 1007 1006 1005 1004 1003
115852024-02-12T01:39:30.624ZUpstairs last five completed: 30 29 28 27 26
115862024-02-12T01:39:30.624Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:29.504207081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1551}
115872024-02-12T01:39:30.624ZWrote out file "/tmp/test_repair_verify.data"
115882024-02-12T01:39:30.627Z
115892024-02-12T01:39:30.627ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115902024-02-12T01:39:30.641ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115912024-02-12T01:39:30.641Z 0 000-019 99 99 99 173 173 173 F F F
115922024-02-12T01:39:30.641Z 1 020-039 96 96 96 166 166 166 F F F
115932024-02-12T01:39:30.641Z 2 040-059 104 104 104 183 183 183 F F F
115942024-02-12T01:39:30.641Z 3 060-079 104 104 104 183 186 186 F F F <---
115952024-02-12T01:39:30.641Z 4 080-099 103 103 103 182 182 182 F F F
115962024-02-12T01:39:30.641Z 5 100-119 100 100 100 175 175 175 F F F
115972024-02-12T01:39:30.641Z 6 120-139 100 104 104 177 185 185 T F F <---
115982024-02-12T01:39:30.641Z 7 140-159 100 100 100 176 176 176 F F F
115992024-02-12T01:39:30.641Z 8 160-179 102 102 102 181 181 181 F F F
116002024-02-12T01:39:30.644Z 9 180-199 101 101 101 178 178 178 F F F
116012024-02-12T01:39:30.644Z 10 200-219 97 97 97 167 167 167 F F F
116022024-02-12T01:39:30.645Z 11 220-239 103 103 103 182 182 182 F F F
116032024-02-12T01:39:30.645Z 12 240-259 103 103 103 182 182 182 F F F
116042024-02-12T01:39:30.645Z 13 260-279 103 104 104 182 186 186 F F F <---
116052024-02-12T01:39:30.645Z 14 280-299 102 102 102 181 181 181 F F F
116062024-02-12T01:39:30.645Z 15 300-319 102 102 102 180 180 180 F F F
116072024-02-12T01:39:30.645Z 16 320-339 102 102 102 180 180 180 F T T <---
116082024-02-12T01:39:30.645Z 17 340-359 102 102 102 181 181 181 F T T <---
116092024-02-12T01:39:30.645Z 18 360-379 101 101 101 178 178 178 F T T <---
116102024-02-12T01:39:30.645Z 19 380-399 102 102 102 181 181 181 F F F
116112024-02-12T01:39:30.645Z 20 400-419 102 102 102 180 180 180 F T T <---
116122024-02-12T01:39:30.645Z 21 420-439 102 104 104 179 186 186 F F F <---
116132024-02-12T01:39:30.645Z 22 440-459 101 104 104 178 186 186 F F F <---
116142024-02-12T01:39:30.645Z 23 460-479 102 104 104 179 185 185 F F F <---
116152024-02-12T01:39:30.645Z 24 480-499 102 102 102 179 179 179 F F F
116162024-02-12T01:39:30.645Z 25 500-519 98 98 98 170 170 170 F F F
116172024-02-12T01:39:30.645Z 26 520-539 98 98 98 170 170 170 F F F
116182024-02-12T01:39:30.645Z 27 540-559 98 98 98 170 170 170 F F F
116192024-02-12T01:39:30.645Z 28 560-579 104 104 104 184 184 184 F F F
116202024-02-12T01:39:30.645Z 29 580-599 96 96 96 166 166 166 F F F
116212024-02-12T01:39:30.645ZMax gen: 104, Max flush: 186
116222024-02-12T01:39:30.645ZError: Difference in extent metadata found!
116232024-02-12T01:39:30.645ZA Difference in extent metadata is expected here
116242024-02-12T01:39:30.645ZOn loop 52
116252024-02-12T01:39:30.645Z
116262024-02-12T01:39:30.649ZVerifying data now
116272024-02-12T01:39:36.154ZNo change after verify
116282024-02-12T01:39:36.154ZLoop: 52 Downstairs dump after verify (and repair):
116292024-02-12T01:39:36.168ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116302024-02-12T01:39:36.169Z 0 000-019 99 99 99 173 173 173 F F F
116312024-02-12T01:39:36.169Z 1 020-039 96 96 96 166 166 166 F F F
116322024-02-12T01:39:36.169Z 2 040-059 104 104 104 183 183 183 F F F
116332024-02-12T01:39:36.169Z 3 060-079 104 104 104 186 186 186 F F F
116342024-02-12T01:39:36.169Z 4 080-099 103 103 103 182 182 182 F F F
116352024-02-12T01:39:36.169Z 5 100-119 100 100 100 175 175 175 F F F
116362024-02-12T01:39:36.169Z 6 120-139 104 104 104 185 185 185 F F F
116372024-02-12T01:39:36.169Z 7 140-159 100 100 100 176 176 176 F F F
116382024-02-12T01:39:36.169Z 8 160-179 102 102 102 181 181 181 F F F
116392024-02-12T01:39:36.169Z 9 180-199 101 101 101 178 178 178 F F F
116402024-02-12T01:39:36.169Z 10 200-219 97 97 97 167 167 167 F F F
116412024-02-12T01:39:36.169Z 11 220-239 103 103 103 182 182 182 F F F
116422024-02-12T01:39:36.170Z 12 240-259 103 103 103 182 182 182 F F F
116432024-02-12T01:39:36.173Z 13 260-279 104 104 104 186 186 186 F F F
116442024-02-12T01:39:36.173Z 14 280-299 102 102 102 181 181 181 F F F
116452024-02-12T01:39:36.173Z 15 300-319 102 102 102 180 180 180 F F F
116462024-02-12T01:39:36.173Z 16 320-339 105 105 105 187 187 187 F F F
116472024-02-12T01:39:36.173Z 17 340-359 105 105 105 187 187 187 F F F
116482024-02-12T01:39:36.173Z 18 360-379 105 105 105 187 187 187 F F F
116492024-02-12T01:39:36.173Z 19 380-399 102 102 102 181 181 181 F F F
116502024-02-12T01:39:36.173Z 20 400-419 105 105 105 187 187 187 F F F
116512024-02-12T01:39:36.173Z 21 420-439 104 104 104 186 186 186 F F F
116522024-02-12T01:39:36.174Z 22 440-459 104 104 104 186 186 186 F F F
116532024-02-12T01:39:36.174Z 23 460-479 104 104 104 185 185 185 F F F
116542024-02-12T01:39:36.174Z 24 480-499 102 102 102 179 179 179 F F F
116552024-02-12T01:39:36.174Z 25 500-519 98 98 98 170 170 170 F F F
116562024-02-12T01:39:36.174Z 26 520-539 98 98 98 170 170 170 F F F
116572024-02-12T01:39:36.174Z 27 540-559 98 98 98 170 170 170 F F F
116582024-02-12T01:39:36.174Z 28 560-579 104 104 104 184 184 184 F F F
116592024-02-12T01:39:36.174Z 29 580-599 96 96 96 166 166 166 F F F
116602024-02-12T01:39:36.174ZMax gen: 105, Max flush: 187
116612024-02-12T01:39:36.174Z
116622024-02-12T01:39:36.174ZBegin loop 53
116632024-02-12T01:39:36.174ZDownstairs to restart: 0
116642024-02-12T01:39:36.189Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075545427Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559}
116652024-02-12T01:39:36.189Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075811165Z","hostname":"Crucible runtime is spawned
116662024-02-12T01:39:36.189Zip-10-150-1-24.us-west-2.compute.internal","pid":1559}
116672024-02-12T01:39:36.193Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075865612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559}
116682024-02-12T01:39:36.193Z{"msg":"Crucible 032b1087-259b-4e1d-aab7-2c191e4422a7 has session id: f7caa37d-624c-4cfd-ab49-0954102e16a9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075892911Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116692024-02-12T01:39:36.193Z{"msg":"Upstairs opts: Upstairs UUID: 032b1087-259b-4e1d-aab7-2c191e4422a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075933549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116702024-02-12T01:39:36.193Z{"msg":"Crucible stats registered with UUID: 032b1087-259b-4e1d-aab7-2c191e4422a7","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.075975367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116712024-02-12T01:39:36.193Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076003815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559}
116722024-02-12T01:39:36.193Z{"msg":"032b1087-259b-4e1d-aab7-2c191e4422a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076028454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116732024-02-12T01:39:36.193Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076049243Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116742024-02-12T01:39:36.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.0761191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116752024-02-12T01:39:36.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076155858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116762024-02-12T01:39:36.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076190687Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116772024-02-12T01:39:36.194Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076220405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116782024-02-12T01:39:36.194Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076250144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116792024-02-12T01:39:36.194Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076279522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116802024-02-12T01:39:36.194Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.0763185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116812024-02-12T01:39:36.194Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076370518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116822024-02-12T01:39:36.194Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076406636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116832024-02-12T01:39:36.194Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:35.076482863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116842024-02-12T01:39:36.194Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076515051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116852024-02-12T01:39:36.194Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:35.076569908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116862024-02-12T01:39:36.194Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:35.076611986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116872024-02-12T01:39:36.194Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076643015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116882024-02-12T01:39:36.194Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076673183Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116892024-02-12T01:39:36.194Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076704062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116902024-02-12T01:39:36.194Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076909092Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116912024-02-12T01:39:36.194Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.07694645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116922024-02-12T01:39:36.194Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.076993428Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116932024-02-12T01:39:36.194Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.077033186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116942024-02-12T01:39:36.194Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.077482395Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116952024-02-12T01:39:36.194Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.07759113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116962024-02-12T01:39:36.195Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.077949463Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116972024-02-12T01:39:36.195Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.078028669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116982024-02-12T01:39:36.195Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.078068917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
116992024-02-12T01:39:36.195Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.078094396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117002024-02-12T01:39:36.195Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.078132524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117012024-02-12T01:39:36.195Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:35.078168212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117022024-02-12T01:39:37.191Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.077947484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117032024-02-12T01:39:37.191Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.078104686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117042024-02-12T01:39:37.198Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.079194684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117052024-02-12T01:39:37.198Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.079231283Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117062024-02-12T01:39:37.198Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.07970758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117072024-02-12T01:39:37.198Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080296382Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117082024-02-12T01:39:37.198Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080349109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117092024-02-12T01:39:37.198Z{"msg":"[0]R flush_numbers[0..12]: [173, 166, 183, 186, 182, 175, 185, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
117102024-02-12T01:39:37.198Z,"time":"2024-02-12T01:39:36.080422546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117112024-02-12T01:39:37.198Z{"msg":"[0]R generation[0..12]: [99, 96, 104, 104, 103, 100, 104, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080465464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117122024-02-12T01:39:37.199Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080505302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117132024-02-12T01:39:37.199Z{"msg":"[1]R flush_numbers[0..12]: [173, 166, 183, 186, 182, 175, 185, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.0805376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117142024-02-12T01:39:37.199Z{"msg":"[1]R generation[0..12]: [99, 96, 104, 104, 103, 100, 104, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080580148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117152024-02-12T01:39:37.199Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080613546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117162024-02-12T01:39:37.199Z{"msg":"[2]R flush_numbers[0..12]: [173, 166, 183, 186, 182, 175, 185, 176, 181, 178, 167, 182]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080644715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117172024-02-12T01:39:37.199Z{"msg":"[2]R generation[0..12]: [99, 96, 104, 104, 103, 100, 104, 100, 102, 101, 97, 103]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080675854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117182024-02-12T01:39:37.199Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080707732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117192024-02-12T01:39:37.199Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.08074517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117202024-02-12T01:39:37.199Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080782238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117212024-02-12T01:39:37.199Z{"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080819187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117222024-02-12T01:39:37.199Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080856745Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117232024-02-12T01:39:37.199Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080893323Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117242024-02-12T01:39:37.199ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
117252024-02-12T01:39:37.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080929402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117262024-02-12T01:39:37.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.08096358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117272024-02-12T01:39:37.199Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.080981059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117282024-02-12T01:39:37.199Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.081001958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117292024-02-12T01:39:37.199Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.081014937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117302024-02-12T01:39:37.199Z{"msg":"032b1087-259b-4e1d-aab7-2c191e4422a7 is now active with session: f7caa37d-624c-4cfd-ab49-0954102e16a9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.081035536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117312024-02-12T01:39:37.199Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.081048176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559,"session_id":"f7caa37d-624c-4cfd-ab49-0954102e16a9"}
117322024-02-12T01:39:37.199Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:36.081066935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559}
117332024-02-12T01:39:37.199ZLoading write count information from file "/tmp/test_repair_verify.data"
117342024-02-12T01:39:37.199ZRun Repair workload
117352024-02-12T01:39:37.203Z01/30 Write block 83 len 20480 data: 12 13 13 11 14
117362024-02-12T01:39:37.203Z02/30 Write block 551 len 32768 data: 12 10 10 7 5 5 5 5
117372024-02-12T01:39:37.203Z03/30 Read block 121 len 4096
117382024-02-12T01:39:37.203Z04/30 Write block 135 len 8192 data: 5 7
117392024-02-12T01:39:37.203Z05/30 Write block 45 len 16384 data: 7 7 9 10
117402024-02-12T01:39:37.203Z06/30 Read block 416 len 20480
117412024-02-12T01:39:37.203Z07/30 Read block 523 len 28672
117422024-02-12T01:39:37.203Z08/30 Read block 317 len 4096
117432024-02-12T01:39:37.203Z09/30 Write block 133 len 20480 data: 7 6 6 8 8
117442024-02-12T01:39:37.203Z10/30 Flush
117452024-02-12T01:39:37.207Z11/30 Write block 59 len 8192 data: 9 8
117462024-02-12T01:39:37.207Z12/30 Write block 169 len 4096 data: 13
117472024-02-12T01:39:37.207Z13/30 Write block 469 len 24576 data: 9 8 7 9 10 11
117482024-02-12T01:39:37.207Z14/30 Write block 26 len 12288 data: 6 7 9
117492024-02-12T01:39:37.207Z15/30 Read block 11 len 40960
117502024-02-12T01:39:37.210Z16/30 Write block 182 len 40960 data: 8 8 9 6 6 9 11 11 9 8
117512024-02-12T01:39:37.210Z17/30 Write block 393 len 8192 data: 13 11
117522024-02-12T01:39:37.210Z18/30 Read block 428 len 36864
117532024-02-12T01:39:37.212Z19/30 Read block 115 len 32768
117542024-02-12T01:39:37.212Z20/30 Read block 129 len 40960
117552024-02-12T01:39:37.215Z21/30 Read block 170 len 24576
117562024-02-12T01:39:37.215Z22/30 Write block 452 len 28672 data: 6 6 8 7 7 7 9
117572024-02-12T01:39:37.215Z23/30 Write block 516 len 20480 data: 9 9 7 8 9
117582024-02-12T01:39:37.218Z24/30 Write block 374 len 8192 data: 13 14
117592024-02-12T01:39:37.218Z25/30 Read block 209 len 8192
117602024-02-12T01:39:37.218Z26/30 Write block 429 len 12288 data: 4 5 3
117612024-02-12T01:39:37.218Z27/30 Write block 492 len 40960 data: 6 6 7 10 8 7 8 8 6 7
117622024-02-12T01:39:37.221Z28/30 Write block 265 len 32768 data: 8 9 8 7 5 5 6 5
117632024-02-12T01:39:37.221Z29/30 Write block 203 len 40960 data: 8 8 6 9 11 11 9 7 8 7
117642024-02-12T01:39:37.224Z30/30 Write block 236 len 36864 data: 9 12 13 12 12 9 11 12 9
117652024-02-12T01:39:37.225Z----------------------------------------------------------------
117662024-02-12T01:39:37.225Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
117672024-02-12T01:39:37.225ZStates: Active Active Active
117682024-02-12T01:39:37.225ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117692024-02-12T01:39:37.225Z 1 Acked 1000 Write 5 Sent Done Done false
117702024-02-12T01:39:37.225Z 2 Acked 1001 Write 8 Sent Done Done false
117712024-02-12T01:39:37.225Z 3 Acked 1002 Read 1 Sent Done Done false
117722024-02-12T01:39:37.225Z 4 Acked 1003 Write 2 Sent Done Done false
117732024-02-12T01:39:37.225Z 5 Acked 1004 Write 4 Sent Done Done false
117742024-02-12T01:39:37.225Z 6 Acked 1005 Read 5 Sent Done Done false
117752024-02-12T01:39:37.225Z 7 Acked 1006 Read 7 Sent Done Done false
117762024-02-12T01:39:37.225Z 8 Acked 1007 Read 1 Sent Done Done false
117772024-02-12T01:39:37.225Z 9 Acked 1008 Write 5 Sent Done Done false
117782024-02-12T01:39:37.225Z 10 Acked 1009 Flush 0 Sent Done Done false
117792024-02-12T01:39:37.225Z 11 Acked 1010 Write 2 Sent Done Done false
117802024-02-12T01:39:37.225Z 12 Acked 1011 Write 1 Sent Done Done false
117812024-02-12T01:39:37.225Z 13 Acked 1012 Write 6 Sent Done Done false
117822024-02-12T01:39:37.225Z 14 Acked 1013 Write 3 Sent Done Done false
117832024-02-12T01:39:37.225Z 15 Acked 1014 Read 10 Sent Done Done false
117842024-02-12T01:39:37.225Z 16 Acked 1015 Write 10 Sent Done Done false
117852024-02-12T01:39:37.225Z 17 Acked 1016 Write 2 Sent Done Done false
117862024-02-12T01:39:37.225Z 18 Acked 1017 Read 9 Sent Done Done false
117872024-02-12T01:39:37.225Z 19 Acked 1018 Read 8 Sent Done Done false
117882024-02-12T01:39:37.226Z 20 Acked 1019 Read 10 Sent Done Done false
117892024-02-12T01:39:37.226Z 21 Acked 1020 Read 6 Sent Done Done false
117902024-02-12T01:39:37.226Z 22 Acked 1021 Write 7 Sent Done Done false
117912024-02-12T01:39:37.226Z 23 Acked 1022 Write 5 Sent Done Done false
117922024-02-12T01:39:37.226Z 24 Acked 1023 Write 2 Sent Done Done false
117932024-02-12T01:39:37.226Z 25 Acked 1024 Read 2 Sent Done Done false
117942024-02-12T01:39:37.226Z 26 Acked 1025 Write 3 Sent Done Done false
117952024-02-12T01:39:37.226Z 27 Acked 1026 Write 10 Sent Done Done false
117962024-02-12T01:39:37.226Z 28 Acked 1027 Write 8 Sent Done Done false
117972024-02-12T01:39:37.226Z 29 Acked 1028 Write 10 Sent Done Done false
117982024-02-12T01:39:37.226Z 30 Acked 1029 Write 9 Sent Sent Sent false
117992024-02-12T01:39:37.226Z STATES DS:0 DS:1 DS:2 TOTAL
118002024-02-12T01:39:37.226Z New 0 0 0 0
118012024-02-12T01:39:37.226Z Sent 30 1 1 32
118022024-02-12T01:39:37.226Z Done 0 29 29 58
118032024-02-12T01:39:37.226Z Skipped 0 0 0 0
118042024-02-12T01:39:37.226Z Error 0 0 0 0
118052024-02-12T01:39:37.226ZLast Flush: 0 1009 1009
118062024-02-12T01:39:37.226ZDownstairs last five completed:
118072024-02-12T01:39:37.226ZUpstairs last five completed: 30 29 28 27 26
118082024-02-12T01:39:37.226Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:36.109174734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1559}
118092024-02-12T01:39:37.226ZWrote out file "/tmp/test_repair_verify.data"
118102024-02-12T01:39:37.229Z
118112024-02-12T01:39:37.232ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
118122024-02-12T01:39:37.252ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118132024-02-12T01:39:37.252Z 0 000-019 99 99 99 173 173 173 F F F
118142024-02-12T01:39:37.252Z 1 020-039 96 96 96 166 166 166 F T T <---
118152024-02-12T01:39:37.252Z 2 040-059 104 106 106 183 188 188 F T T <---
118162024-02-12T01:39:37.252Z 3 060-079 104 104 104 186 186 186 F T T <---
118172024-02-12T01:39:37.252Z 4 080-099 103 106 106 182 188 188 F F F <---
118182024-02-12T01:39:37.252Z 5 100-119 100 100 100 175 175 175 F F F
118192024-02-12T01:39:37.252Z 6 120-139 104 106 106 185 188 188 F F F <---
118202024-02-12T01:39:37.252Z 7 140-159 100 100 100 176 176 176 F F F
118212024-02-12T01:39:37.252Z 8 160-179 102 102 102 181 181 181 F T T <---
118222024-02-12T01:39:37.252Z 9 180-199 101 101 101 178 178 178 F T T <---
118232024-02-12T01:39:37.252Z 10 200-219 97 97 97 167 167 167 F T T <---
118242024-02-12T01:39:37.253Z 11 220-239 103 103 103 182 182 182 F T T <---
118252024-02-12T01:39:37.253Z 12 240-259 103 103 103 182 182 182 F T T <---
118262024-02-12T01:39:37.256Z 13 260-279 104 104 104 186 186 186 F T T <---
118272024-02-12T01:39:37.256Z 14 280-299 102 102 102 181 181 181 F F F
118282024-02-12T01:39:37.256Z 15 300-319 102 102 102 180 180 180 F F F
118292024-02-12T01:39:37.256Z 16 320-339 105 105 105 187 187 187 F F F
118302024-02-12T01:39:37.256Z 17 340-359 105 105 105 187 187 187 F F F
118312024-02-12T01:39:37.256Z 18 360-379 105 105 105 187 187 187 F T T <---
118322024-02-12T01:39:37.256Z 19 380-399 102 102 102 181 181 181 F T T <---
118332024-02-12T01:39:37.256Z 20 400-419 105 105 105 187 187 187 F F F
118342024-02-12T01:39:37.256Z 21 420-439 104 104 104 186 186 186 F T T <---
118352024-02-12T01:39:37.256Z 22 440-459 104 104 104 186 186 186 F T T <---
118362024-02-12T01:39:37.256Z 23 460-479 104 104 104 185 185 185 F T T <---
118372024-02-12T01:39:37.256Z 24 480-499 102 102 102 179 179 179 F T T <---
118382024-02-12T01:39:37.257Z 25 500-519 98 98 98 170 170 170 F T T <---
118392024-02-12T01:39:37.257Z 26 520-539 98 98 98 170 170 170 F T T <---
118402024-02-12T01:39:37.257Z 27 540-559 98 106 106 170 188 188 F F F <---
118412024-02-12T01:39:37.257Z 28 560-579 104 104 104 184 184 184 F F F
118422024-02-12T01:39:37.257Z 29 580-599 96 96 96 166 166 166 F F F
118432024-02-12T01:39:37.257ZMax gen: 106, Max flush: 188
118442024-02-12T01:39:37.257ZError: Difference in extent metadata found!
118452024-02-12T01:39:37.257ZA Difference in extent metadata is expected here
118462024-02-12T01:39:37.257ZOn loop 53
118472024-02-12T01:39:37.257Z
118482024-02-12T01:39:37.260ZVerifying data now
118492024-02-12T01:39:42.169ZNo change after verify
118502024-02-12T01:39:42.172ZLoop: 53 Downstairs dump after verify (and repair):
118512024-02-12T01:39:42.183ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
118522024-02-12T01:39:42.183Z 0 000-019 99 99 99 173 173 173 F F F
118532024-02-12T01:39:42.183Z 1 020-039 107 107 107 189 189 189 F F F
118542024-02-12T01:39:42.183Z 2 040-059 107 107 107 189 189 189 F F F
118552024-02-12T01:39:42.184Z 3 060-079 107 107 107 189 189 189 F F F
118562024-02-12T01:39:42.184Z 4 080-099 106 106 106 188 188 188 F F F
118572024-02-12T01:39:42.184Z 5 100-119 100 100 100 175 175 175 F F F
118582024-02-12T01:39:42.184Z 6 120-139 106 106 106 188 188 188 F F F
118592024-02-12T01:39:42.184Z 7 140-159 100 100 100 176 176 176 F F F
118602024-02-12T01:39:42.184Z 8 160-179 107 107 107 189 189 189 F F F
118612024-02-12T01:39:42.187Z 9 180-199 107 107 107 189 189 189 F F F
118622024-02-12T01:39:42.187Z 10 200-219 107 107 107 189 189 189 F F F
118632024-02-12T01:39:42.187Z 11 220-239 107 107 107 189 189 189 F F F
118642024-02-12T01:39:42.187Z 12 240-259 107 107 107 189 189 189 F F F
118652024-02-12T01:39:42.187Z 13 260-279 107 107 107 189 189 189 F F F
118662024-02-12T01:39:42.187Z 14 280-299 102 102 102 181 181 181 F F F
118672024-02-12T01:39:42.187Z 15 300-319 102 102 102 180 180 180 F F F
118682024-02-12T01:39:42.187Z 16 320-339 105 105 105 187 187 187 F F F
118692024-02-12T01:39:42.187Z 17 340-359 105 105 105 187 187 187 F F F
118702024-02-12T01:39:42.187Z 18 360-379 107 107 107 189 189 189 F F F
118712024-02-12T01:39:42.187Z 19 380-399 107 107 107 189 189 189 F F F
118722024-02-12T01:39:42.187Z 20 400-419 105 105 105 187 187 187 F F F
118732024-02-12T01:39:42.187Z 21 420-439 107 107 107 189 189 189 F F F
118742024-02-12T01:39:42.187Z 22 440-459 107 107 107 189 189 189 F F F
118752024-02-12T01:39:42.187Z 23 460-479 107 107 107 189 189 189 F F F
118762024-02-12T01:39:42.187Z 24 480-499 107 107 107 189 189 189 F F F
118772024-02-12T01:39:42.187Z 25 500-519 107 107 107 189 189 189 F F F
118782024-02-12T01:39:42.188Z 26 520-539 107 107 107 189 189 189 F F F
118792024-02-12T01:39:42.188Z 27 540-559 106 106 106 188 188 188 F F F
118802024-02-12T01:39:42.188Z 28 560-579 104 104 104 184 184 184 F F F
118812024-02-12T01:39:42.188Z 29 580-599 96 96 96 166 166 166 F F F
118822024-02-12T01:39:42.188ZMax gen: 107, Max flush: 189
118832024-02-12T01:39:42.188Z
118842024-02-12T01:39:42.188ZBegin loop 54
118852024-02-12T01:39:42.188ZDownstairs to restart: 2
118862024-02-12T01:39:42.207Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093330344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
118872024-02-12T01:39:42.207Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093604461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
118882024-02-12T01:39:42.207Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.09362636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
118892024-02-12T01:39:42.207Z{"msg":"Crucible 86edddb8-e957-4d06-811d-ecf00ec53a0d has session id: 75806ceb-3d10-4176-a9c3-8321c283db7d","v":0,"name":"crucible","level":30,"timeCrucible runtime is spawned
118902024-02-12T01:39:42.207Z":"2024-02-12T01:39:41.093653029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118912024-02-12T01:39:42.211Z{"msg":"Upstairs opts: Upstairs UUID: 86edddb8-e957-4d06-811d-ecf00ec53a0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093709706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118922024-02-12T01:39:42.211Z{"msg":"Crucible stats registered with UUID: 86edddb8-e957-4d06-811d-ecf00ec53a0d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093765633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118932024-02-12T01:39:42.211Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093805381Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
118942024-02-12T01:39:42.212Z{"msg":"86edddb8-e957-4d06-811d-ecf00ec53a0d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093959254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118952024-02-12T01:39:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.093989993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118962024-02-12T01:39:42.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094008892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118972024-02-12T01:39:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094027291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118982024-02-12T01:39:42.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094064819Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
118992024-02-12T01:39:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094090388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119002024-02-12T01:39:42.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094129466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119012024-02-12T01:39:42.212Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094148765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119022024-02-12T01:39:42.212Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094186473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119032024-02-12T01:39:42.212Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094223551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119042024-02-12T01:39:42.212Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094328806Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119052024-02-12T01:39:42.212Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:41.094507908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119062024-02-12T01:39:42.212Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:41.094575494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119072024-02-12T01:39:42.212Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:41.094618323Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119082024-02-12T01:39:42.212Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.09466215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119092024-02-12T01:39:42.212Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094688899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119102024-02-12T01:39:42.212Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094711428Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119112024-02-12T01:39:42.212Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.094756596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119122024-02-12T01:39:42.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.095185565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119132024-02-12T01:39:42.212Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.095227493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119142024-02-12T01:39:42.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.095256292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119152024-02-12T01:39:42.213Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.09530402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119162024-02-12T01:39:42.213Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.095682932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119172024-02-12T01:39:42.213Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.095811215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119182024-02-12T01:39:42.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.09614402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119192024-02-12T01:39:42.213Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.096193367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119202024-02-12T01:39:42.213Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.096212216Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119212024-02-12T01:39:42.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.09633563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119222024-02-12T01:39:42.213Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.096368639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119232024-02-12T01:39:42.213Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:41.096393328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119242024-02-12T01:39:43.210Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.096103912Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119252024-02-12T01:39:43.210Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.096182549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119262024-02-12T01:39:43.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.097051987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119272024-02-12T01:39:43.215Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.097088855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119282024-02-12T01:39:43.215Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.097603241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119292024-02-12T01:39:43.215Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098158594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119302024-02-12T01:39:43.215Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098207182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119312024-02-12T01:39:43.215Z{"msg":"[0]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 175, 188, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.09825318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119322024-02-12T01:39:43.215Z{"msg":"[0]R generation[0..12]: [99, 107, 107, 107, 106, 100, 106, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098290148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119332024-02-12T01:39:43.215Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098324446Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119342024-02-12T01:39:43.215Z{"msg":"[1]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 175, 188, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098357795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119352024-02-12T01:39:43.215Z{"msg":"[1]R generation[0..12]: [99, 107, 107, 107, 106, 100, 106, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
119362024-02-12T01:39:43.215Z,"time":"2024-02-12T01:39:42.098384303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119372024-02-12T01:39:43.215Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098423512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119382024-02-12T01:39:43.215Z{"msg":"[2]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 175, 188, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.09845677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119392024-02-12T01:39:43.215Z{"msg":"[2]R generation[0..12]: [99, 107, 107, 107, 106, 100, 106, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098477039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119402024-02-12T01:39:43.215Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098503938Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119412024-02-12T01:39:43.215Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098521237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119422024-02-12T01:39:43.215Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098541536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119432024-02-12T01:39:43.215Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098554646Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119442024-02-12T01:39:43.215Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098574225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119452024-02-12T01:39:43.215Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098587934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119462024-02-12T01:39:43.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098606503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119472024-02-12T01:39:43.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098620562Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119482024-02-12T01:39:43.215Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098640111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119492024-02-12T01:39:43.215Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098653701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119502024-02-12T01:39:43.216Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.09867231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119512024-02-12T01:39:43.216Z{"msg":"86edddb8-e957-4d06-811d-ecf00ec53a0d is now active with session: 75806ceb-3d10-4176-a9c3-8321c283db7d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098685469Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119522024-02-12T01:39:43.216Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098703818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567,"session_id":"75806ceb-3d10-4176-a9c3-8321c283db7d"}
119532024-02-12T01:39:43.216Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:42.098716418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
119542024-02-12T01:39:43.216ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119552024-02-12T01:39:43.216ZLoading write count information from file "/tmp/test_repair_verify.data"
119562024-02-12T01:39:43.216ZRun Repair workload
119572024-02-12T01:39:43.219Z01/30 Write block 550 len 24576 data: 13 13 11 11 8 6
119582024-02-12T01:39:43.219Z02/30 Write block 124 len 16384 data: 13 12 14 13
119592024-02-12T01:39:43.219Z03/30 Write block 563 len 24576 data: 8 7 6 9 10 12
119602024-02-12T01:39:43.219Z04/30 Write block 473 len 20480 data: 11 12 9 6 7
119612024-02-12T01:39:43.219Z05/30 Flush
119622024-02-12T01:39:43.222Z06/30 Flush
119632024-02-12T01:39:43.222Z07/30 Write block 375 len 28672 data: 15 12 11 10 11 9 10
119642024-02-12T01:39:43.222Z08/30 Read block 575 len 24576
119652024-02-12T01:39:43.225Z09/30 Write block 434 len 16384 data: 5 5 6 7
119662024-02-12T01:39:43.225Z10/30 Write block 114 len 40960 data: 10 8 9 11 12 11 11 10 9 12
119672024-02-12T01:39:43.225Z11/30 Flush
119682024-02-12T01:39:43.228Z12/30 Write block 264 len 40960 data: 5 9 10 9 8 6 6 7 6 7
119692024-02-12T01:39:43.231Z13/30 Write block 582 len 20480 data: 8 7 8 8 8
119702024-02-12T01:39:43.231Z14/30 Write block 324 len 24576 data: 7 6 7 7 7 6
119712024-02-12T01:39:43.231Z15/30 Read block 304 len 16384
119722024-02-12T01:39:43.233Z16/30 Write block 386 len 36864 data: 12 11 11 10 14 14 13 14 12
119732024-02-12T01:39:43.233Z17/30 Read block 519 len 24576
119742024-02-12T01:39:43.234Z18/30 Read block 27 len 36864
119752024-02-12T01:39:43.236Z19/30 Read block 112 len 12288
119762024-02-12T01:39:43.237Z20/30 Read block 66 len 32768
119772024-02-12T01:39:43.237Z21/30 Read block 533 len 8192
119782024-02-12T01:39:43.237Z22/30 Read block 518 len 20480
119792024-02-12T01:39:43.239Z23/30 Read block 457 len 20480
119802024-02-12T01:39:43.240Z24/30 Write block 561 len 20480 data: 8 7 9 8 7
119812024-02-12T01:39:43.240Z25/30 Write block 478 len 40960 data: 7 11 11 9 9 10 10 10 11 11
119822024-02-12T01:39:43.242Z26/30 Read block 13 len 28672
119832024-02-12T01:39:43.242Z27/30 Write block 278 len 20480 data: 11 12 11 9 12
119842024-02-12T01:39:43.242Z28/30 Read block 372 len 36864
119852024-02-12T01:39:43.246Z29/30 Write block 494 len 40960 data: 8 11 9 8 9 9 7 8 9 7
119862024-02-12T01:39:43.246Z30/30 Write block 309 len 12288 data: 10 10 8
119872024-02-12T01:39:43.246Z----------------------------------------------------------------
119882024-02-12T01:39:43.246Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
119892024-02-12T01:39:43.246ZStates: Active Active Active
119902024-02-12T01:39:43.246ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119912024-02-12T01:39:43.246Z 1 Acked 1000 Write 6 Done Done Done false
119922024-02-12T01:39:43.246Z 2 Acked 1001 Write 4 Done Done Done false
119932024-02-12T01:39:43.246Z 3 Acked 1002 Write 6 Done Done Sent false
119942024-02-12T01:39:43.246Z 4 Acked 1003 Write 5 Done Done Sent false
119952024-02-12T01:39:43.246Z 5 Acked 1004 Flush 0 Done Done Sent false
119962024-02-12T01:39:43.246Z 6 Acked 1005 Flush 0 Done Done Sent false
119972024-02-12T01:39:43.246Z 7 Acked 1006 Write 7 Done Done Sent false
119982024-02-12T01:39:43.246Z 8 Acked 1007 Read 6 Done Done Sent false
119992024-02-12T01:39:43.246Z 9 Acked 1008 Write 4 Done Done Sent false
120002024-02-12T01:39:43.246Z 10 Acked 1009 Write 10 Done Done Sent false
120012024-02-12T01:39:43.246Z 11 Acked 1010 Flush 0 Done Done Sent false
120022024-02-12T01:39:43.246Z 12 Acked 1011 Write 10 Done Done Sent false
120032024-02-12T01:39:43.247Z 13 Acked 1012 Write 5 Done Done Sent false
120042024-02-12T01:39:43.247Z 14 Acked 1013 Write 6 Done Done Sent false
120052024-02-12T01:39:43.247Z 15 Acked 1014 Read 4 Done Done Sent false
120062024-02-12T01:39:43.247Z 16 Acked 1015 Write 9 Done Done Sent false
120072024-02-12T01:39:43.247Z 17 Acked 1016 Read 6 Done Done Sent false
120082024-02-12T01:39:43.247Z 18 Acked 1017 Read 9 Done Done Sent false
120092024-02-12T01:39:43.247Z 19 Acked 1018 Read 3 Done Done Sent false
120102024-02-12T01:39:43.247Z 20 Acked 1019 Read 8 Done Done Sent false
120112024-02-12T01:39:43.247Z 21 Acked 1020 Read 2 Done Done Sent false
120122024-02-12T01:39:43.247Z 22 Acked 1021 Read 5 Done Done Sent false
120132024-02-12T01:39:43.247Z 23 Acked 1022 Read 5 Done Done Sent false
120142024-02-12T01:39:43.247Z 24 Acked 1023 Write 5 Done Done Sent false
120152024-02-12T01:39:43.247Z 25 Acked 1024 Write 10 Done Done Sent false
120162024-02-12T01:39:43.247Z 26 Acked 1025 Read 7 Done Done Sent false
120172024-02-12T01:39:43.247Z 27 Acked 1026 Write 5 Done Done Sent false
120182024-02-12T01:39:43.247Z 28 Acked 1027 Read 9 Done Done Sent false
120192024-02-12T01:39:43.247Z 29 Acked 1028 Write 10 Sent Sent Sent false
120202024-02-12T01:39:43.247Z 30 Acked 1029 Write 3 Sent Sent Sent false
120212024-02-12T01:39:43.247Z STATES DS:0 DS:1 DS:2 TOTAL
120222024-02-12T01:39:43.247Z New 0 0 0 0
120232024-02-12T01:39:43.247Z Sent 2 2 28 32
120242024-02-12T01:39:43.247Z Done 28 28 2 58
120252024-02-12T01:39:43.247Z Skipped 0 0 0 0
120262024-02-12T01:39:43.247Z Error 0 0 0 0
120272024-02-12T01:39:43.248ZLast Flush: 1010 1010 0
120282024-02-12T01:39:43.248ZDownstairs last five completed:
120292024-02-12T01:39:43.248ZUpstairs last five completed: 30 29 28 27 26
120302024-02-12T01:39:43.248Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:42.131452116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1567}
120312024-02-12T01:39:43.248ZWrote out file "/tmp/test_repair_verify.data"
120322024-02-12T01:39:43.251Z
120332024-02-12T01:39:43.253ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
120342024-02-12T01:39:43.275ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120352024-02-12T01:39:43.275Z 0 000-019 99 99 99 173 173 173 F F F
120362024-02-12T01:39:43.275Z 1 020-039 107 107 107 189 189 189 F F F
120372024-02-12T01:39:43.275Z 2 040-059 107 107 107 189 189 189 F F F
120382024-02-12T01:39:43.275Z 3 060-079 107 107 107 189 189 189 F F F
120392024-02-12T01:39:43.276Z 4 080-099 106 106 106 188 188 188 F F F
120402024-02-12T01:39:43.276Z 5 100-119 108 108 100 192 192 175 F F F <---
120412024-02-12T01:39:43.276Z 6 120-139 108 108 106 192 192 188 F F T <---
120422024-02-12T01:39:43.276Z 7 140-159 100 100 100 176 176 176 F F F
120432024-02-12T01:39:43.276Z 8 160-179 107 107 107 189 189 189 F F F
120442024-02-12T01:39:43.276Z 9 180-199 107 107 107 189 189 189 F F F
120452024-02-12T01:39:43.276Z 10 200-219 107 107 107 189 189 189 F F F
120462024-02-12T01:39:43.276Z 11 220-239 107 107 107 189 189 189 F F F
120472024-02-12T01:39:43.276Z 12 240-259 107 107 107 189 189 189 F F F
120482024-02-12T01:39:43.276Z 13 260-279 107 107 107 189 189 189 T T F <---
120492024-02-12T01:39:43.279Z 14 280-299 102 102 102 181 181 181 T T F <---
120502024-02-12T01:39:43.279Z 15 300-319 102 102 102 180 180 180 T T F <---
120512024-02-12T01:39:43.279Z 16 320-339 105 105 105 187 187 187 T T F <---
120522024-02-12T01:39:43.279Z 17 340-359 105 105 105 187 187 187 F F F
120532024-02-12T01:39:43.279Z 18 360-379 108 108 107 192 192 189 F F F <---
120542024-02-12T01:39:43.279Z 19 380-399 108 108 107 192 192 189 T T F <---
120552024-02-12T01:39:43.279Z 20 400-419 105 105 105 187 187 187 F F F
120562024-02-12T01:39:43.280Z 21 420-439 108 108 107 192 192 189 F F F <---
120572024-02-12T01:39:43.280Z 22 440-459 107 107 107 189 189 189 F F F
120582024-02-12T01:39:43.280Z 23 460-479 108 108 107 190 190 189 T T F <---
120592024-02-12T01:39:43.280Z 24 480-499 107 107 107 189 189 189 T T F <---
120602024-02-12T01:39:43.280Z 25 500-519 107 107 107 189 189 189 T T F <---
120612024-02-12T01:39:43.280Z 26 520-539 107 107 107 189 189 189 F F F
120622024-02-12T01:39:43.280Z 27 540-559 108 108 106 190 190 188 F F T <---
120632024-02-12T01:39:43.280Z 28 560-579 108 108 104 190 190 184 T T F <---
120642024-02-12T01:39:43.280Z 29 580-599 96 96 96 166 166 166 T T F <---
120652024-02-12T01:39:43.280ZMax gen: 108, Max flush: 192
120662024-02-12T01:39:43.280ZError: Difference in extent metadata found!
120672024-02-12T01:39:43.280ZA Difference in extent metadata is expected here
120682024-02-12T01:39:43.280ZOn loop 54
120692024-02-12T01:39:43.280Z
120702024-02-12T01:39:43.280ZVerifying data now
120712024-02-12T01:39:49.026ZNo change after verify
120722024-02-12T01:39:49.026ZLoop: 54 Downstairs dump after verify (and repair):
120732024-02-12T01:39:49.040ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120742024-02-12T01:39:49.040Z 0 000-019 99 99 99 173 173 173 F F F
120752024-02-12T01:39:49.041Z 1 020-039 107 107 107 189 189 189 F F F
120762024-02-12T01:39:49.041Z 2 040-059 107 107 107 189 189 189 F F F
120772024-02-12T01:39:49.044Z 3 060-079 107 107 107 189 189 189 F F F
120782024-02-12T01:39:49.044Z 4 080-099 106 106 106 188 188 188 F F F
120792024-02-12T01:39:49.044Z 5 100-119 108 108 108 192 192 192 F F F
120802024-02-12T01:39:49.044Z 6 120-139 108 108 108 192 192 192 F F F
120812024-02-12T01:39:49.044Z 7 140-159 100 100 100 176 176 176 F F F
120822024-02-12T01:39:49.044Z 8 160-179 107 107 107 189 189 189 F F F
120832024-02-12T01:39:49.044Z 9 180-199 107 107 107 189 189 189 F F F
120842024-02-12T01:39:49.044Z 10 200-219 107 107 107 189 189 189 F F F
120852024-02-12T01:39:49.044Z 11 220-239 107 107 107 189 189 189 F F F
120862024-02-12T01:39:49.044Z 12 240-259 107 107 107 189 189 189 F F F
120872024-02-12T01:39:49.044Z 13 260-279 109 109 109 193 193 193 F F F
120882024-02-12T01:39:49.044Z 14 280-299 109 109 109 193 193 193 F F F
120892024-02-12T01:39:49.044Z 15 300-319 109 109 109 193 193 193 F F F
120902024-02-12T01:39:49.044Z 16 320-339 109 109 109 193 193 193 F F F
120912024-02-12T01:39:49.044Z 17 340-359 105 105 105 187 187 187 F F F
120922024-02-12T01:39:49.044Z 18 360-379 108 108 108 192 192 192 F F F
120932024-02-12T01:39:49.044Z 19 380-399 109 109 109 193 193 193 F F F
120942024-02-12T01:39:49.045Z 20 400-419 105 105 105 187 187 187 F F F
120952024-02-12T01:39:49.045Z 21 420-439 108 108 108 192 192 192 F F F
120962024-02-12T01:39:49.045Z 22 440-459 107 107 107 189 189 189 F F F
120972024-02-12T01:39:49.045Z 23 460-479 109 109 109 193 193 193 F F F
120982024-02-12T01:39:49.045Z 24 480-499 109 109 109 193 193 193 F F F
120992024-02-12T01:39:49.045Z 25 500-519 109 109 109 193 193 193 F F F
121002024-02-12T01:39:49.045Z 26 520-539 107 107 107 189 189 189 F F F
121012024-02-12T01:39:49.045Z 27 540-559 108 108 108 190 190 190 F F F
121022024-02-12T01:39:49.045Z 28 560-579 109 109 109 193 193 193 F F F
121032024-02-12T01:39:49.045Z 29 580-599 109 109 109 193 193 193 F F F
121042024-02-12T01:39:49.045ZMax gen: 109, Max flush: 193
121052024-02-12T01:39:49.045Z
121062024-02-12T01:39:49.045ZBegin loop 55
121072024-02-12T01:39:49.045ZDownstairs to restart: 1
121082024-02-12T01:39:49.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948206844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
121092024-02-12T01:39:49.063ZCrucible runtime is spawned
121102024-02-12T01:39:49.067Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948529729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
121112024-02-12T01:39:49.067Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948565797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
121122024-02-12T01:39:49.067Z{"msg":"Crucible 4cf45b9a-1574-4ec4-8ae6-7844f65520a6 has session id: 4deada9b-6040-4150-9ec2-1fefe9d0abac","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948595796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121132024-02-12T01:39:49.067Z{"msg":"Upstairs opts: Upstairs UUID: 4cf45b9a-1574-4ec4-8ae6-7844f65520a6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948643084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121142024-02-12T01:39:49.067Z{"msg":"Crucible stats registered with UUID: 4cf45b9a-1574-4ec4-8ae6-7844f65520a6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948663403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121152024-02-12T01:39:49.067Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948687542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
121162024-02-12T01:39:49.067Z{"msg":"4cf45b9a-1574-4ec4-8ae6-7844f65520a6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948793626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121172024-02-12T01:39:49.067Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948826555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121182024-02-12T01:39:49.067Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948855414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121192024-02-12T01:39:49.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948888792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121202024-02-12T01:39:49.068Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.94892209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121212024-02-12T01:39:49.068Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.948969338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121222024-02-12T01:39:49.068Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949006526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121232024-02-12T01:39:49.068Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949048274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121242024-02-12T01:39:49.068Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949089412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121252024-02-12T01:39:49.068Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.94913371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121262024-02-12T01:39:49.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949167699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121272024-02-12T01:39:49.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.94934339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121282024-02-12T01:39:49.068Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949372309Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121292024-02-12T01:39:49.068Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949861926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121302024-02-12T01:39:49.068Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949896734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121312024-02-12T01:39:49.068Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949913043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121322024-02-12T01:39:49.068Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.949926842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121332024-02-12T01:39:49.068Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.950455167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121342024-02-12T01:39:49.068Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.950626049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121352024-02-12T01:39:49.068Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.950945704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121362024-02-12T01:39:49.068Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.950981702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121372024-02-12T01:39:49.068Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.950997321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121382024-02-12T01:39:49.068Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.95103072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121392024-02-12T01:39:49.068Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951055809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121402024-02-12T01:39:49.068Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951159353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121412024-02-12T01:39:49.068Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951188052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121422024-02-12T01:39:49.068Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951204721Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121432024-02-12T01:39:49.073Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951509917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121442024-02-12T01:39:49.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951912058Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121452024-02-12T01:39:49.073Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951955846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121462024-02-12T01:39:49.073Z{"msg":"[0]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 192, 192, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.951977185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121472024-02-12T01:39:49.073Z{"msg":"[0]R generation[0..12]: [99, 107, 107, 107, 106, 108, 108, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952009723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121482024-02-12T01:39:49.073Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952025932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121492024-02-12T01:39:49.073Z{"msg":"[1]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 192, 192, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952047931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121502024-02-12T01:39:49.073Z{"msg":"[1]R generation[0..12]: [99, 107, 107, 107, 106, 108, 108, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.95206207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121512024-02-12T01:39:49.073Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.95208268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121522024-02-12T01:39:49.073Z{"msg":"[2]R flush_numbers[0..12]: [173, 189, 189, 189, 188, 192, 192, 176, 189, 189, 189, 189]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
121532024-02-12T01:39:49.073Z,"time":"2024-02-12T01:39:47.952096999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121542024-02-12T01:39:49.073Z{"msg":"[2]R generation[0..12]: [99, 107, 107, 107, 106, 108, 108, 100, 107, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952119138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121552024-02-12T01:39:49.073Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952133727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121562024-02-12T01:39:49.073Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952153466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121572024-02-12T01:39:49.073Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952167016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121582024-02-12T01:39:49.073Z{"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952186295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121592024-02-12T01:39:49.073Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952199854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121602024-02-12T01:39:49.073Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952213943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121612024-02-12T01:39:49.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952232392Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121622024-02-12T01:39:49.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952245862Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121632024-02-12T01:39:49.073Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952261171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121642024-02-12T01:39:49.074Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.95228126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121652024-02-12T01:39:49.074Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952305619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121662024-02-12T01:39:49.074Z{"msg":"4cf45b9a-1574-4ec4-8ae6-7844f65520a6 is now active with session: 4deada9b-6040-4150-9ec2-1fefe9d0abac","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952326878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121672024-02-12T01:39:49.074Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952345497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575,"session_id":"4deada9b-6040-4150-9ec2-1fefe9d0abac"}
121682024-02-12T01:39:49.074Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:47.952357956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
121692024-02-12T01:39:49.074ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121702024-02-12T01:39:49.074ZLoading write count information from file "/tmp/test_repair_verify.data"
121712024-02-12T01:39:49.074ZRun Repair workload
121722024-02-12T01:39:49.074Z01/30 Write block 13 len 4096 data: 9
121732024-02-12T01:39:49.074Z02/30 Write block 419 len 36864 data: 5 5 7 6 5 8 7 6 4
121742024-02-12T01:39:49.074Z03/30 Write block 367 len 4096 data: 13
121752024-02-12T01:39:49.074Z04/30 Read block 59 len 12288
121762024-02-12T01:39:49.074Z05/30 Read block 567 len 32768
121772024-02-12T01:39:49.077Z06/30 Read block 550 len 40960
121782024-02-12T01:39:49.077Z07/30 Read block 132 len 32768
121792024-02-12T01:39:49.077Z08/30 Write block 5 len 4096 data: 7
121802024-02-12T01:39:49.077Z09/30 Read block 202 len 24576
121812024-02-12T01:39:49.077Z10/30 Read block 487 len 4096
121822024-02-12T01:39:49.077Z11/30 Read block 441 len 40960
121832024-02-12T01:39:49.077Z12/30 Read block 385 len 8192
121842024-02-12T01:39:49.080Z13/30 Write block 466 len 4096 data: 8
121852024-02-12T01:39:49.080Z14/30 Write block 103 len 28672 data: 6 9 10 9 9 9 8
121862024-02-12T01:39:49.081Z15/30 Read block 217 len 4096
121872024-02-12T01:39:49.081Z16/30 Write block 563 len 12288 data: 10 9 8
121882024-02-12T01:39:49.081Z17/30 Read block 472 len 36864
121892024-02-12T01:39:49.083Z18/30 Write block 541 len 40960 data: 7 7 5 6 7 10 10 10 10 14
121902024-02-12T01:39:49.083Z19/30 Read block 139 len 4096
121912024-02-12T01:39:49.084Z20/30 Read block 197 len 40960
121922024-02-12T01:39:49.086Z21/30 Write block 473 len 4096 data: 12
121932024-02-12T01:39:49.086Z22/30 Read block 515 len 8192
121942024-02-12T01:39:49.086Z23/30 Read block 384 len 24576
121952024-02-12T01:39:49.087Z24/30 Read block 225 len 28672
121962024-02-12T01:39:49.089Z25/30 Write block 43 len 24576 data: 10 10 8 8 10 11
121972024-02-12T01:39:49.089Z26/30 Write block 540 len 12288 data: 10 8 8
121982024-02-12T01:39:49.089Z27/30 Write block 171 len 32768 data: 11 8 9 8 11 12 11 12
121992024-02-12T01:39:49.090Z28/30 Read block 295 len 36864
122002024-02-12T01:39:49.095Z29/30 Read block 85 len 28672
122012024-02-12T01:39:49.096Z30/30 Read block 108 len 12288
122022024-02-12T01:39:49.096Z----------------------------------------------------------------
122032024-02-12T01:39:49.096Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
122042024-02-12T01:39:49.096ZStates: Active Active Active
122052024-02-12T01:39:49.096ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122062024-02-12T01:39:49.096Z 1 Acked 1000 Write 1 Done Done Done false
122072024-02-12T01:39:49.096Z 2 Acked 1001 Write 9 Done Done Done false
122082024-02-12T01:39:49.096Z 3 Acked 1002 Write 1 Done Done Done false
122092024-02-12T01:39:49.096Z 4 Acked 1003 Read 3 Done Done Done false
122102024-02-12T01:39:49.096Z 5 Acked 1004 Read 8 Done Done Done false
122112024-02-12T01:39:49.096Z 6 Acked 1005 Read 10 Done Done Done false
122122024-02-12T01:39:49.096Z 7 Acked 1006 Read 8 Done Done Done false
122132024-02-12T01:39:49.096Z 8 Acked 1007 Write 1 Done Sent Done false
122142024-02-12T01:39:49.096Z 9 Acked 1008 Read 6 Done Sent Done false
122152024-02-12T01:39:49.096Z 10 Acked 1009 Read 1 Done Sent Done false
122162024-02-12T01:39:49.096Z 11 Acked 1010 Read 10 Done Sent Done false
122172024-02-12T01:39:49.096Z 12 Acked 1011 Read 2 Done Sent Done false
122182024-02-12T01:39:49.096Z 13 Acked 1012 Write 1 Done Sent Done false
122192024-02-12T01:39:49.096Z 14 Acked 1013 Write 7 Done Sent Done false
122202024-02-12T01:39:49.096Z 15 Acked 1014 Read 1 Done Sent Done false
122212024-02-12T01:39:49.096Z 16 Acked 1015 Write 3 Done Sent Done false
122222024-02-12T01:39:49.096Z 17 Acked 1016 Read 9 Done Sent Done false
122232024-02-12T01:39:49.096Z 18 Acked 1017 Write 10 Done Sent Done false
122242024-02-12T01:39:49.097Z 19 Acked 1018 Read 1 Done Sent Done false
122252024-02-12T01:39:49.097Z 20 Acked 1019 Read 10 Done Sent Done false
122262024-02-12T01:39:49.097Z 21 Acked 1020 Write 1 Done Sent Done false
122272024-02-12T01:39:49.097Z 22 Acked 1021 Read 2 Done Sent Done false
122282024-02-12T01:39:49.097Z 23 Acked 1022 Read 6 Done Sent Done false
122292024-02-12T01:39:49.097Z 24 Acked 1023 Read 7 Done Sent Done false
122302024-02-12T01:39:49.097Z 25 Acked 1024 Write 6 Done Sent Done false
122312024-02-12T01:39:49.097Z 26 Acked 1025 Write 3 Done Sent Done false
122322024-02-12T01:39:49.097Z 27 Acked 1026 Write 8 Done Sent Done false
122332024-02-12T01:39:49.097Z 28 Acked 1027 Read 9 Done Sent Done false
122342024-02-12T01:39:49.097Z 29 Acked 1028 Read 7 Done Sent Done false
122352024-02-12T01:39:49.097Z 30 Acked 1029 Read 3 Done Sent Done false
122362024-02-12T01:39:49.097Z STATES DS:0 DS:1 DS:2 TOTAL
122372024-02-12T01:39:49.097Z New 0 0 0 0
122382024-02-12T01:39:49.097Z Sent 0 23 0 23
122392024-02-12T01:39:49.097Z Done 30 7 30 67
122402024-02-12T01:39:49.097Z Skipped 0 0 0 0
122412024-02-12T01:39:49.097Z Error 0 0 0 0
122422024-02-12T01:39:49.097ZLast Flush: 0 0 0
122432024-02-12T01:39:49.097ZDownstairs last five completed:
122442024-02-12T01:39:49.097ZUpstairs last five completed: 30 29 28 27 26
122452024-02-12T01:39:49.097Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:47.978158676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1575}
122462024-02-12T01:39:49.097ZWrote out file "/tmp/test_repair_verify.data"
122472024-02-12T01:39:49.101Z
122482024-02-12T01:39:49.101ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
122492024-02-12T01:39:49.119ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122502024-02-12T01:39:49.119Z 0 000-019 99 99 99 173 173 173 T T T
122512024-02-12T01:39:49.119Z 1 020-039 107 107 107 189 189 189 F F F
122522024-02-12T01:39:49.119Z 2 040-059 107 107 107 189 189 189 T F T <---
122532024-02-12T01:39:49.119Z 3 060-079 107 107 107 189 189 189 F F F
122542024-02-12T01:39:49.119Z 4 080-099 106 106 106 188 188 188 F F F
122552024-02-12T01:39:49.119Z 5 100-119 108 108 108 192 192 192 T F T <---
122562024-02-12T01:39:49.119Z 6 120-139 108 108 108 192 192 192 F F F
122572024-02-12T01:39:49.119Z 7 140-159 100 100 100 176 176 176 F F F
122582024-02-12T01:39:49.119Z 8 160-179 107 107 107 189 189 189 T F T <---
122592024-02-12T01:39:49.119Z 9 180-199 107 107 107 189 189 189 F F F
122602024-02-12T01:39:49.119Z 10 200-219 107 107 107 189 189 189 F F F
122612024-02-12T01:39:49.119Z 11 220-239 107 107 107 189 189 189 F F F
122622024-02-12T01:39:49.119Z 12 240-259 107 107 107 189 189 189 F F F
122632024-02-12T01:39:49.119Z 13 260-279 109 109 109 193 193 193 F F F
122642024-02-12T01:39:49.119Z 14 280-299 109 109 109 193 193 193 F F F
122652024-02-12T01:39:49.119Z 15 300-319 109 109 109 193 193 193 F F F
122662024-02-12T01:39:49.119Z 16 320-339 109 109 109 193 193 193 F F F
122672024-02-12T01:39:49.119Z 17 340-359 105 105 105 187 187 187 F F F
122682024-02-12T01:39:49.123Z 18 360-379 108 108 108 192 192 192 T T T
122692024-02-12T01:39:49.123Z 19 380-399 109 109 109 193 193 193 F F F
122702024-02-12T01:39:49.123Z 20 400-419 105 105 105 187 187 187 T T T
122712024-02-12T01:39:49.123Z 21 420-439 108 108 108 192 192 192 T T T
122722024-02-12T01:39:49.123Z 22 440-459 107 107 107 189 189 189 F F F
122732024-02-12T01:39:49.123Z 23 460-479 109 109 109 193 193 193 T F T <---
122742024-02-12T01:39:49.123Z 24 480-499 109 109 109 193 193 193 F F F
122752024-02-12T01:39:49.123Z 25 500-519 109 109 109 193 193 193 F F F
122762024-02-12T01:39:49.123Z 26 520-539 107 107 107 189 189 189 F F F
122772024-02-12T01:39:49.123Z 27 540-559 108 108 108 190 190 190 T F T <---
122782024-02-12T01:39:49.123Z 28 560-579 109 109 109 193 193 193 T F T <---
122792024-02-12T01:39:49.123Z 29 580-599 109 109 109 193 193 193 F F F
122802024-02-12T01:39:49.123ZMax gen: 109, Max flush: 193
122812024-02-12T01:39:49.123ZError: Difference in extent metadata found!
122822024-02-12T01:39:49.123ZA Difference in extent metadata is expected here
122832024-02-12T01:39:49.123ZOn loop 55
122842024-02-12T01:39:49.123Z
122852024-02-12T01:39:49.126ZVerifying data now
122862024-02-12T01:39:50.656ZNo change after verify
122872024-02-12T01:39:50.656ZLoop: 55 Downstairs dump after verify (and repair):
122882024-02-12T01:39:50.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122892024-02-12T01:39:50.671Z 0 000-019 110 110 110 194 194 194 F F F
122902024-02-12T01:39:50.671Z 1 020-039 107 107 107 189 189 189 F F F
122912024-02-12T01:39:50.671Z 2 040-059 110 110 110 194 194 194 F F F
122922024-02-12T01:39:50.671Z 3 060-079 107 107 107 189 189 189 F F F
122932024-02-12T01:39:50.671Z 4 080-099 106 106 106 188 188 188 F F F
122942024-02-12T01:39:50.672Z 5 100-119 110 110 110 194 194 194 F F F
122952024-02-12T01:39:50.672Z 6 120-139 108 108 108 192 192 192 F F F
122962024-02-12T01:39:50.672Z 7 140-159 100 100 100 176 176 176 F F F
122972024-02-12T01:39:50.672Z 8 160-179 110 110 110 194 194 194 F F F
122982024-02-12T01:39:50.672Z 9 180-199 107 107 107 189 189 189 F F F
122992024-02-12T01:39:50.672Z 10 200-219 107 107 107 189 189 189 F F F
123002024-02-12T01:39:50.672Z 11 220-239 107 107 107 189 189 189 F F F
123012024-02-12T01:39:50.672Z 12 240-259 107 107 107 189 189 189 F F F
123022024-02-12T01:39:50.672Z 13 260-279 109 109 109 193 193 193 F F F
123032024-02-12T01:39:50.675Z 14 280-299 109 109 109 193 193 193 F F F
123042024-02-12T01:39:50.675Z 15 300-319 109 109 109 193 193 193 F F F
123052024-02-12T01:39:50.675Z 16 320-339 109 109 109 193 193 193 F F F
123062024-02-12T01:39:50.675Z 17 340-359 105 105 105 187 187 187 F F F
123072024-02-12T01:39:50.675Z 18 360-379 110 110 110 194 194 194 F F F
123082024-02-12T01:39:50.675Z 19 380-399 109 109 109 193 193 193 F F F
123092024-02-12T01:39:50.675Z 20 400-419 110 110 110 194 194 194 F F F
123102024-02-12T01:39:50.675Z 21 420-439 110 110 110 194 194 194 F F F
123112024-02-12T01:39:50.676Z 22 440-459 107 107 107 189 189 189 F F F
123122024-02-12T01:39:50.676Z 23 460-479 110 110 110 194 194 194 F F F
123132024-02-12T01:39:50.676Z 24 480-499 109 109 109 193 193 193 F F F
123142024-02-12T01:39:50.676Z 25 500-519 109 109 109 193 193 193 F F F
123152024-02-12T01:39:50.676Z 26 520-539 107 107 107 189 189 189 F F F
123162024-02-12T01:39:50.676Z 27 540-559 110 110 110 194 194 194 F F F
123172024-02-12T01:39:50.676Z 28 560-579 110 110 110 194 194 194 F F F
123182024-02-12T01:39:50.676Z 29 580-599 109 109 109 193 193 193 F F F
123192024-02-12T01:39:50.676ZMax gen: 110, Max flush: 194
123202024-02-12T01:39:50.676Z
123212024-02-12T01:39:50.676ZBegin loop 56
123222024-02-12T01:39:50.676ZDownstairs to restart: 0
123232024-02-12T01:39:50.693Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.578733863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
123242024-02-12T01:39:50.693Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",Crucible runtime is spawned
123252024-02-12T01:39:50.694Z\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579056798Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
123262024-02-12T01:39:50.702Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579092446Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
123272024-02-12T01:39:50.702Z{"msg":"Crucible f28b5480-e637-4551-ab9d-a09151ca9b6a has session id: a35598e2-bb0f-4d0c-8b52-de1dfb982200","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579132754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123282024-02-12T01:39:50.702Z{"msg":"Upstairs opts: Upstairs UUID: f28b5480-e637-4551-ab9d-a09151ca9b6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579192461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123292024-02-12T01:39:50.702Z{"msg":"Crucible stats registered with UUID: f28b5480-e637-4551-ab9d-a09151ca9b6a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579235559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123302024-02-12T01:39:50.702Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579253698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
123312024-02-12T01:39:50.702Z{"msg":"f28b5480-e637-4551-ab9d-a09151ca9b6a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579282127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123322024-02-12T01:39:50.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579300936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123332024-02-12T01:39:50.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579324615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123342024-02-12T01:39:50.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579363493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123352024-02-12T01:39:50.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579392842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123362024-02-12T01:39:50.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579413641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123372024-02-12T01:39:50.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579443249Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123382024-02-12T01:39:50.702Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579473208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123392024-02-12T01:39:50.702Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579499067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123402024-02-12T01:39:50.702Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579522976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123412024-02-12T01:39:50.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579548014Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123422024-02-12T01:39:50.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.579594922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123432024-02-12T01:39:50.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.57963322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123442024-02-12T01:39:50.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.58027162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123452024-02-12T01:39:50.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.580308218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123462024-02-12T01:39:50.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.580325677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123472024-02-12T01:39:50.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.580340837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123482024-02-12T01:39:50.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.580819604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123492024-02-12T01:39:50.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.580955217Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123502024-02-12T01:39:50.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.58132172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123512024-02-12T01:39:50.703Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581372217Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123522024-02-12T01:39:50.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581398436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123532024-02-12T01:39:50.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581413155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123542024-02-12T01:39:50.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581459233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123552024-02-12T01:39:50.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581473692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123562024-02-12T01:39:50.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581797087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123572024-02-12T01:39:50.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.581824506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123582024-02-12T01:39:50.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582292953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123592024-02-12T01:39:50.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582708713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123602024-02-12T01:39:50.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582742422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123612024-02-12T01:39:50.703Z{"msg":"[0]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 194, 192, 176, 194, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582759151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123622024-02-12T01:39:50.703Z{"msg":"[0]R generation[0..12]: [110, 107, 110, 107, 106, 110, 108, 100, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.58278108Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123632024-02-12T01:39:50.703Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582796319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123642024-02-12T01:39:50.703Z{"msg":"[1]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 194, 192, 176, 194, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582817038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123652024-02-12T01:39:50.703Z{"msg":"[1]R generation[0..12]: [110, 107, 110, 107, 106, 110, 108, 100, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582831198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123662024-02-12T01:39:50.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582857337Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123672024-02-12T01:39:50.704ZWait for a query_work_queue command to finish before sending IO
123682024-02-12T01:39:50.704Z{"msg":"[2]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 194, 192, 176, 194, 189, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582897524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123692024-02-12T01:39:50.704Z{"msg":"[2]R generation[0..12]: [110, 107, 110, 107, 106, 110, 108, 100, 110, 107, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582934063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123702024-02-12T01:39:50.704Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.582965251Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123712024-02-12T01:39:50.704Z{"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.58299998Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123722024-02-12T01:39:50.704Z{"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583026118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123732024-02-12T01:39:50.704Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583059927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123742024-02-12T01:39:50.704Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583095065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123752024-02-12T01:39:50.704Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583128864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123762024-02-12T01:39:50.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583143523Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123772024-02-12T01:39:50.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583171111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123782024-02-12T01:39:50.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.58320882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123792024-02-12T01:39:50.704Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583233968Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123802024-02-12T01:39:50.704Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583249948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123812024-02-12T01:39:50.704Z{"msg":"f28b5480-e637-4551-ab9d-a09151ca9b6a is now active with session: a35598e2-bb0f-4d0c-8b52-de1dfb982200","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583285056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123822024-02-12T01:39:50.704Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583308045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583,"session_id":"a35598e2-bb0f-4d0c-8b52-de1dfb982200"}
123832024-02-12T01:39:50.704Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:49.583321034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
123842024-02-12T01:39:50.704ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123852024-02-12T01:39:50.704ZLoading write count information from file "/tmp/test_repair_verify.data"
123862024-02-12T01:39:50.704ZRun Repair workload
123872024-02-12T01:39:50.704Z01/30 Write block 566 len 16384 data: 10 11 13 14
123882024-02-12T01:39:50.704Z02/30 Read block 510 len 36864
123892024-02-12T01:39:50.710Z03/30 Read block 502 len 40960
123902024-02-12T01:39:50.710Z04/30 Read block 172 len 36864
123912024-02-12T01:39:50.710Z05/30 Write block 141 len 28672 data: 13 15 15 15 15 11 10
123922024-02-12T01:39:50.710Z06/30 Write block 134 len 36864 data: 7 7 9 9 11 11 10 14 16
123932024-02-12T01:39:50.710Z07/30 Write block 196 len 8192 data: 5 7
123942024-02-12T01:39:50.710Z08/30 Read block 326 len 12288
123952024-02-12T01:39:50.710Z09/30 Read block 273 len 16384
123962024-02-12T01:39:50.710Z10/30 Read block 196 len 36864
123972024-02-12T01:39:50.710Z11/30 Read block 219 len 8192
123982024-02-12T01:39:50.710Z12/30 Read block 463 len 12288
123992024-02-12T01:39:50.713Z13/30 Read block 133 len 40960
124002024-02-12T01:39:50.714Z14/30 Write block 149 len 16384 data: 13 14 13 11
124012024-02-12T01:39:50.714Z15/30 Flush
124022024-02-12T01:39:50.720Z16/30 Write block 575 len 12288 data: 12 9 10
124032024-02-12T01:39:50.720Z17/30 Flush
124042024-02-12T01:39:50.723Z18/30 Write block 408 len 40960 data: 11 11 12 10 10 12 11 12 10 9
124052024-02-12T01:39:50.723Z19/30 Read block 572 len 12288
124062024-02-12T01:39:50.723Z20/30 Read block 428 len 24576
124072024-02-12T01:39:50.723Z21/30 Read block 198 len 4096
124082024-02-12T01:39:50.723Z22/30 Write block 264 len 16384 data: 6 10 11 10
124092024-02-12T01:39:50.723Z23/30 Write block 370 len 4096 data: 12
124102024-02-12T01:39:50.723Z24/30 Read block 373 len 28672
124112024-02-12T01:39:50.726Z25/30 Write block 115 len 40960 data: 9 10 12 13 12 12 11 10 13 14
124122024-02-12T01:39:50.726Z26/30 Write block 131 len 16384 data: 10 10 8 8
124132024-02-12T01:39:50.726Z27/30 Read block 77 len 20480
124142024-02-12T01:39:50.726Z28/30 Read block 359 len 20480
124152024-02-12T01:39:50.734Z29/30 Read block 520 len 36864
124162024-02-12T01:39:50.735Z30/30 Read block 345 len 8192
124172024-02-12T01:39:50.735Z----------------------------------------------------------------
124182024-02-12T01:39:50.735Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
124192024-02-12T01:39:50.735ZStates: Active Active Active
124202024-02-12T01:39:50.735ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124212024-02-12T01:39:50.735Z 1 Acked 1000 Write 4 Done Done Done false
124222024-02-12T01:39:50.735Z 2 Acked 1001 Read 9 Done Done Done false
124232024-02-12T01:39:50.735Z 3 Acked 1002 Read 10 Done Done Done false
124242024-02-12T01:39:50.735Z 4 Acked 1003 Read 9 Done Done Done false
124252024-02-12T01:39:50.735Z 5 Acked 1004 Write 7 Done Done Done false
124262024-02-12T01:39:50.735Z 6 Acked 1005 Write 9 Sent Done Done false
124272024-02-12T01:39:50.735Z 7 Acked 1006 Write 2 Sent Done Done false
124282024-02-12T01:39:50.735Z 8 Acked 1007 Read 3 Sent Done Done false
124292024-02-12T01:39:50.735Z 9 Acked 1008 Read 4 Sent Done Done false
124302024-02-12T01:39:50.735Z 10 Acked 1009 Read 9 Sent Done Done false
124312024-02-12T01:39:50.735Z 11 Acked 1010 Read 2 Sent Done Done false
124322024-02-12T01:39:50.735Z 12 Acked 1011 Read 3 Sent Done Done false
124332024-02-12T01:39:50.735Z 13 Acked 1012 Read 10 Sent Done Done false
124342024-02-12T01:39:50.735Z 14 Acked 1013 Write 4 Sent Done Done false
124352024-02-12T01:39:50.735Z 15 Acked 1014 Flush 0 Sent Done Done false
124362024-02-12T01:39:50.735Z 16 Acked 1015 Write 3 Sent Done Done false
124372024-02-12T01:39:50.735Z 17 Acked 1016 Flush 0 Sent Done Done false
124382024-02-12T01:39:50.735Z 18 Acked 1017 Write 10 Sent Done Done false
124392024-02-12T01:39:50.735Z 19 Acked 1018 Read 3 Sent Done Done false
124402024-02-12T01:39:50.736Z 20 Acked 1019 Read 6 Sent Done Done false
124412024-02-12T01:39:50.736Z 21 Acked 1020 Read 1 Sent Done Done false
124422024-02-12T01:39:50.736Z 22 Acked 1021 Write 4 Sent Done Done false
124432024-02-12T01:39:50.736Z 23 Acked 1022 Write 1 Sent Done Done false
124442024-02-12T01:39:50.736Z 24 Acked 1023 Read 7 Sent Done Done false
124452024-02-12T01:39:50.736Z 25 Acked 1024 Write 10 Sent Done Done false
124462024-02-12T01:39:50.736Z 26 Acked 1025 Write 4 Sent Done Done false
124472024-02-12T01:39:50.736Z 27 Acked 1026 Read 5 Sent Done Done false
124482024-02-12T01:39:50.736Z 28 Acked 1027 Read 5 Sent Done Done false
124492024-02-12T01:39:50.736Z 29 Acked 1028 Read 9 Sent Done Done false
124502024-02-12T01:39:50.736Z 30 Acked 1029 Read 2 Sent Done Done false
124512024-02-12T01:39:50.736Z STATES DS:0 DS:1 DS:2 TOTAL
124522024-02-12T01:39:50.736Z New 0 0 0 0
124532024-02-12T01:39:50.736Z Sent 25 0 0 25
124542024-02-12T01:39:50.736Z Done 5 30 30 65
124552024-02-12T01:39:50.736Z Skipped 0 0 0 0
124562024-02-12T01:39:50.736Z Error 0 0 0 0
124572024-02-12T01:39:50.736ZLast Flush: 0 1016 1016
124582024-02-12T01:39:50.736ZDownstairs last five completed:
124592024-02-12T01:39:50.736ZUpstairs last five completed: 30 29 28 27 26
124602024-02-12T01:39:50.736Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:49.614104366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1583}
124612024-02-12T01:39:50.736ZWrote out file "/tmp/test_repair_verify.data"
124622024-02-12T01:39:50.736Z
124632024-02-12T01:39:50.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124642024-02-12T01:39:50.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124652024-02-12T01:39:50.753Z 0 000-019 110 110 110 194 194 194 F F F
124662024-02-12T01:39:50.753Z 1 020-039 107 107 107 189 189 189 F F F
124672024-02-12T01:39:50.753Z 2 040-059 110 110 110 194 194 194 F F F
124682024-02-12T01:39:50.753Z 3 060-079 107 107 107 189 189 189 F F F
124692024-02-12T01:39:50.754Z 4 080-099 106 106 106 188 188 188 F F F
124702024-02-12T01:39:50.754Z 5 100-119 110 110 110 194 194 194 F T T <---
124712024-02-12T01:39:50.754Z 6 120-139 108 112 112 192 195 195 F T T <---
124722024-02-12T01:39:50.754Z 7 140-159 100 112 112 176 195 195 T F F <---
124732024-02-12T01:39:50.754Z 8 160-179 110 110 110 194 194 194 F F F
124742024-02-12T01:39:50.757Z 9 180-199 107 112 112 189 195 195 F F F <---
124752024-02-12T01:39:50.757Z 10 200-219 107 107 107 189 189 189 F F F
124762024-02-12T01:39:50.757Z 11 220-239 107 107 107 189 189 189 F F F
124772024-02-12T01:39:50.757Z 12 240-259 107 107 107 189 189 189 F F F
124782024-02-12T01:39:50.757Z 13 260-279 109 109 109 193 193 193 F T T <---
124792024-02-12T01:39:50.757Z 14 280-299 109 109 109 193 193 193 F F F
124802024-02-12T01:39:50.757Z 15 300-319 109 109 109 193 193 193 F F F
124812024-02-12T01:39:50.757Z 16 320-339 109 109 109 193 193 193 F F F
124822024-02-12T01:39:50.757Z 17 340-359 105 105 105 187 187 187 F F F
124832024-02-12T01:39:50.757Z 18 360-379 110 110 110 194 194 194 F T T <---
124842024-02-12T01:39:50.757Z 19 380-399 109 109 109 193 193 193 F F F
124852024-02-12T01:39:50.757Z 20 400-419 110 110 110 194 194 194 F T T <---
124862024-02-12T01:39:50.757Z 21 420-439 110 110 110 194 194 194 F F F
124872024-02-12T01:39:50.757Z 22 440-459 107 107 107 189 189 189 F F F
124882024-02-12T01:39:50.757Z 23 460-479 110 110 110 194 194 194 F F F
124892024-02-12T01:39:50.758Z 24 480-499 109 109 109 193 193 193 F F F
124902024-02-12T01:39:50.758Z 25 500-519 109 109 109 193 193 193 F F F
124912024-02-12T01:39:50.758Z 26 520-539 107 107 107 189 189 189 F F F
124922024-02-12T01:39:50.758Z 27 540-559 110 110 110 194 194 194 F F F
124932024-02-12T01:39:50.758Z 28 560-579 110 112 112 194 196 196 T F F <---
124942024-02-12T01:39:50.758Z 29 580-599 109 109 109 193 193 193 F F F
124952024-02-12T01:39:50.758ZMax gen: 112, Max flush: 196
124962024-02-12T01:39:50.758ZError: Difference in extent metadata found!
124972024-02-12T01:39:50.758ZA Difference in extent metadata is expected here
124982024-02-12T01:39:50.758ZOn loop 56
124992024-02-12T01:39:50.758Z
125002024-02-12T01:39:50.762ZVerifying data now
125012024-02-12T01:39:56.202ZNo change after verify
125022024-02-12T01:39:56.202ZLoop: 56 Downstairs dump after verify (and repair):
125032024-02-12T01:39:56.216ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125042024-02-12T01:39:56.216Z 0 000-019 110 110 110 194 194 194 F F F
125052024-02-12T01:39:56.216Z 1 020-039 107 107 107 189 189 189 F F F
125062024-02-12T01:39:56.216Z 2 040-059 110 110 110 194 194 194 F F F
125072024-02-12T01:39:56.216Z 3 060-079 107 107 107 189 189 189 F F F
125082024-02-12T01:39:56.216Z 4 080-099 106 106 106 188 188 188 F F F
125092024-02-12T01:39:56.220Z 5 100-119 113 113 113 197 197 197 F F F
125102024-02-12T01:39:56.220Z 6 120-139 113 113 113 197 197 197 F F F
125112024-02-12T01:39:56.220Z 7 140-159 112 112 112 195 195 195 F F F
125122024-02-12T01:39:56.220Z 8 160-179 110 110 110 194 194 194 F F F
125132024-02-12T01:39:56.220Z 9 180-199 112 112 112 195 195 195 F F F
125142024-02-12T01:39:56.220Z 10 200-219 107 107 107 189 189 189 F F F
125152024-02-12T01:39:56.220Z 11 220-239 107 107 107 189 189 189 F F F
125162024-02-12T01:39:56.220Z 12 240-259 107 107 107 189 189 189 F F F
125172024-02-12T01:39:56.220Z 13 260-279 113 113 113 197 197 197 F F F
125182024-02-12T01:39:56.220Z 14 280-299 109 109 109 193 193 193 F F F
125192024-02-12T01:39:56.220Z 15 300-319 109 109 109 193 193 193 F F F
125202024-02-12T01:39:56.220Z 16 320-339 109 109 109 193 193 193 F F F
125212024-02-12T01:39:56.220Z 17 340-359 105 105 105 187 187 187 F F F
125222024-02-12T01:39:56.220Z 18 360-379 113 113 113 197 197 197 F F F
125232024-02-12T01:39:56.220Z 19 380-399 109 109 109 193 193 193 F F F
125242024-02-12T01:39:56.220Z 20 400-419 113 113 113 197 197 197 F F F
125252024-02-12T01:39:56.220Z 21 420-439 110 110 110 194 194 194 F F F
125262024-02-12T01:39:56.220Z 22 440-459 107 107 107 189 189 189 F F F
125272024-02-12T01:39:56.220Z 23 460-479 110 110 110 194 194 194 F F F
125282024-02-12T01:39:56.220Z 24 480-499 109 109 109 193 193 193 F F F
125292024-02-12T01:39:56.220Z 25 500-519 109 109 109 193 193 193 F F F
125302024-02-12T01:39:56.220Z 26 520-539 107 107 107 189 189 189 F F F
125312024-02-12T01:39:56.220Z 27 540-559 110 110 110 194 194 194 F F F
125322024-02-12T01:39:56.221Z 28 560-579 112 112 112 196 196 196 F F F
125332024-02-12T01:39:56.221Z 29 580-599 109 109 109 193 193 193 F F F
125342024-02-12T01:39:56.221ZMax gen: 113, Max flush: 197
125352024-02-12T01:39:56.221Z
125362024-02-12T01:39:56.221ZBegin loop 57
125372024-02-12T01:39:56.221ZDownstairs to restart: 2
125382024-02-12T01:39:56.241Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.122675048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
125392024-02-12T01:39:56.241Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"Crucible runtime is spawned
125402024-02-12T01:39:56.246Z2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12304453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
125412024-02-12T01:39:56.247Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123072519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
125422024-02-12T01:39:56.247Z{"msg":"Crucible a08d2c26-083e-46ee-a518-d5a499680608 has session id: 13fb11d2-1df8-4ad9-a31e-21bc23326018","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123095078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125432024-02-12T01:39:56.247Z{"msg":"Upstairs opts: Upstairs UUID: a08d2c26-083e-46ee-a518-d5a499680608, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123129296Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125442024-02-12T01:39:56.247Z{"msg":"Crucible stats registered with UUID: a08d2c26-083e-46ee-a518-d5a499680608","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123166375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125452024-02-12T01:39:56.247Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123189493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
125462024-02-12T01:39:56.247Z{"msg":"a08d2c26-083e-46ee-a518-d5a499680608 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123248691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125472024-02-12T01:39:56.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123290409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125482024-02-12T01:39:56.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123319007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125492024-02-12T01:39:56.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123346246Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125502024-02-12T01:39:56.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123372585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125512024-02-12T01:39:56.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123409823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125522024-02-12T01:39:56.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123452381Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125532024-02-12T01:39:56.247Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123513488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125542024-02-12T01:39:56.247Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123555806Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125552024-02-12T01:39:56.247Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123583735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125562024-02-12T01:39:56.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123620543Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125572024-02-12T01:39:56.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123656741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125582024-02-12T01:39:56.247Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.123694829Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125592024-02-12T01:39:56.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.124261192Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125602024-02-12T01:39:56.247Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12430149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125612024-02-12T01:39:56.247Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.124379766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125622024-02-12T01:39:56.247Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.124408365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125632024-02-12T01:39:56.247Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.124904461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125642024-02-12T01:39:56.248Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125120121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125652024-02-12T01:39:56.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12535546Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125662024-02-12T01:39:56.248Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125403838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125672024-02-12T01:39:56.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125476004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125682024-02-12T01:39:56.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125618007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125692024-02-12T01:39:56.248Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125668435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125702024-02-12T01:39:56.248Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125709643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125712024-02-12T01:39:56.248Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.125739632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125722024-02-12T01:39:56.248Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12576633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125732024-02-12T01:39:56.248Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126157852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125742024-02-12T01:39:56.248Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12661165Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125752024-02-12T01:39:56.248Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126649068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125762024-02-12T01:39:56.248Z{"msg":"[0]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 197, 197, 195, 194, 195, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126685196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125772024-02-12T01:39:56.248Z{"msg":"[0]R generation[0..12]: [110, 107, 110, 107, 106, 113, 113, 112, 110, 112, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126708026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125782024-02-12T01:39:56.248Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126723445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125792024-02-12T01:39:56.248Z{"msg":"[1]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 197, 197, 195, 194, 195, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126744654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125802024-02-12T01:39:56.248Z{"msg":"[1]R generation[0..12]: [110, 107, 110, 107, 106, 113, 113, 112, 110, 112, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126763633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125812024-02-12T01:39:56.248Z{"msg":"Wait for a query_work_queue command to finish before sending IO
125822024-02-12T01:39:56.248Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126800191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125832024-02-12T01:39:56.248Z{"msg":"[2]R flush_numbers[0..12]: [194, 189, 194, 189, 188, 197, 197, 195, 194, 195, 189, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126837699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125842024-02-12T01:39:56.248Z{"msg":"[2]R generation[0..12]: [110, 107, 110, 107, 106, 113, 113, 112, 110, 112, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126862648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125852024-02-12T01:39:56.248Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126896497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125862024-02-12T01:39:56.249Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126934985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125872024-02-12T01:39:56.249Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126958374Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125882024-02-12T01:39:56.249Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.126985942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125892024-02-12T01:39:56.249Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12703134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125902024-02-12T01:39:56.249Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127057759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125912024-02-12T01:39:56.249Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127085957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125922024-02-12T01:39:56.249Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127110046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125932024-02-12T01:39:56.249Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127136765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125942024-02-12T01:39:56.249Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127160474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125952024-02-12T01:39:56.249Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127186803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125962024-02-12T01:39:56.249Z{"msg":"a08d2c26-083e-46ee-a518-d5a499680608 is now active with session: 13fb11d2-1df8-4ad9-a31e-21bc23326018","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127209691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125972024-02-12T01:39:56.249ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125982024-02-12T01:39:56.249Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.12723752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591,"session_id":"13fb11d2-1df8-4ad9-a31e-21bc23326018"}
125992024-02-12T01:39:56.249Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:39:55.127259759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
126002024-02-12T01:39:56.249ZLoading write count information from file "/tmp/test_repair_verify.data"
126012024-02-12T01:39:56.249ZRun Repair workload
126022024-02-12T01:39:56.249Z01/30 Write block 254 len 12288 data: 6 6 5
126032024-02-12T01:39:56.249Z02/30 Write block 121 len 12288 data: 12 11 14
126042024-02-12T01:39:56.253Z03/30 Write block 49 len 40960 data: 12 11 12 8 6 6 7 7 10 13
126052024-02-12T01:39:56.253Z04/30 Write block 29 len 28672 data: 12 13 14 14 16 15 11
126062024-02-12T01:39:56.253Z05/30 Read block 258 len 8192
126072024-02-12T01:39:56.253Z06/30 Write block 151 len 16384 data: 14 12 9 8
126082024-02-12T01:39:56.253Z07/30 Write block 269 len 12288 data: 7 7 8
126092024-02-12T01:39:56.253Z08/30 Write block 255 len 28672 data: 7 6 6 6 6 9 6
126102024-02-12T01:39:56.253Z09/30 Write block 352 len 8192 data: 11 9
126112024-02-12T01:39:56.253Z10/30 Read block 341 len 32768
126122024-02-12T01:39:56.253Z11/30 Write block 290 len 36864 data: 8 9 7 7 7 10 8 8 8
126132024-02-12T01:39:56.253Z12/30 Read block 88 len 36864
126142024-02-12T01:39:56.256Z13/30 Write block 461 len 24576 data: 5 6 6 7 7 9
126152024-02-12T01:39:56.256Z14/30 Write block 326 len 40960 data: 8 8 8 7 6 7 6 5 5 5
126162024-02-12T01:39:56.256Z15/30 Write block 235 len 20480 data: 7 10 13 14 13
126172024-02-12T01:39:56.256Z16/30 Read block 367 len 20480
126182024-02-12T01:39:56.259Z17/30 Write block 272 len 12288 data: 7 8 8
126192024-02-12T01:39:56.259Z18/30 Write block 267 len 28672 data: 11 9 8 8 9 8 9
126202024-02-12T01:39:56.259Z19/30 Read block 221 len 4096
126212024-02-12T01:39:56.262Z20/30 Read block 59 len 36864
126222024-02-12T01:39:56.262Z21/30 Read block 518 len 32768
126232024-02-12T01:39:56.262Z22/30 Write block 340 len 20480 data: 6 4 4 7 5
126242024-02-12T01:39:56.265Z23/30 Write block 53 len 28672 data: 7 7 8 8 11 14 10
126252024-02-12T01:39:56.265Z24/30 Read block 359 len 16384
126262024-02-12T01:39:56.265Z25/30 Read block 77 len 8192
126272024-02-12T01:39:56.265Z26/30 Flush
126282024-02-12T01:39:56.270Z27/30 Write block 392 len 28672 data: 14 15 13 9 8 8 8
126292024-02-12T01:39:56.273Z28/30 Write block 448 len 12288 data: 9 9 9
126302024-02-12T01:39:56.273Z29/30 Write block 218 len 20480 data: 10 11 9 8 8
126312024-02-12T01:39:56.274Z30/30 Write block 154 len 28672 data: 9 5 4 5 6 6 9
126322024-02-12T01:39:56.274Z----------------------------------------------------------------
126332024-02-12T01:39:56.274Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
126342024-02-12T01:39:56.274ZStates: Active Active Active
126352024-02-12T01:39:56.274ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126362024-02-12T01:39:56.274Z 1 Acked 1000 Write 3 Done Done Done false
126372024-02-12T01:39:56.274Z 2 Acked 1001 Write 3 Done Done Done false
126382024-02-12T01:39:56.274Z 3 Acked 1002 Write 10 Done Done Sent false
126392024-02-12T01:39:56.274Z 4 Acked 1003 Write 7 Done Done Sent false
126402024-02-12T01:39:56.274Z 5 Acked 1004 Read 2 Done Done Sent false
126412024-02-12T01:39:56.274Z 6 Acked 1005 Write 4 Done Done Sent false
126422024-02-12T01:39:56.274Z 7 Acked 1006 Write 3 Done Done Sent false
126432024-02-12T01:39:56.274Z 8 Acked 1007 Write 7 Done Done Sent false
126442024-02-12T01:39:56.274Z 9 Acked 1008 Write 2 Done Done Sent false
126452024-02-12T01:39:56.274Z 10 Acked 1009 Read 8 Done Done Sent false
126462024-02-12T01:39:56.274Z 11 Acked 1010 Write 9 Done Done Sent false
126472024-02-12T01:39:56.274Z 12 Acked 1011 Read 9 Done Done Sent false
126482024-02-12T01:39:56.274Z 13 Acked 1012 Write 6 Done Done Sent false
126492024-02-12T01:39:56.274Z 14 Acked 1013 Write 10 Done Done Sent false
126502024-02-12T01:39:56.274Z 15 Acked 1014 Write 5 Done Done Sent false
126512024-02-12T01:39:56.274Z 16 Acked 1015 Read 5 Done Done Sent false
126522024-02-12T01:39:56.274Z 17 Acked 1016 Write 3 Done Done Sent false
126532024-02-12T01:39:56.274Z 18 Acked 1017 Write 7 Done Done Sent false
126542024-02-12T01:39:56.274Z 19 Acked 1018 Read 1 Done Done Sent false
126552024-02-12T01:39:56.275Z 20 Acked 1019 Read 9 Done Done Sent false
126562024-02-12T01:39:56.275Z 21 Acked 1020 Read 8 Done Done Sent false
126572024-02-12T01:39:56.275Z 22 Acked 1021 Write 5 Done Done Sent false
126582024-02-12T01:39:56.275Z 23 Acked 1022 Write 7 Done Done Sent false
126592024-02-12T01:39:56.275Z 24 Acked 1023 Read 4 Done Done Sent false
126602024-02-12T01:39:56.275Z 25 Acked 1024 Read 2 Done Done Sent false
126612024-02-12T01:39:56.275Z 26 Acked 1025 Flush 0 Done Done Sent false
126622024-02-12T01:39:56.275Z 27 Acked 1026 Write 7 Done Done Sent false
126632024-02-12T01:39:56.275Z 28 Acked 1027 Write 3 Done Done Sent false
126642024-02-12T01:39:56.275Z 29 Acked 1028 Write 5 Done Done Sent false
126652024-02-12T01:39:56.275Z 30 Acked 1029 Write 7 Sent Sent Sent false
126662024-02-12T01:39:56.275Z STATES DS:0 DS:1 DS:2 TOTAL
126672024-02-12T01:39:56.275Z New 0 0 0 0
126682024-02-12T01:39:56.275Z Sent 1 1 28 30
126692024-02-12T01:39:56.275Z Done 29 29 2 60
126702024-02-12T01:39:56.275Z Skipped 0 0 0 0
126712024-02-12T01:39:56.275Z Error 0 0 0 0
126722024-02-12T01:39:56.275ZLast Flush: 1025 1025 0
126732024-02-12T01:39:56.275ZDownstairs last five completed:
126742024-02-12T01:39:56.275ZUpstairs last five completed: 30 29 28 27 26
126752024-02-12T01:39:56.278Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:39:55.158413453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1591}
126762024-02-12T01:39:56.279ZWrote out file "/tmp/test_repair_verify.data"
126772024-02-12T01:39:56.279Z
126782024-02-12T01:39:56.281ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126792024-02-12T01:39:56.299ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126802024-02-12T01:39:56.299Z 0 000-019 110 110 110 194 194 194 F F F
126812024-02-12T01:39:56.299Z 1 020-039 114 114 107 198 198 189 F F F <---
126822024-02-12T01:39:56.299Z 2 040-059 114 114 110 198 198 194 F F F <---
126832024-02-12T01:39:56.299Z 3 060-079 107 107 107 189 189 189 F F F
126842024-02-12T01:39:56.299Z 4 080-099 106 106 106 188 188 188 F F F
126852024-02-12T01:39:56.302Z 5 100-119 113 113 113 197 197 197 F F F
126862024-02-12T01:39:56.302Z 6 120-139 114 114 113 198 198 197 F F T <---
126872024-02-12T01:39:56.302Z 7 140-159 114 114 112 198 198 195 T T F <---
126882024-02-12T01:39:56.302Z 8 160-179 110 110 110 194 194 194 T T F <---
126892024-02-12T01:39:56.302Z 9 180-199 112 112 112 195 195 195 F F F
126902024-02-12T01:39:56.302Z 10 200-219 107 107 107 189 189 189 T T F <---
126912024-02-12T01:39:56.302Z 11 220-239 114 114 107 198 198 189 T T F <---
126922024-02-12T01:39:56.302Z 12 240-259 114 114 107 198 198 189 F F T <---
126932024-02-12T01:39:56.302Z 13 260-279 114 114 113 198 198 197 F F F <---
126942024-02-12T01:39:56.302Z 14 280-299 114 114 109 198 198 193 F F F <---
126952024-02-12T01:39:56.302Z 15 300-319 109 109 109 193 193 193 F F F
126962024-02-12T01:39:56.302Z 16 320-339 114 114 109 198 198 193 F F F <---
126972024-02-12T01:39:56.302Z 17 340-359 114 114 105 198 198 187 F F F <---
126982024-02-12T01:39:56.302Z 18 360-379 113 113 113 197 197 197 F F F
126992024-02-12T01:39:56.303Z 19 380-399 109 109 109 193 193 193 T T F <---
127002024-02-12T01:39:56.303Z 20 400-419 113 113 113 197 197 197 F F F
127012024-02-12T01:39:56.303Z 21 420-439 110 110 110 194 194 194 F F F
127022024-02-12T01:39:56.303Z 22 440-459 107 107 107 189 189 189 T T F <---
127032024-02-12T01:39:56.303Z 23 460-479 114 114 110 198 198 194 F F F <---
127042024-02-12T01:39:56.303Z 24 480-499 109 109 109 193 193 193 F F F
127052024-02-12T01:39:56.303Z 25 500-519 109 109 109 193 193 193 F F F
127062024-02-12T01:39:56.303Z 26 520-539 107 107 107 189 189 189 F F F
127072024-02-12T01:39:56.303Z 27 540-559 110 110 110 194 194 194 F F F
127082024-02-12T01:39:56.303Z 28 560-579 112 112 112 196 196 196 F F F
127092024-02-12T01:39:56.303Z 29 580-599 109 109 109 193 193 193 F F F
127102024-02-12T01:39:56.303ZMax gen: 114, Max flush: 198
127112024-02-12T01:39:56.303ZError: Difference in extent metadata found!
127122024-02-12T01:39:56.303ZA Difference in extent metadata is expected here
127132024-02-12T01:39:56.303ZOn loop 57
127142024-02-12T01:39:56.303Z
127152024-02-12T01:39:56.309ZVerifying data now
127162024-02-12T01:40:02.033ZNo change after verify
127172024-02-12T01:40:02.033ZLoop: 57 Downstairs dump after verify (and repair):
127182024-02-12T01:40:02.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127192024-02-12T01:40:02.049Z 0 000-019 110 110 110 194 194 194 F F F
127202024-02-12T01:40:02.049Z 1 020-039 114 114 114 198 198 198 F F F
127212024-02-12T01:40:02.049Z 2 040-059 114 114 114 198 198 198 F F F
127222024-02-12T01:40:02.049Z 3 060-079 107 107 107 189 189 189 F F F
127232024-02-12T01:40:02.049Z 4 080-099 106 106 106 188 188 188 F F F
127242024-02-12T01:40:02.049Z 5 100-119 113 113 113 197 197 197 F F F
127252024-02-12T01:40:02.049Z 6 120-139 114 114 114 198 198 198 F F F
127262024-02-12T01:40:02.049Z 7 140-159 115 115 115 199 199 199 F F F
127272024-02-12T01:40:02.049Z 8 160-179 115 115 115 199 199 199 F F F
127282024-02-12T01:40:02.049Z 9 180-199 112 112 112 195 195 195 F F F
127292024-02-12T01:40:02.049Z 10 200-219 115 115 115 199 199 199 F F F
127302024-02-12T01:40:02.049Z 11 220-239 115 115 115 199 199 199 F F F
127312024-02-12T01:40:02.049Z 12 240-259 114 114 114 198 198 198 F F F
127322024-02-12T01:40:02.049Z 13 260-279 114 114 114 198 198 198 F F F
127332024-02-12T01:40:02.053Z 14 280-299 114 114 114 198 198 198 F F F
127342024-02-12T01:40:02.053Z 15 300-319 109 109 109 193 193 193 F F F
127352024-02-12T01:40:02.053Z 16 320-339 114 114 114 198 198 198 F F F
127362024-02-12T01:40:02.053Z 17 340-359 114 114 114 198 198 198 F F F
127372024-02-12T01:40:02.053Z 18 360-379 113 113 113 197 197 197 F F F
127382024-02-12T01:40:02.053Z 19 380-399 115 115 115 199 199 199 F F F
127392024-02-12T01:40:02.053Z 20 400-419 113 113 113 197 197 197 F F F
127402024-02-12T01:40:02.053Z 21 420-439 110 110 110 194 194 194 F F F
127412024-02-12T01:40:02.054Z 22 440-459 115 115 115 199 199 199 F F F
127422024-02-12T01:40:02.054Z 23 460-479 114 114 114 198 198 198 F F F
127432024-02-12T01:40:02.054Z 24 480-499 109 109 109 193 193 193 F F F
127442024-02-12T01:40:02.054Z 25 500-519 109 109 109 193 193 193 F F F
127452024-02-12T01:40:02.054Z 26 520-539 107 107 107 189 189 189 F F F
127462024-02-12T01:40:02.054Z 27 540-559 110 110 110 194 194 194 F F F
127472024-02-12T01:40:02.054Z 28 560-579 112 112 112 196 196 196 F F F
127482024-02-12T01:40:02.054Z 29 580-599 109 109 109 193 193 193 F F F
127492024-02-12T01:40:02.054ZMax gen: 115, Max flush: 199
127502024-02-12T01:40:02.054Z
127512024-02-12T01:40:02.054ZBegin loop 58
127522024-02-12T01:40:02.054ZDownstairs to restart: 0
127532024-02-12T01:40:02.069Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953212097Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
127542024-02-12T01:40:02.069Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953476204Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
127552024-02-12T01:40:02.076Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953496494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
127562024-02-12T01:40:02.077Z{"msg":"Crucible 7283bbd7-1aeb-4659-b830-48241d172b2c has session id: 42248c11-85cb-45b1-9e36-99ba653d5eef","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953528512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127572024-02-12T01:40:02.077Z{"msg":"Upstairs opts: Upstairs UUID: 7283bbd7-1aeb-4659-b830-48241d172b2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
127582024-02-12T01:40:02.077Z,"time":"2024-02-12T01:40:00.95357677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127592024-02-12T01:40:02.077Z{"msg":"Crucible stats registered with UUID: 7283bbd7-1aeb-4659-b830-48241d172b2c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953615758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127602024-02-12T01:40:02.077Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953651226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
127612024-02-12T01:40:02.077Z{"msg":"7283bbd7-1aeb-4659-b830-48241d172b2c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953688025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127622024-02-12T01:40:02.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953714643Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127632024-02-12T01:40:02.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953739432Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127642024-02-12T01:40:02.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95377831Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127652024-02-12T01:40:02.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953801909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127662024-02-12T01:40:02.077Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953846007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127672024-02-12T01:40:02.077Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953874396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127682024-02-12T01:40:02.077Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.953908284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127692024-02-12T01:40:02.077Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95399032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127702024-02-12T01:40:02.077Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954035888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127712024-02-12T01:40:02.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954071706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127722024-02-12T01:40:02.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954108125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127732024-02-12T01:40:02.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954135113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127742024-02-12T01:40:02.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95482264Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127752024-02-12T01:40:02.077Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954869308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127762024-02-12T01:40:02.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954915426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127772024-02-12T01:40:02.078Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.954956094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127782024-02-12T01:40:02.078Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.955410852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127792024-02-12T01:40:02.078Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95546208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127802024-02-12T01:40:02.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956127128Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127812024-02-12T01:40:02.078Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956196125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127822024-02-12T01:40:02.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956232293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127832024-02-12T01:40:02.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956247292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127842024-02-12T01:40:02.078Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956262542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127852024-02-12T01:40:02.078Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956276101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127862024-02-12T01:40:02.078Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95629735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127872024-02-12T01:40:02.078Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956357997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127882024-02-12T01:40:02.078Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.956837834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127892024-02-12T01:40:02.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957274233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127902024-02-12T01:40:02.078Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957311821Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127912024-02-12T01:40:02.078Z{"msg":"[0]R flush_numbers[0..12]: [194, 198, 198, 189, 188, 197, 198, 199, 199, 195, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957328041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127922024-02-12T01:40:02.078Z{"msg":"[0]R generation[0..12]: [110, 114, 114, 107, 106, 113, 114, 115, 115, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95735004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127932024-02-12T01:40:02.078Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957365029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127942024-02-12T01:40:02.078Z{"msg":"[1]R flush_numbers[0..12]: [194, 198, 198, 189, 188, 197, 198, 199, 199, 195, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957393498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127952024-02-12T01:40:02.078Z{"msg":"[1]R generation[0..12]: [110, 114, 114, 107, 106, 113, 114, 115, 115, 112, 115, 115]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
127962024-02-12T01:40:02.082Z,"time":"2024-02-12T01:40:00.957427226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127972024-02-12T01:40:02.082Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957450705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127982024-02-12T01:40:02.082Z{"msg":"[2]R flush_numbers[0..12]: [194, 198, 198, 189, 188, 197, 198, 199, 199, 195, 199, 199]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957465854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
127992024-02-12T01:40:02.082Z{"msg":"[2]R generation[0..12]: [110, 114, 114, 107, 106, 113, 114, 115, 115, 112, 115, 115]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957488113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128002024-02-12T01:40:02.082Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957502692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128012024-02-12T01:40:02.082Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957522791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128022024-02-12T01:40:02.083Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957536941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128032024-02-12T01:40:02.083Z{"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.95756174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128042024-02-12T01:40:02.083Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957575509Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128052024-02-12T01:40:02.083Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957589078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128062024-02-12T01:40:02.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957608678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128072024-02-12T01:40:02.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957630017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128082024-02-12T01:40:02.083Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957650416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128092024-02-12T01:40:02.083Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957681064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128102024-02-12T01:40:02.083Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957695353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128112024-02-12T01:40:02.083Z{"msg":"7283bbd7-1aeb-4659-b830-48241d172b2c is now active with session: 42248c11-85cb-45b1-9e36-99ba653d5eef","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957714972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128122024-02-12T01:40:02.083Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.957745471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599,"session_id":"42248c11-85cb-45b1-9e36-99ba653d5eef"}
128132024-02-12T01:40:02.083Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:00.9577652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
128142024-02-12T01:40:02.083ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128152024-02-12T01:40:02.083ZLoading write count information from file "/tmp/test_repair_verify.data"
128162024-02-12T01:40:02.083ZRun Repair workload
128172024-02-12T01:40:02.083Z01/30 Write block 114 len 20480 data: 11 10 11 13 14
128182024-02-12T01:40:02.083Z02/30 Write block 107 len 20480 data: 10 10 9 8 8
128192024-02-12T01:40:02.083Z03/30 Write block 218 len 24576 data: 11 12 10 9 9 9
128202024-02-12T01:40:02.083Z04/30 Read block 311 len 20480
128212024-02-12T01:40:02.083Z05/30 Read block 586 len 4096
128222024-02-12T01:40:02.083Z06/30 Read block 371 len 36864
128232024-02-12T01:40:02.083Z07/30 Read block 427 len 28672
128242024-02-12T01:40:02.083Z08/30 Read block 317 len 28672
128252024-02-12T01:40:02.087Z09/30 Read block 189 len 16384
128262024-02-12T01:40:02.087Z10/30 Read block 305 len 20480
128272024-02-12T01:40:02.087Z11/30 Write block 131 len 24576 data: 11 11 9 9 8 10
128282024-02-12T01:40:02.087Z12/30 Read block 334 len 24576
128292024-02-12T01:40:02.087Z13/30 Write block 232 len 32768 data: 7 7 8 8 11 14 15 14
128302024-02-12T01:40:02.087Z14/30 Write block 549 len 32768 data: 11 15 14 12 12 9 7 6
128312024-02-12T01:40:02.090Z15/30 Read block 453 len 12288
128322024-02-12T01:40:02.090Z16/30 Read block 164 len 16384
128332024-02-12T01:40:02.090Z17/30 Write block 113 len 28672 data: 8 12 11 12 14 15 13
128342024-02-12T01:40:02.090Z18/30 Flush
128352024-02-12T01:40:02.094Z19/30 Write block 486 len 32768 data: 12 12 12 8 10 7 7 7
128362024-02-12T01:40:02.097Z20/30 Read block 540 len 36864
128372024-02-12T01:40:02.097Z21/30 Write block 45 len 40960 data: 9 9 11 12 13 12 13 9 8 8
128382024-02-12T01:40:02.097Z22/30 Write block 255 len 16384 data: 8 7 7 7
128392024-02-12T01:40:02.100Z23/30 Write block 192 len 12288 data: 9 7 5
128402024-02-12T01:40:02.100Z24/30 Read block 474 len 32768
128412024-02-12T01:40:02.100Z25/30 Read block 297 len 24576
128422024-02-12T01:40:02.100Z26/30 Read block 449 len 32768
128432024-02-12T01:40:02.111Z27/30 Write block 85 len 4096 data: 14
128442024-02-12T01:40:02.111Z28/30 Read block 502 len 4096
128452024-02-12T01:40:02.111Z29/30 Write block 163 len 40960 data: 11 11 13 12 11 11 14 13 12 9
128462024-02-12T01:40:02.115Z30/30 Write block 309 len 36864 data: 11 11 9 7 8 5 6 5 7
128472024-02-12T01:40:02.115Z----------------------------------------------------------------
128482024-02-12T01:40:02.115Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
128492024-02-12T01:40:02.115ZStates: Active Active Active
128502024-02-12T01:40:02.115ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
128512024-02-12T01:40:02.115Z 1 Acked 1000 Write 5 Done Done Done false
128522024-02-12T01:40:02.115Z 2 Acked 1001 Write 5 Sent Done Done false
128532024-02-12T01:40:02.115Z 3 Acked 1002 Write 6 Sent Done Done false
128542024-02-12T01:40:02.115Z 4 Acked 1003 Read 5 Sent Done Done false
128552024-02-12T01:40:02.115Z 5 Acked 1004 Read 1 Sent Done Done false
128562024-02-12T01:40:02.115Z 6 Acked 1005 Read 9 Sent Done Done false
128572024-02-12T01:40:02.116Z 7 Acked 1006 Read 7 Sent Done Done false
128582024-02-12T01:40:02.116Z 8 Acked 1007 Read 7 Sent Done Done false
128592024-02-12T01:40:02.116Z 9 Acked 1008 Read 4 Sent Done Done false
128602024-02-12T01:40:02.116Z 10 Acked 1009 Read 5 Sent Done Done false
128612024-02-12T01:40:02.116Z 11 Acked 1010 Write 6 Sent Done Done false
128622024-02-12T01:40:02.116Z 12 Acked 1011 Read 6 Sent Done Done false
128632024-02-12T01:40:02.116Z 13 Acked 1012 Write 8 Sent Done Done false
128642024-02-12T01:40:02.116Z 14 Acked 1013 Write 8 Sent Done Done false
128652024-02-12T01:40:02.116Z 15 Acked 1014 Read 3 Sent Done Done false
128662024-02-12T01:40:02.116Z 16 Acked 1015 Read 4 Sent Done Done false
128672024-02-12T01:40:02.116Z 17 Acked 1016 Write 7 Sent Done Done false
128682024-02-12T01:40:02.116Z 18 Acked 1017 Flush 0 Sent Done Done false
128692024-02-12T01:40:02.116Z 19 Acked 1018 Write 8 Sent Done Done false
128702024-02-12T01:40:02.116Z 20 Acked 1019 Read 9 Sent Done Done false
128712024-02-12T01:40:02.116Z 21 Acked 1020 Write 10 Sent Done Done false
128722024-02-12T01:40:02.116Z 22 Acked 1021 Write 4 Sent Done Done false
128732024-02-12T01:40:02.116Z 23 Acked 1022 Write 3 Sent Done Done false
128742024-02-12T01:40:02.116Z 24 Acked 1023 Read 8 Sent Done Done false
128752024-02-12T01:40:02.116Z 25 Acked 1024 Read 6 Sent Done Done false
128762024-02-12T01:40:02.116Z 26 Acked 1025 Read 8 Sent Done Done false
128772024-02-12T01:40:02.116Z 27 Acked 1026 Write 1 Sent Done Done false
128782024-02-12T01:40:02.116Z 28 Acked 1027 Read 1 Sent Done Done false
128792024-02-12T01:40:02.116Z 29 Acked 1028 Write 10 Sent Done Done false
128802024-02-12T01:40:02.116Z 30 Acked 1029 Write 9 Sent Sent Sent false
128812024-02-12T01:40:02.116Z STATES DS:0 DS:1 DS:2 TOTAL
128822024-02-12T01:40:02.117Z New 0 0 0 0
128832024-02-12T01:40:02.117Z Sent 29 1 1 31
128842024-02-12T01:40:02.117Z Done 1 29 29 59
128852024-02-12T01:40:02.117Z Skipped 0 0 0 0
128862024-02-12T01:40:02.117Z Error 0 0 0 0
128872024-02-12T01:40:02.117ZLast Flush: 0 1017 1017
128882024-02-12T01:40:02.117ZDownstairs last five completed:
128892024-02-12T01:40:02.117ZUpstairs last five completed: 30 29 28 27 26
128902024-02-12T01:40:02.117Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:00.988688775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1599}
128912024-02-12T01:40:02.117ZWrote out file "/tmp/test_repair_verify.data"
128922024-02-12T01:40:02.117Z
128932024-02-12T01:40:02.117ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128942024-02-12T01:40:02.131ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128952024-02-12T01:40:02.131Z 0 000-019 110 110 110 194 194 194 F F F
128962024-02-12T01:40:02.131Z 1 020-039 114 114 114 198 198 198 F F F
128972024-02-12T01:40:02.131Z 2 040-059 114 114 114 198 198 198 F T T <---
128982024-02-12T01:40:02.131Z 3 060-079 107 107 107 189 189 189 F F F
128992024-02-12T01:40:02.131Z 4 080-099 106 106 106 188 188 188 F T T <---
129002024-02-12T01:40:02.131Z 5 100-119 113 116 116 197 200 200 T F F <---
129012024-02-12T01:40:02.131Z 6 120-139 114 116 116 198 200 200 F F F <---
129022024-02-12T01:40:02.131Z 7 140-159 115 115 115 199 199 199 F F F
129032024-02-12T01:40:02.135Z 8 160-179 115 115 115 199 199 199 F T T <---
129042024-02-12T01:40:02.135Z 9 180-199 112 112 112 195 195 195 F T T <---
129052024-02-12T01:40:02.135Z 10 200-219 115 116 116 199 200 200 F F F <---
129062024-02-12T01:40:02.135Z 11 220-239 115 116 116 199 200 200 F F F <---
129072024-02-12T01:40:02.135Z 12 240-259 114 114 114 198 198 198 F T T <---
129082024-02-12T01:40:02.135Z 13 260-279 114 114 114 198 198 198 F F F
129092024-02-12T01:40:02.135Z 14 280-299 114 114 114 198 198 198 F F F
129102024-02-12T01:40:02.135Z 15 300-319 109 109 109 193 193 193 F T T <---
129112024-02-12T01:40:02.135Z 16 320-339 114 114 114 198 198 198 F F F
129122024-02-12T01:40:02.135Z 17 340-359 114 114 114 198 198 198 F F F
129132024-02-12T01:40:02.135Z 18 360-379 113 113 113 197 197 197 F F F
129142024-02-12T01:40:02.135Z 19 380-399 115 115 115 199 199 199 F F F
129152024-02-12T01:40:02.135Z 20 400-419 113 113 113 197 197 197 F F F
129162024-02-12T01:40:02.136Z 21 420-439 110 110 110 194 194 194 F F F
129172024-02-12T01:40:02.136Z 22 440-459 115 115 115 199 199 199 F F F
129182024-02-12T01:40:02.136Z 23 460-479 114 114 114 198 198 198 F F F
129192024-02-12T01:40:02.136Z 24 480-499 109 109 109 193 193 193 F T T <---
129202024-02-12T01:40:02.136Z 25 500-519 109 109 109 193 193 193 F F F
129212024-02-12T01:40:02.136Z 26 520-539 107 107 107 189 189 189 F F F
129222024-02-12T01:40:02.136Z 27 540-559 110 116 116 194 200 200 F F F <---
129232024-02-12T01:40:02.136Z 28 560-579 112 112 112 196 196 196 F F F
129242024-02-12T01:40:02.136Z 29 580-599 109 109 109 193 193 193 F F F
129252024-02-12T01:40:02.136ZMax gen: 116, Max flush: 200
129262024-02-12T01:40:02.136ZError: Difference in extent metadata found!
129272024-02-12T01:40:02.136ZA Difference in extent metadata is expected here
129282024-02-12T01:40:02.136ZOn loop 58
129292024-02-12T01:40:02.136Z
129302024-02-12T01:40:02.140ZVerifying data now
129312024-02-12T01:40:07.761ZNo change after verify
129322024-02-12T01:40:07.761ZLoop: 58 Downstairs dump after verify (and repair):
129332024-02-12T01:40:07.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129342024-02-12T01:40:07.779Z 0 000-019 110 110 110 194 194 194 F F F
129352024-02-12T01:40:07.779Z 1 020-039 114 114 114 198 198 198 F F F
129362024-02-12T01:40:07.780Z 2 040-059 117 117 117 201 201 201 F F F
129372024-02-12T01:40:07.780Z 3 060-079 107 107 107 189 189 189 F F F
129382024-02-12T01:40:07.780Z 4 080-099 117 117 117 201 201 201 F F F
129392024-02-12T01:40:07.780Z 5 100-119 116 116 116 200 200 200 F F F
129402024-02-12T01:40:07.780Z 6 120-139 116 116 116 200 200 200 F F F
129412024-02-12T01:40:07.780Z 7 140-159 115 115 115 199 199 199 F F F
129422024-02-12T01:40:07.780Z 8 160-179 117 117 117 201 201 201 F F F
129432024-02-12T01:40:07.780Z 9 180-199 117 117 117 201 201 201 F F F
129442024-02-12T01:40:07.780Z 10 200-219 116 116 116 200 200 200 F F F
129452024-02-12T01:40:07.783Z 11 220-239 116 116 116 200 200 200 F F F
129462024-02-12T01:40:07.783Z 12 240-259 117 117 117 201 201 201 F F F
129472024-02-12T01:40:07.783Z 13 260-279 114 114 114 198 198 198 F F F
129482024-02-12T01:40:07.783Z 14 280-299 114 114 114 198 198 198 F F F
129492024-02-12T01:40:07.783Z 15 300-319 117 117 117 201 201 201 F F F
129502024-02-12T01:40:07.783Z 16 320-339 114 114 114 198 198 198 F F F
129512024-02-12T01:40:07.783Z 17 340-359 114 114 114 198 198 198 F F F
129522024-02-12T01:40:07.784Z 18 360-379 113 113 113 197 197 197 F F F
129532024-02-12T01:40:07.784Z 19 380-399 115 115 115 199 199 199 F F F
129542024-02-12T01:40:07.784Z 20 400-419 113 113 113 197 197 197 F F F
129552024-02-12T01:40:07.784Z 21 420-439 110 110 110 194 194 194 F F F
129562024-02-12T01:40:07.784Z 22 440-459 115 115 115 199 199 199 F F F
129572024-02-12T01:40:07.784Z 23 460-479 114 114 114 198 198 198 F F F
129582024-02-12T01:40:07.784Z 24 480-499 117 117 117 201 201 201 F F F
129592024-02-12T01:40:07.784Z 25 500-519 109 109 109 193 193 193 F F F
129602024-02-12T01:40:07.784Z 26 520-539 107 107 107 189 189 189 F F F
129612024-02-12T01:40:07.784Z 27 540-559 116 116 116 200 200 200 F F F
129622024-02-12T01:40:07.784Z 28 560-579 112 112 112 196 196 196 F F F
129632024-02-12T01:40:07.784Z 29 580-599 109 109 109 193 193 193 F F F
129642024-02-12T01:40:07.784ZMax gen: 117, Max flush: 201
129652024-02-12T01:40:07.784Z
129662024-02-12T01:40:07.784ZBegin loop 59
129672024-02-12T01:40:07.784ZDownstairs to restart: 1
129682024-02-12T01:40:07.798Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.681630459Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
129692024-02-12T01:40:07.805Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.681893867Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
129702024-02-12T01:40:07.805Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
129712024-02-12T01:40:07.805Z,"time":"2024-02-12T01:40:06.681966733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
129722024-02-12T01:40:07.805Z{"msg":"Crucible a985b388-d453-4ed4-8bcf-b2c22033d492 has session id: 1301cea8-0522-4c81-a054-7208d1bd1223","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682004652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129732024-02-12T01:40:07.805Z{"msg":"Upstairs opts: Upstairs UUID: a985b388-d453-4ed4-8bcf-b2c22033d492, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68204697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129742024-02-12T01:40:07.805Z{"msg":"Crucible stats registered with UUID: a985b388-d453-4ed4-8bcf-b2c22033d492","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682077468Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129752024-02-12T01:40:07.805Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682118356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
129762024-02-12T01:40:07.805Z{"msg":"a985b388-d453-4ed4-8bcf-b2c22033d492 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682223971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129772024-02-12T01:40:07.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68224953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129782024-02-12T01:40:07.805Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682268349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129792024-02-12T01:40:07.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682316627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129802024-02-12T01:40:07.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682348535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129812024-02-12T01:40:07.806Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682397683Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129822024-02-12T01:40:07.806Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682430851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129832024-02-12T01:40:07.806Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68246089Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129842024-02-12T01:40:07.806Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682498698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129852024-02-12T01:40:07.806Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682529856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129862024-02-12T01:40:07.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682573665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129872024-02-12T01:40:07.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682596103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129882024-02-12T01:40:07.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.682610633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129892024-02-12T01:40:07.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.683364397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129902024-02-12T01:40:07.806Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.683400275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129912024-02-12T01:40:07.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.683415944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129922024-02-12T01:40:07.806Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.683442093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129932024-02-12T01:40:07.806Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.683897051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129942024-02-12T01:40:07.806Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684002266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129952024-02-12T01:40:07.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684464684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129962024-02-12T01:40:07.806Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684501522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129972024-02-12T01:40:07.806Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684517222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129982024-02-12T01:40:07.806Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684531741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
129992024-02-12T01:40:07.806Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68455218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130002024-02-12T01:40:07.806Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684565869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130012024-02-12T01:40:07.806Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684592198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130022024-02-12T01:40:07.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.684634876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130032024-02-12T01:40:07.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685133372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130042024-02-12T01:40:07.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685528004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130052024-02-12T01:40:07.811Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685568891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130062024-02-12T01:40:07.811Z{"msg":"[0]R flush_numbers[0..12]: [194, 198, 201, 189, 201, 200, 200, 199, 201, 201, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685623559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130072024-02-12T01:40:07.811Z{"msg":"[0]R generation[0..12]: [110, 114, 117, 107, 117, 116, 116, 115, 117, 117, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685660467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130082024-02-12T01:40:07.811Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685678706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130092024-02-12T01:40:07.811Z{"msg":"[1]R flush_numbers[0..12]: [194, 198, 201, 189, 201, 200, 200, 199, 201, 201, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685701925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130102024-02-12T01:40:07.811Z{Wait for a query_work_queue command to finish before sending IO
130112024-02-12T01:40:07.811Z"msg":"[1]R generation[0..12]: [110, 114, 117, 107, 117, 116, 116, 115, 117, 117, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685720034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130122024-02-12T01:40:07.811Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685734644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130132024-02-12T01:40:07.811Z{"msg":"[2]R flush_numbers[0..12]: [194, 198, 201, 189, 201, 200, 200, 199, 201, 201, 200, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685755803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130142024-02-12T01:40:07.811Z{"msg":"[2]R generation[0..12]: [110, 114, 117, 107, 117, 116, 116, 115, 117, 117, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685770062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130152024-02-12T01:40:07.811Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685791211Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130162024-02-12T01:40:07.811Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68580495Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130172024-02-12T01:40:07.811Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685825959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130182024-02-12T01:40:07.811Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685854338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130192024-02-12T01:40:07.811Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685899396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130202024-02-12T01:40:07.811Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685941304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130212024-02-12T01:40:07.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.685966843Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130222024-02-12T01:40:07.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686002341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130232024-02-12T01:40:07.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.68602737Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130242024-02-12T01:40:07.812Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686051688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130252024-02-12T01:40:07.812Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686085707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130262024-02-12T01:40:07.812Z{"msg":"a985b388-d453-4ed4-8bcf-b2c22033d492 is now active with session: 1301cea8-0522-4c81-a054-7208d1bd1223","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686121145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130272024-02-12T01:40:07.812Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686155103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607,"session_id":"1301cea8-0522-4c81-a054-7208d1bd1223"}
130282024-02-12T01:40:07.812Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:06.686181482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
130292024-02-12T01:40:07.812ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
130302024-02-12T01:40:07.812ZLoading write count information from file "/tmp/test_repair_verify.data"
130312024-02-12T01:40:07.812ZRun Repair workload
130322024-02-12T01:40:07.812Z01/30 Write block 125 len 36864 data: 13 15 14 11 10 9 12 12 10
130332024-02-12T01:40:07.812Z02/30 Read block 367 len 20480
130342024-02-12T01:40:07.812Z03/30 Read block 559 len 8192
130352024-02-12T01:40:07.812Z04/30 Read block 106 len 4096
130362024-02-12T01:40:07.812Z05/30 Write block 448 len 8192 data: 10 10
130372024-02-12T01:40:07.812Z06/30 Write block 199 len 20480 data: 7 6 6 8 9
130382024-02-12T01:40:07.812Z07/30 Write block 477 len 40960 data: 8 8 12 12 10 10 11 11 11 13
130392024-02-12T01:40:07.812Z08/30 Read block 414 len 28672
130402024-02-12T01:40:07.816Z09/30 Write block 481 len 16384 data: 11 11 12 12
130412024-02-12T01:40:07.816Z10/30 Write block 488 len 32768 data: 13 9 11 8 8 8 9 12
130422024-02-12T01:40:07.816Z11/30 Read block 329 len 4096
130432024-02-12T01:40:07.816Z12/30 Flush
130442024-02-12T01:40:07.819Z13/30 Write block 13 len 12288 data: 10 10 10
130452024-02-12T01:40:07.819Z14/30 Read block 539 len 24576
130462024-02-12T01:40:07.819Z15/30 Write block 495 len 28672 data: 13 10 9 10 10 8 9
130472024-02-12T01:40:07.819Z16/30 Read block 248 len 24576
130482024-02-12T01:40:07.822Z17/30 Write block 266 len 36864 data: 12 12 10 9 9 10 9 10 9
130492024-02-12T01:40:07.822Z18/30 Write block 479 len 28672 data: 13 13 12 12 13 13 12
130502024-02-12T01:40:07.822Z19/30 Write block 199 len 32768 data: 8 7 7 9 10 9 7 10
130512024-02-12T01:40:07.825Z20/30 Write block 183 len 12288 data: 9 10 7
130522024-02-12T01:40:07.825Z21/30 Write block 293 len 4096 data: 8
130532024-02-12T01:40:07.825Z22/30 Read block 201 len 32768
130542024-02-12T01:40:07.825Z23/30 Read block 325 len 16384
130552024-02-12T01:40:07.828Z24/30 Read block 62 len 12288
130562024-02-12T01:40:07.828Z25/30 Read block 238 len 36864
130572024-02-12T01:40:07.828Z26/30 Flush
130582024-02-12T01:40:07.831Z27/30 Write block 33 len 40960 data: 17 16 12 12 10 13 10 10 9 6
130592024-02-12T01:40:07.836Z28/30 Read block 439 len 36864
130602024-02-12T01:40:07.836Z29/30 Read block 38 len 8192
130612024-02-12T01:40:07.836Z30/30 Read block 80 len 28672
130622024-02-12T01:40:07.840Z----------------------------------------------------------------
130632024-02-12T01:40:07.840Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
130642024-02-12T01:40:07.840ZStates: Active Active Active
130652024-02-12T01:40:07.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130662024-02-12T01:40:07.840Z 1 Acked 1000 Write 9 Done Done Done false
130672024-02-12T01:40:07.840Z 2 Acked 1001 Read 5 Done Done Done false
130682024-02-12T01:40:07.840Z 3 Acked 1002 Read 2 Done Done Done false
130692024-02-12T01:40:07.840Z 4 Acked 1003 Read 1 Done Sent Done false
130702024-02-12T01:40:07.840Z 5 Acked 1004 Write 2 Done Sent Done false
130712024-02-12T01:40:07.840Z 6 Acked 1005 Write 5 Done Sent Done false
130722024-02-12T01:40:07.840Z 7 Acked 1006 Write 10 Done Sent Done false
130732024-02-12T01:40:07.840Z 8 Acked 1007 Read 7 Done Sent Done false
130742024-02-12T01:40:07.840Z 9 Acked 1008 Write 4 Done Sent Done false
130752024-02-12T01:40:07.840Z 10 Acked 1009 Write 8 Done Sent Done false
130762024-02-12T01:40:07.841Z 11 Acked 1010 Read 1 Done Sent Done false
130772024-02-12T01:40:07.841Z 12 Acked 1011 Flush 0 Done Sent Done false
130782024-02-12T01:40:07.841Z 13 Acked 1012 Write 3 Done Sent Done false
130792024-02-12T01:40:07.841Z 14 Acked 1013 Read 6 Done Sent Done false
130802024-02-12T01:40:07.841Z 15 Acked 1014 Write 7 Done Sent Done false
130812024-02-12T01:40:07.841Z 16 Acked 1015 Read 6 Done Sent Done false
130822024-02-12T01:40:07.841Z 17 Acked 1016 Write 9 Done Sent Done false
130832024-02-12T01:40:07.841Z 18 Acked 1017 Write 7 Done Sent Done false
130842024-02-12T01:40:07.841Z 19 Acked 1018 Write 8 Done Sent Done false
130852024-02-12T01:40:07.841Z 20 Acked 1019 Write 3 Done Sent Done false
130862024-02-12T01:40:07.841Z 21 Acked 1020 Write 1 Done Sent Done false
130872024-02-12T01:40:07.841Z 22 Acked 1021 Read 8 Done Sent Done false
130882024-02-12T01:40:07.841Z 23 Acked 1022 Read 4 Done Sent Done false
130892024-02-12T01:40:07.841Z 24 Acked 1023 Read 3 Done Sent Done false
130902024-02-12T01:40:07.841Z 25 Acked 1024 Read 9 Done Sent Done false
130912024-02-12T01:40:07.841Z 26 Acked 1025 Flush 0 Done Sent Done false
130922024-02-12T01:40:07.841Z 27 Acked 1026 Write 10 Done Sent Done false
130932024-02-12T01:40:07.841Z 28 Acked 1027 Read 9 Done Sent Done false
130942024-02-12T01:40:07.841Z 29 Acked 1028 Read 2 Done Sent Done false
130952024-02-12T01:40:07.841Z 30 Acked 1029 Read 7 Done Sent Sent false
130962024-02-12T01:40:07.841Z STATES DS:0 DS:1 DS:2 TOTAL
130972024-02-12T01:40:07.841Z New 0 0 0 0
130982024-02-12T01:40:07.841Z Sent 0 27 1 28
130992024-02-12T01:40:07.841Z Done 30 3 29 62
131002024-02-12T01:40:07.841Z Skipped 0 0 0 0
131012024-02-12T01:40:07.841Z Error 0 0 0 0
131022024-02-12T01:40:07.842ZLast Flush: 1025 0 1025
131032024-02-12T01:40:07.842ZDownstairs last five completed:
131042024-02-12T01:40:07.842ZUpstairs last five completed: 30 29 28 27 26
131052024-02-12T01:40:07.842Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:06.718868833Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1607}
131062024-02-12T01:40:07.842ZWrote out file "/tmp/test_repair_verify.data"
131072024-02-12T01:40:07.842Z
131082024-02-12T01:40:07.845ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131092024-02-12T01:40:07.859ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131102024-02-12T01:40:07.859Z 0 000-019 118 110 118 203 194 203 F F F <---
131112024-02-12T01:40:07.859Z 1 020-039 114 114 114 198 198 198 T F T <---
131122024-02-12T01:40:07.859Z 2 040-059 117 117 117 201 201 201 T F T <---
131132024-02-12T01:40:07.859Z 3 060-079 107 107 107 189 189 189 F F F
131142024-02-12T01:40:07.860Z 4 080-099 117 117 117 201 201 201 F F F
131152024-02-12T01:40:07.860Z 5 100-119 116 116 116 200 200 200 F F F
131162024-02-12T01:40:07.860Z 6 120-139 118 116 118 202 200 202 F T F <---
131172024-02-12T01:40:07.860Z 7 140-159 115 115 115 199 199 199 F F F
131182024-02-12T01:40:07.860Z 8 160-179 117 117 117 201 201 201 F F F
131192024-02-12T01:40:07.860Z 9 180-199 118 117 118 203 201 203 F F F <---
131202024-02-12T01:40:07.863Z 10 200-219 118 116 118 203 200 203 F F F <---
131212024-02-12T01:40:07.863Z 11 220-239 116 116 116 200 200 200 F F F
131222024-02-12T01:40:07.863Z 12 240-259 117 117 117 201 201 201 F F F
131232024-02-12T01:40:07.863Z 13 260-279 118 114 118 203 198 203 F F F <---
131242024-02-12T01:40:07.863Z 14 280-299 118 114 118 203 198 203 F F F <---
131252024-02-12T01:40:07.863Z 15 300-319 117 117 117 201 201 201 F F F
131262024-02-12T01:40:07.863Z 16 320-339 114 114 114 198 198 198 F F F
131272024-02-12T01:40:07.863Z 17 340-359 114 114 114 198 198 198 F F F
131282024-02-12T01:40:07.863Z 18 360-379 113 113 113 197 197 197 F F F
131292024-02-12T01:40:07.863Z 19 380-399 115 115 115 199 199 199 F F F
131302024-02-12T01:40:07.863Z 20 400-419 113 113 113 197 197 197 F F F
131312024-02-12T01:40:07.863Z 21 420-439 110 110 110 194 194 194 F F F
131322024-02-12T01:40:07.863Z 22 440-459 118 115 118 202 199 202 F F F <---
131332024-02-12T01:40:07.863Z 23 460-479 118 114 118 203 198 203 F F F <---
131342024-02-12T01:40:07.864Z 24 480-499 118 117 118 203 201 203 F F F <---
131352024-02-12T01:40:07.864Z 25 500-519 118 109 118 203 193 203 F F F <---
131362024-02-12T01:40:07.864Z 26 520-539 107 107 107 189 189 189 F F F
131372024-02-12T01:40:07.864Z 27 540-559 116 116 116 200 200 200 F F F
131382024-02-12T01:40:07.864Z 28 560-579 112 112 112 196 196 196 F F F
131392024-02-12T01:40:07.864Z 29 580-599 109 109 109 193 193 193 F F F
131402024-02-12T01:40:07.864ZMax gen: 118, Max flush: 203
131412024-02-12T01:40:07.864ZError: Difference in extent metadata found!
131422024-02-12T01:40:07.864ZA Difference in extent metadata is expected here
131432024-02-12T01:40:07.864ZOn loop 59
131442024-02-12T01:40:07.864Z
131452024-02-12T01:40:07.867ZVerifying data now
131462024-02-12T01:40:13.459ZNo change after verify
131472024-02-12T01:40:13.459ZLoop: 59 Downstairs dump after verify (and repair):
131482024-02-12T01:40:13.474ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131492024-02-12T01:40:13.474Z 0 000-019 118 118 118 203 203 203 F F F
131502024-02-12T01:40:13.474Z 1 020-039 119 119 119 204 204 204 F F F
131512024-02-12T01:40:13.474Z 2 040-059 119 119 119 204 204 204 F F F
131522024-02-12T01:40:13.474Z 3 060-079 107 107 107 189 189 189 F F F
131532024-02-12T01:40:13.474Z 4 080-099 117 117 117 201 201 201 F F F
131542024-02-12T01:40:13.474Z 5 100-119 116 116 116 200 200 200 F F F
131552024-02-12T01:40:13.474Z 6 120-139 118 118 118 202 202 202 F F F
131562024-02-12T01:40:13.474Z 7 140-159 115 115 115 199 199 199 F F F
131572024-02-12T01:40:13.474Z 8 160-179 117 117 117 201 201 201 F F F
131582024-02-12T01:40:13.481Z 9 180-199 118 118 118 203 203 203 F F F
131592024-02-12T01:40:13.481Z 10 200-219 118 118 118 203 203 203 F F F
131602024-02-12T01:40:13.481Z 11 220-239 116 116 116 200 200 200 F F F
131612024-02-12T01:40:13.481Z 12 240-259 117 117 117 201 201 201 F F F
131622024-02-12T01:40:13.481Z 13 260-279 118 118 118 203 203 203 F F F
131632024-02-12T01:40:13.481Z 14 280-299 118 118 118 203 203 203 F F F
131642024-02-12T01:40:13.481Z 15 300-319 117 117 117 201 201 201 F F F
131652024-02-12T01:40:13.481Z 16 320-339 114 114 114 198 198 198 F F F
131662024-02-12T01:40:13.481Z 17 340-359 114 114 114 198 198 198 F F F
131672024-02-12T01:40:13.481Z 18 360-379 113 113 113 197 197 197 F F F
131682024-02-12T01:40:13.481Z 19 380-399 115 115 115 199 199 199 F F F
131692024-02-12T01:40:13.481Z 20 400-419 113 113 113 197 197 197 F F F
131702024-02-12T01:40:13.481Z 21 420-439 110 110 110 194 194 194 F F F
131712024-02-12T01:40:13.481Z 22 440-459 118 118 118 202 202 202 F F F
131722024-02-12T01:40:13.481Z 23 460-479 118 118 118 203 203 203 F F F
131732024-02-12T01:40:13.481Z 24 480-499 118 118 118 203 203 203 F F F
131742024-02-12T01:40:13.481Z 25 500-519 118 118 118 203 203 203 F F F
131752024-02-12T01:40:13.481Z 26 520-539 107 107 107 189 189 189 F F F
131762024-02-12T01:40:13.481Z 27 540-559 116 116 116 200 200 200 F F F
131772024-02-12T01:40:13.481Z 28 560-579 112 112 112 196 196 196 F F F
131782024-02-12T01:40:13.482Z 29 580-599 109 109 109 193 193 193 F F F
131792024-02-12T01:40:13.482ZMax gen: 119, Max flush: 204
131802024-02-12T01:40:13.482Z
131812024-02-12T01:40:13.482ZBegin loop 60
131822024-02-12T01:40:13.482ZDownstairs to restart: 1
131832024-02-12T01:40:13.494Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377302775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
131842024-02-12T01:40:13.494Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377533274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
131852024-02-12T01:40:13.494Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377555623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
131862024-02-12T01:40:13.499Z{"msg":"Crucible 697311f8-d711-4bb9-83f7-30b15a6de128 has session id: c9ad9999-0480-46fb-a5e7-19bc00674870","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377590161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131872024-02-12T01:40:13.499Z{"msg":"Upstairs opts: Upstairs UUID: 697311f8-d711-4bb9-83f7-30b15a6de128, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"Crucible runtime is spawned
131882024-02-12T01:40:13.499Zcrucible","level":30,"time":"2024-02-12T01:40:12.377653388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131892024-02-12T01:40:13.499Z{"msg":"Crucible stats registered with UUID: 697311f8-d711-4bb9-83f7-30b15a6de128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377678557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131902024-02-12T01:40:13.499Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377737784Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
131912024-02-12T01:40:13.499Z{"msg":"697311f8-d711-4bb9-83f7-30b15a6de128 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.3778134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131922024-02-12T01:40:13.499Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377860658Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131932024-02-12T01:40:13.499Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377909136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131942024-02-12T01:40:13.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377938625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131952024-02-12T01:40:13.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.377969833Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131962024-02-12T01:40:13.500Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378000691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131972024-02-12T01:40:13.500Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.37803369Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131982024-02-12T01:40:13.500Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378065028Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
131992024-02-12T01:40:13.500Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.37823438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132002024-02-12T01:40:13.500Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378280228Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132012024-02-12T01:40:13.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378298137Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132022024-02-12T01:40:13.500Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378331486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132032024-02-12T01:40:13.500Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:12.378355885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132042024-02-12T01:40:13.500Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:12.378418202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132052024-02-12T01:40:13.500Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:12.37844784Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132062024-02-12T01:40:13.500Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378485718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132072024-02-12T01:40:13.500Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378502578Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132082024-02-12T01:40:13.500Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378531536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132092024-02-12T01:40:13.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378855911Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132102024-02-12T01:40:13.500Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.378954226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132112024-02-12T01:40:13.500Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.379008043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132122024-02-12T01:40:13.500Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.379062031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132132024-02-12T01:40:13.500Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.379433813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132142024-02-12T01:40:13.500Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.379536878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132152024-02-12T01:40:13.500Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.379991516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132162024-02-12T01:40:13.500Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.380027065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132172024-02-12T01:40:13.500Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.380042854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132182024-02-12T01:40:13.501Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.380056653Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132192024-02-12T01:40:13.501Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.380077812Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132202024-02-12T01:40:13.501Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:12.380091492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132212024-02-12T01:40:14.497Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.379752649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132222024-02-12T01:40:14.497Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.379828515Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132232024-02-12T01:40:14.502Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.380714763Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132242024-02-12T01:40:14.502Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.380747701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132252024-02-12T01:40:14.502Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381237118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132262024-02-12T01:40:14.502Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381745174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132272024-02-12T01:40:14.502Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381784152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132282024-02-12T01:40:14.502Z{"msg":"[0]R flush_numbers[0..12]: [203, 204, 204, 189, 201, 200, 202, 199, 201, 203, 203, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381807011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132292024-02-12T01:40:14.502Z{"msg":"[0]R generation[0..12]: [118, 119, 119, 107, 117, 116, 118, 115, 117, 118, 118, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.38182272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132302024-02-12T01:40:14.502Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381851009Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132312024-02-12T01:40:14.502Z{"msg":"[1]R flush_numbers[0..12]: [203, 204, 204, 189, 201, 200, 202, 199, 201, 203, 203, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381866888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132322024-02-12T01:40:14.502Z{"msg":"[1]R generation[0..12]: [118, 119, 119, 107, 117, 116, 118, 115, 117, 118, 118, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381881007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132332024-02-12T01:40:14.502Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381902056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132342024-02-12T01:40:14.502Z{"msg":"[2]R flush_numbers[0..12]: [203, 204, 204, 189, 201, 200, 202, 199, 201, 203, 203, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381916186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132352024-02-12T01:40:14.503Z{"msg":"[2]R generation[0..12]: [118, 119, 119, 107, 117, 116, 118, 115, 117, 118, 118, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381936414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132362024-02-12T01:40:14.503Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381951094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132372024-02-12T01:40:14.503Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381964453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132382024-02-12T01:40:14.503Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.381985262Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO
132392024-02-12T01:40:14.503Z,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132402024-02-12T01:40:14.503Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382011521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132412024-02-12T01:40:14.503Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.38202493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132422024-02-12T01:40:14.503Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382050059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132432024-02-12T01:40:14.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382062508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132442024-02-12T01:40:14.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382082427Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132452024-02-12T01:40:14.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382095877Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132462024-02-12T01:40:14.503Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382121336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132472024-02-12T01:40:14.503Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382134805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132482024-02-12T01:40:14.503Z{"msg":"697311f8-d711-4bb9-83f7-30b15a6de128 is now active with session: c9ad9999-0480-46fb-a5e7-19bc00674870","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382148494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132492024-02-12T01:40:14.503Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382161004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615,"session_id":"c9ad9999-0480-46fb-a5e7-19bc00674870"}
132502024-02-12T01:40:14.503Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:13.382173243Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
132512024-02-12T01:40:14.503ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132522024-02-12T01:40:14.503ZLoading write count information from file "/tmp/test_repair_verify.data"
132532024-02-12T01:40:14.503ZRun Repair workload
132542024-02-12T01:40:14.503Z01/30 Write block 561 len 20480 data: 9 8 11 10 9
132552024-02-12T01:40:14.507Z02/30 Read block 200 len 16384
132562024-02-12T01:40:14.507Z03/30 Read block 553 len 4096
132572024-02-12T01:40:14.507Z04/30 Write block 348 len 32768 data: 6 6 8 9 12 10 6 6
132582024-02-12T01:40:14.507Z05/30 Read block 348 len 36864
132592024-02-12T01:40:14.507Z06/30 Write block 313 len 16384 data: 9 6 7 6
132602024-02-12T01:40:14.507Z07/30 Write block 491 len 12288 data: 9 9 9
132612024-02-12T01:40:14.510Z08/30 Write block 257 len 32768 data: 8 8 7 10 7 5 7 7
132622024-02-12T01:40:14.510Z09/30 Read block 115 len 24576
132632024-02-12T01:40:14.510Z10/30 Read block 365 len 12288
132642024-02-12T01:40:14.515Z11/30 Read block 6 len 4096
132652024-02-12T01:40:14.515Z12/30 Write block 52 len 4096 data: 10
132662024-02-12T01:40:14.515Z13/30 Write block 183 len 36864 data: 10 11 8 7 10 12 12 10 9
132672024-02-12T01:40:14.515Z14/30 Read block 208 len 16384
132682024-02-12T01:40:14.518Z15/30 Write block 405 len 4096 data: 10
132692024-02-12T01:40:14.519Z16/30 Write block 310 len 36864 data: 12 10 8 10 7 8 7 8 9
132702024-02-12T01:40:14.519Z17/30 Write block 306 len 40960 data: 9 11 12 12 13 11 9 11 8 9
132712024-02-12T01:40:14.519Z18/30 Read block 66 len 12288
132722024-02-12T01:40:14.519Z19/30 Write block 256 len 20480 data: 8 9 9 8 11
132732024-02-12T01:40:14.522Z20/30 Write block 549 len 8192 data: 12 16
132742024-02-12T01:40:14.522Z21/30 Read block 9 len 20480
132752024-02-12T01:40:14.522Z22/30 Write block 322 len 32768 data: 10 8 8 7 9 9 9 8
132762024-02-12T01:40:14.522Z23/30 Read block 336 len 8192
132772024-02-12T01:40:14.525Z24/30 Write block 205 len 4096 data: 8
132782024-02-12T01:40:14.525Z25/30 Read block 312 len 32768
132792024-02-12T01:40:14.525Z26/30 Write block 187 len 16384 data: 11 13 13 11
132802024-02-12T01:40:14.525Z27/30 Write block 181 len 24576 data: 12 9 11 12 9 8
132812024-02-12T01:40:14.525Z28/30 Write block 182 len 4096 data: 10
132822024-02-12T01:40:14.529Z29/30 Read block 327 len 36864
132832024-02-12T01:40:14.529Z30/30 Write block 211 len 32768 data: 9 8 7 8 8 12 9 12
132842024-02-12T01:40:14.529Z----------------------------------------------------------------
132852024-02-12T01:40:14.529Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
132862024-02-12T01:40:14.529ZStates: Active Active Active
132872024-02-12T01:40:14.529ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132882024-02-12T01:40:14.529Z 1 Acked 1000 Write 5 Done Done Done false
132892024-02-12T01:40:14.529Z 2 Acked 1001 Read 4 Done Done Done false
132902024-02-12T01:40:14.529Z 3 Acked 1002 Read 1 Done Done Done false
132912024-02-12T01:40:14.529Z 4 Acked 1003 Write 8 Done Done Done false
132922024-02-12T01:40:14.529Z 5 Acked 1004 Read 9 Done Done Done false
132932024-02-12T01:40:14.529Z 6 Acked 1005 Write 4 Done Done Done false
132942024-02-12T01:40:14.529Z 7 Acked 1006 Write 3 Done Sent Done false
132952024-02-12T01:40:14.529Z 8 Acked 1007 Write 8 Done Sent Done false
132962024-02-12T01:40:14.529Z 9 Acked 1008 Read 6 Done Sent Done false
132972024-02-12T01:40:14.530Z 10 Acked 1009 Read 3 Done Sent Done false
132982024-02-12T01:40:14.530Z 11 Acked 1010 Read 1 Done Sent Done false
132992024-02-12T01:40:14.530Z 12 Acked 1011 Write 1 Done Sent Done false
133002024-02-12T01:40:14.530Z 13 Acked 1012 Write 9 Done Sent Done false
133012024-02-12T01:40:14.530Z 14 Acked 1013 Read 4 Done Sent Done false
133022024-02-12T01:40:14.530Z 15 Acked 1014 Write 1 Done Sent Done false
133032024-02-12T01:40:14.530Z 16 Acked 1015 Write 9 Done Sent Done false
133042024-02-12T01:40:14.530Z 17 Acked 1016 Write 10 Done Sent Done false
133052024-02-12T01:40:14.530Z 18 Acked 1017 Read 3 Done Sent Done false
133062024-02-12T01:40:14.530Z 19 Acked 1018 Write 5 Done Sent Done false
133072024-02-12T01:40:14.530Z 20 Acked 1019 Write 2 Done Sent Done false
133082024-02-12T01:40:14.530Z 21 Acked 1020 Read 5 Done Sent Done false
133092024-02-12T01:40:14.530Z 22 Acked 1021 Write 8 Done Sent Done false
133102024-02-12T01:40:14.530Z 23 Acked 1022 Read 2 Done Sent Done false
133112024-02-12T01:40:14.530Z 24 Acked 1023 Write 1 Done Sent Done false
133122024-02-12T01:40:14.530Z 25 Acked 1024 Read 8 Done Sent Done false
133132024-02-12T01:40:14.530Z 26 Acked 1025 Write 4 Done Sent Done false
133142024-02-12T01:40:14.530Z 27 Acked 1026 Write 6 Done Sent Done false
133152024-02-12T01:40:14.530Z 28 Acked 1027 Write 1 Done Sent Done false
133162024-02-12T01:40:14.530Z 29 Acked 1028 Read 9 Done Sent Done false
133172024-02-12T01:40:14.530Z 30 Acked 1029 Write 8 Sent Sent Sent false
133182024-02-12T01:40:14.530Z STATES DS:0 DS:1 DS:2 TOTAL
133192024-02-12T01:40:14.530Z New 0 0 0 0
133202024-02-12T01:40:14.530Z Sent 1 24 1 26
133212024-02-12T01:40:14.530Z Done 29 6 29 64
133222024-02-12T01:40:14.531Z Skipped 0 0 0 0
133232024-02-12T01:40:14.531Z Error 0 0 0 0
133242024-02-12T01:40:14.531ZLast Flush: 0 0 0
133252024-02-12T01:40:14.531ZDownstairs last five completed:
133262024-02-12T01:40:14.531ZUpstairs last five completed: 30 29 28 27 26
133272024-02-12T01:40:14.531Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:13.41054878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1615}
133282024-02-12T01:40:14.531ZWrote out file "/tmp/test_repair_verify.data"
133292024-02-12T01:40:14.534Z
133302024-02-12T01:40:14.537ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133312024-02-12T01:40:14.556ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133322024-02-12T01:40:14.556Z 0 000-019 118 118 118 203 203 203 F F F
133332024-02-12T01:40:14.556Z 1 020-039 119 119 119 204 204 204 F F F
133342024-02-12T01:40:14.556Z 2 040-059 119 119 119 204 204 204 T F T <---
133352024-02-12T01:40:14.556Z 3 060-079 107 107 107 189 189 189 F F F
133362024-02-12T01:40:14.556Z 4 080-099 117 117 117 201 201 201 F F F
133372024-02-12T01:40:14.557Z 5 100-119 116 116 116 200 200 200 F F F
133382024-02-12T01:40:14.557Z 6 120-139 118 118 118 202 202 202 F F F
133392024-02-12T01:40:14.557Z 7 140-159 115 115 115 199 199 199 F F F
133402024-02-12T01:40:14.557Z 8 160-179 117 117 117 201 201 201 F F F
133412024-02-12T01:40:14.557Z 9 180-199 118 118 118 203 203 203 T F T <---
133422024-02-12T01:40:14.560Z 10 200-219 118 118 118 203 203 203 T F T <---
133432024-02-12T01:40:14.560Z 11 220-239 116 116 116 200 200 200 F F F
133442024-02-12T01:40:14.560Z 12 240-259 117 117 117 201 201 201 T F T <---
133452024-02-12T01:40:14.560Z 13 260-279 118 118 118 203 203 203 T F T <---
133462024-02-12T01:40:14.560Z 14 280-299 118 118 118 203 203 203 F F F
133472024-02-12T01:40:14.560Z 15 300-319 117 117 117 201 201 201 T T T
133482024-02-12T01:40:14.560Z 16 320-339 114 114 114 198 198 198 T F T <---
133492024-02-12T01:40:14.560Z 17 340-359 114 114 114 198 198 198 T T T
133502024-02-12T01:40:14.560Z 18 360-379 113 113 113 197 197 197 F F F
133512024-02-12T01:40:14.560Z 19 380-399 115 115 115 199 199 199 F F F
133522024-02-12T01:40:14.560Z 20 400-419 113 113 113 197 197 197 T F T <---
133532024-02-12T01:40:14.560Z 21 420-439 110 110 110 194 194 194 F F F
133542024-02-12T01:40:14.560Z 22 440-459 118 118 118 202 202 202 F F F
133552024-02-12T01:40:14.561Z 23 460-479 118 118 118 203 203 203 F F F
133562024-02-12T01:40:14.561Z 24 480-499 118 118 118 203 203 203 T F T <---
133572024-02-12T01:40:14.561Z 25 500-519 118 118 118 203 203 203 F F F
133582024-02-12T01:40:14.561Z 26 520-539 107 107 107 189 189 189 F F F
133592024-02-12T01:40:14.561Z 27 540-559 116 116 116 200 200 200 T F T <---
133602024-02-12T01:40:14.561Z 28 560-579 112 112 112 196 196 196 T T T
133612024-02-12T01:40:14.561Z 29 580-599 109 109 109 193 193 193 F F F
133622024-02-12T01:40:14.561ZMax gen: 119, Max flush: 204
133632024-02-12T01:40:14.561ZError: Difference in extent metadata found!
133642024-02-12T01:40:14.561ZA Difference in extent metadata is expected here
133652024-02-12T01:40:14.561ZOn loop 60
133662024-02-12T01:40:14.561Z
133672024-02-12T01:40:14.564ZVerifying data now
133682024-02-12T01:40:20.175ZNo change after verify
133692024-02-12T01:40:20.175ZLoop: 60 Downstairs dump after verify (and repair):
133702024-02-12T01:40:20.190ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133712024-02-12T01:40:20.190Z 0 000-019 118 118 118 203 203 203 F F F
133722024-02-12T01:40:20.190Z 1 020-039 119 119 119 204 204 204 F F F
133732024-02-12T01:40:20.190Z 2 040-059 120 120 120 205 205 205 F F F
133742024-02-12T01:40:20.190Z 3 060-079 107 107 107 189 189 189 F F F
133752024-02-12T01:40:20.190Z 4 080-099 117 117 117 201 201 201 F F F
133762024-02-12T01:40:20.190Z 5 100-119 116 116 116 200 200 200 F F F
133772024-02-12T01:40:20.190Z 6 120-139 118 118 118 202 202 202 F F F
133782024-02-12T01:40:20.190Z 7 140-159 115 115 115 199 199 199 F F F
133792024-02-12T01:40:20.190Z 8 160-179 117 117 117 201 201 201 F F F
133802024-02-12T01:40:20.194Z 9 180-199 120 120 120 205 205 205 F F F
133812024-02-12T01:40:20.194Z 10 200-219 120 120 120 205 205 205 F F F
133822024-02-12T01:40:20.194Z 11 220-239 116 116 116 200 200 200 F F F
133832024-02-12T01:40:20.194Z 12 240-259 120 120 120 205 205 205 F F F
133842024-02-12T01:40:20.194Z 13 260-279 120 120 120 205 205 205 F F F
133852024-02-12T01:40:20.194Z 14 280-299 118 118 118 203 203 203 F F F
133862024-02-12T01:40:20.194Z 15 300-319 120 120 120 205 205 205 F F F
133872024-02-12T01:40:20.194Z 16 320-339 120 120 120 205 205 205 F F F
133882024-02-12T01:40:20.194Z 17 340-359 120 120 120 205 205 205 F F F
133892024-02-12T01:40:20.194Z 18 360-379 113 113 113 197 197 197 F F F
133902024-02-12T01:40:20.194Z 19 380-399 115 115 115 199 199 199 F F F
133912024-02-12T01:40:20.194Z 20 400-419 120 120 120 205 205 205 F F F
133922024-02-12T01:40:20.194Z 21 420-439 110 110 110 194 194 194 F F F
133932024-02-12T01:40:20.194Z 22 440-459 118 118 118 202 202 202 F F F
133942024-02-12T01:40:20.194Z 23 460-479 118 118 118 203 203 203 F F F
133952024-02-12T01:40:20.194Z 24 480-499 120 120 120 205 205 205 F F F
133962024-02-12T01:40:20.194Z 25 500-519 118 118 118 203 203 203 F F F
133972024-02-12T01:40:20.194Z 26 520-539 107 107 107 189 189 189 F F F
133982024-02-12T01:40:20.194Z 27 540-559 120 120 120 205 205 205 F F F
133992024-02-12T01:40:20.194Z 28 560-579 120 120 120 205 205 205 F F F
134002024-02-12T01:40:20.194Z 29 580-599 109 109 109 193 193 193 F F F
134012024-02-12T01:40:20.195ZMax gen: 120, Max flush: 205
134022024-02-12T01:40:20.195Z
134032024-02-12T01:40:20.195ZBegin loop 61
134042024-02-12T01:40:20.195ZDownstairs to restart: 2
134052024-02-12T01:40:20.212Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094085213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
134062024-02-12T01:40:20.212Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"Crucible runtime is spawned
134072024-02-12T01:40:20.216Z82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094427277Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
134082024-02-12T01:40:20.217Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094484425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
134092024-02-12T01:40:20.217Z{"msg":"Crucible a8d975f8-fb8e-4232-942b-0e7a07705487 has session id: 9bc779a5-4d2c-45e7-9d36-3fe0306833c2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094510003Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134102024-02-12T01:40:20.217Z{"msg":"Upstairs opts: Upstairs UUID: a8d975f8-fb8e-4232-942b-0e7a07705487, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094546662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134112024-02-12T01:40:20.217Z{"msg":"Crucible stats registered with UUID: a8d975f8-fb8e-4232-942b-0e7a07705487","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094588109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134122024-02-12T01:40:20.217Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094628638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
134132024-02-12T01:40:20.217Z{"msg":"a8d975f8-fb8e-4232-942b-0e7a07705487 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094665086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134142024-02-12T01:40:20.217Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094713223Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134152024-02-12T01:40:20.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094761851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134162024-02-12T01:40:20.217Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094820978Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134172024-02-12T01:40:20.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094883615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134182024-02-12T01:40:20.217Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094919054Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134192024-02-12T01:40:20.217Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.094953222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134202024-02-12T01:40:20.217Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.09498575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134212024-02-12T01:40:20.217Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095029508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134222024-02-12T01:40:20.217Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095083886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134232024-02-12T01:40:20.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095125714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134242024-02-12T01:40:20.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095172422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134252024-02-12T01:40:20.217Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.09519884Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134262024-02-12T01:40:20.217Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095713556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134272024-02-12T01:40:20.217Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095744054Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134282024-02-12T01:40:20.217Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.095804851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134292024-02-12T01:40:20.218Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.09584215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134302024-02-12T01:40:20.218Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.096284768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134312024-02-12T01:40:20.218Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.096412393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134322024-02-12T01:40:20.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.096844402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134332024-02-12T01:40:20.218Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.09689579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134342024-02-12T01:40:20.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.096941437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134352024-02-12T01:40:20.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.096977395Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134362024-02-12T01:40:20.218Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.097020284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134372024-02-12T01:40:20.218Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.097049482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134382024-02-12T01:40:20.218Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.097186996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134392024-02-12T01:40:20.218Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.097233573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134402024-02-12T01:40:20.222Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.097696641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134412024-02-12T01:40:20.222Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098131891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134422024-02-12T01:40:20.222Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098176858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134432024-02-12T01:40:20.222Z{"msg":"[0]R flush_numbers[0..12]: [203, 204, 205, 189, 201, 200, 202, 199, 201, 205, 205, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098214887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134442024-02-12T01:40:20.222Z{"msg":"[0]R generation[0..12]: [118, 119, 120, 107, 117, 116, 118, 115, 117, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098240305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134452024-02-12T01:40:20.222Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098275924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134462024-02-12T01:40:20.222Z{"msg":"[1]R flush_numbers[0..12]: [203, 204, 205, 189, 201, 200, 202, 199, 201, 205, 205, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098313232Z","Wait for a query_work_queue command to finish before sending IO
134472024-02-12T01:40:20.222Zhostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134482024-02-12T01:40:20.222Z{"msg":"[1]R generation[0..12]: [118, 119, 120, 107, 117, 116, 118, 115, 117, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.09835518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134492024-02-12T01:40:20.222Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098392748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134502024-02-12T01:40:20.222Z{"msg":"[2]R flush_numbers[0..12]: [203, 204, 205, 189, 201, 200, 202, 199, 201, 205, 205, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098431556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134512024-02-12T01:40:20.222Z{"msg":"[2]R generation[0..12]: [118, 119, 120, 107, 117, 116, 118, 115, 117, 120, 120, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098469344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134522024-02-12T01:40:20.222Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098535361Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134532024-02-12T01:40:20.222Z{"msg":"Max found gen is 121","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098581599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134542024-02-12T01:40:20.223Z{"msg":"Generation requested: 122 >= found:121","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098621587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134552024-02-12T01:40:20.223Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098651516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134562024-02-12T01:40:20.223Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098668725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134572024-02-12T01:40:20.223Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098704043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134582024-02-12T01:40:20.223Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098732612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134592024-02-12T01:40:20.223Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
134602024-02-12T01:40:20.223Zcrucible","level":30,"time":"2024-02-12T01:40:19.0987676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134612024-02-12T01:40:20.223Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098803358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134622024-02-12T01:40:20.223Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098837227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134632024-02-12T01:40:20.223Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098869235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134642024-02-12T01:40:20.223Z{"msg":"a8d975f8-fb8e-4232-942b-0e7a07705487 is now active with session: 9bc779a5-4d2c-45e7-9d36-3fe0306833c2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098900534Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134652024-02-12T01:40:20.223Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098929472Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623,"session_id":"9bc779a5-4d2c-45e7-9d36-3fe0306833c2"}
134662024-02-12T01:40:20.223Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:19.098959651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
134672024-02-12T01:40:20.223ZLoading write count information from file "/tmp/test_repair_verify.data"
134682024-02-12T01:40:20.223ZRun Repair workload
134692024-02-12T01:40:20.223Z01/30 Write block 253 len 40960 data: 7 7 9 9 10 10 9 12 8 6
134702024-02-12T01:40:20.223Z02/30 Flush
134712024-02-12T01:40:20.226Z03/30 Write block 532 len 28672 data: 11 13 12 13 13 13 12
134722024-02-12T01:40:20.226Z04/30 Read block 206 len 8192
134732024-02-12T01:40:20.226Z05/30 Write block 140 len 16384 data: 11 15 17 16
134742024-02-12T01:40:20.227Z06/30 Write block 38 len 4096 data: 14
134752024-02-12T01:40:20.227Z07/30 Write block 386 len 20480 data: 13 12 12 11 15
134762024-02-12T01:40:20.227Z08/30 Read block 115 len 4096
134772024-02-12T01:40:20.227Z09/30 Write block 159 len 24576 data: 7 10 9 11 12 12
134782024-02-12T01:40:20.233Z10/30 Read block 528 len 8192
134792024-02-12T01:40:20.233Z11/30 Flush
134802024-02-12T01:40:20.236Z12/30 Write block 417 len 24576 data: 10 7 6 6 8 7
134812024-02-12T01:40:20.236Z13/30 Write block 139 len 8192 data: 12 12
134822024-02-12T01:40:20.236Z14/30 Write block 116 len 4096 data: 13
134832024-02-12T01:40:20.236Z15/30 Write block 136 len 28672 data: 11 10 12 13 13 16 18
134842024-02-12T01:40:20.236Z16/30 Write block 47 len 24576 data: 12 13 14 13 14 11
134852024-02-12T01:40:20.236Z17/30 Read block 317 len 4096
134862024-02-12T01:40:20.236Z18/30 Read block 508 len 40960
134872024-02-12T01:40:20.239Z19/30 Read block 359 len 32768
134882024-02-12T01:40:20.239Z20/30 Write block 288 len 8192 data: 9 8
134892024-02-12T01:40:20.239Z21/30 Read block 127 len 16384
134902024-02-12T01:40:20.242Z22/30 Write block 202 len 16384 data: 10 11 10 9
134912024-02-12T01:40:20.242Z23/30 Read block 481 len 16384
134922024-02-12T01:40:20.242Z24/30 Write block 505 len 8192 data: 7 5
134932024-02-12T01:40:20.242Z25/30 Write block 207 len 40960 data: 12 12 10 8 10 9 8 9 9 13
134942024-02-12T01:40:20.245Z26/30 Write block 75 len 20480 data: 8 9 8 8 11
134952024-02-12T01:40:20.245Z27/30 Read block 134 len 40960
134962024-02-12T01:40:20.245Z28/30 Write block 277 len 24576 data: 10 12 13 12 10 13
134972024-02-12T01:40:20.245Z29/30 Write block 82 len 12288 data: 15 13 14
134982024-02-12T01:40:20.248Z30/30 Write block 420 len 20480 data: 7 9 8 6 9
134992024-02-12T01:40:20.248Z----------------------------------------------------------------
135002024-02-12T01:40:20.248Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:28
135012024-02-12T01:40:20.249ZStates: Active Active Active
135022024-02-12T01:40:20.249ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135032024-02-12T01:40:20.249Z 3 Acked 1002 Write 7 Done Done Sent false
135042024-02-12T01:40:20.249Z 4 Acked 1003 Read 2 Done Done Sent false
135052024-02-12T01:40:20.249Z 5 Acked 1004 Write 4 Done Done Sent false
135062024-02-12T01:40:20.249Z 6 Acked 1005 Write 1 Done Done Sent false
135072024-02-12T01:40:20.249Z 7 Acked 1006 Write 5 Done Done Sent false
135082024-02-12T01:40:20.249Z 8 Acked 1007 Read 1 Done Done Sent false
135092024-02-12T01:40:20.249Z 9 Acked 1008 Write 6 Done Done Sent false
135102024-02-12T01:40:20.249Z 10 Acked 1009 Read 2 Done Done Sent false
135112024-02-12T01:40:20.249Z 11 Acked 1010 Flush 0 Done Done Sent false
135122024-02-12T01:40:20.249Z 12 Acked 1011 Write 6 Done Done Sent false
135132024-02-12T01:40:20.249Z 13 Acked 1012 Write 2 Done Done Sent false
135142024-02-12T01:40:20.249Z 14 Acked 1013 Write 1 Done Done Sent false
135152024-02-12T01:40:20.249Z 15 Acked 1014 Write 7 Done Done Sent false
135162024-02-12T01:40:20.249Z 16 Acked 1015 Write 6 Done Done Sent false
135172024-02-12T01:40:20.249Z 17 Acked 1016 Read 1 Done Done Sent false
135182024-02-12T01:40:20.249Z 18 Acked 1017 Read 10 Done Done Sent false
135192024-02-12T01:40:20.249Z 19 Acked 1018 Read 8 Done Done Sent false
135202024-02-12T01:40:20.249Z 20 Acked 1019 Write 2 Done Done Sent false
135212024-02-12T01:40:20.249Z 21 Acked 1020 Read 4 Done Done Sent false
135222024-02-12T01:40:20.249Z 22 Acked 1021 Write 4 Done Done Sent false
135232024-02-12T01:40:20.250Z 23 Acked 1022 Read 4 Done Done Sent false
135242024-02-12T01:40:20.250Z 24 Acked 1023 Write 2 Done Done Sent false
135252024-02-12T01:40:20.250Z 25 Acked 1024 Write 10 Done Done Sent false
135262024-02-12T01:40:20.250Z 26 Acked 1025 Write 5 Done Done Sent false
135272024-02-12T01:40:20.250Z 27 Acked 1026 Read 10 Done Done Sent false
135282024-02-12T01:40:20.250Z 28 Acked 1027 Write 6 Done Done Sent false
135292024-02-12T01:40:20.250Z 29 Acked 1028 Write 3 Done Done Sent false
135302024-02-12T01:40:20.250Z 30 Acked 1029 Write 5 Sent Sent Sent false
135312024-02-12T01:40:20.250Z STATES DS:0 DS:1 DS:2 TOTAL
135322024-02-12T01:40:20.250Z New 0 0 0 0
135332024-02-12T01:40:20.250Z Sent 1 1 28 30
135342024-02-12T01:40:20.250Z Done 27 27 0 54
135352024-02-12T01:40:20.250Z Skipped 0 0 0 0
135362024-02-12T01:40:20.250Z Error 0 0 0 0
135372024-02-12T01:40:20.250ZLast Flush: 1010 1010 1001
135382024-02-12T01:40:20.250ZDownstairs last five completed: 1001 1000
135392024-02-12T01:40:20.250ZUpstairs last five completed: 30 29 28 27 26
135402024-02-12T01:40:20.250Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:19.128791308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1623}
135412024-02-12T01:40:20.250ZWrote out file "/tmp/test_repair_verify.data"
135422024-02-12T01:40:20.254Z
135432024-02-12T01:40:20.257ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
135442024-02-12T01:40:20.275ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135452024-02-12T01:40:20.275Z 0 000-019 118 118 118 203 203 203 F F F
135462024-02-12T01:40:20.275Z 1 020-039 122 122 119 207 207 204 F F F <---
135472024-02-12T01:40:20.275Z 2 040-059 120 120 120 205 205 205 T T F <---
135482024-02-12T01:40:20.275Z 3 060-079 107 107 107 189 189 189 T T F <---
135492024-02-12T01:40:20.275Z 4 080-099 117 117 117 201 201 201 T T F <---
135502024-02-12T01:40:20.275Z 5 100-119 116 116 116 200 200 200 T T F <---
135512024-02-12T01:40:20.275Z 6 120-139 118 118 118 202 202 202 T T F <---
135522024-02-12T01:40:20.275Z 7 140-159 122 122 115 207 207 199 T T F <---
135532024-02-12T01:40:20.275Z 8 160-179 122 122 117 207 207 201 F F F <---
135542024-02-12T01:40:20.278Z 9 180-199 120 120 120 205 205 205 F F F
135552024-02-12T01:40:20.278Z 10 200-219 120 120 120 205 205 205 T T F <---
135562024-02-12T01:40:20.279Z 11 220-239 116 116 116 200 200 200 F F F
135572024-02-12T01:40:20.279Z 12 240-259 122 122 122 206 206 206 F F F
135582024-02-12T01:40:20.279Z 13 260-279 122 122 122 206 206 206 T T F <---
135592024-02-12T01:40:20.279Z 14 280-299 118 118 118 203 203 203 T T F <---
135602024-02-12T01:40:20.279Z 15 300-319 120 120 120 205 205 205 F F F
135612024-02-12T01:40:20.279Z 16 320-339 120 120 120 205 205 205 F F F
135622024-02-12T01:40:20.279Z 17 340-359 120 120 120 205 205 205 F F F
135632024-02-12T01:40:20.279Z 18 360-379 113 113 113 197 197 197 F F F
135642024-02-12T01:40:20.279Z 19 380-399 122 122 115 207 207 199 F F F <---
135652024-02-12T01:40:20.279Z 20 400-419 120 120 120 205 205 205 T T F <---
135662024-02-12T01:40:20.279Z 21 420-439 110 110 110 194 194 194 T T F <---
135672024-02-12T01:40:20.279Z 22 440-459 118 118 118 202 202 202 F F F
135682024-02-12T01:40:20.279Z 23 460-479 118 118 118 203 203 203 F F F
135692024-02-12T01:40:20.279Z 24 480-499 120 120 120 205 205 205 F F F
135702024-02-12T01:40:20.279Z 25 500-519 118 118 118 203 203 203 T T F <---
135712024-02-12T01:40:20.279Z 26 520-539 122 122 107 207 207 189 F F F <---
135722024-02-12T01:40:20.279Z 27 540-559 120 120 120 205 205 205 F F F
135732024-02-12T01:40:20.279Z 28 560-579 120 120 120 205 205 205 F F F
135742024-02-12T01:40:20.279Z 29 580-599 109 109 109 193 193 193 F F F
135752024-02-12T01:40:20.279ZMax gen: 122, Max flush: 207
135762024-02-12T01:40:20.279ZError: Difference in extent metadata found!
135772024-02-12T01:40:20.279ZA Difference in extent metadata is expected here
135782024-02-12T01:40:20.279ZOn loop 61
135792024-02-12T01:40:20.279Z
135802024-02-12T01:40:20.283ZVerifying data now
135812024-02-12T01:40:25.057ZNo change after verify
135822024-02-12T01:40:25.057ZLoop: 61 Downstairs dump after verify (and repair):
135832024-02-12T01:40:25.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135842024-02-12T01:40:25.072Z 0 000-019 118 118 118 203 203 203 F F F
135852024-02-12T01:40:25.072Z 1 020-039 122 122 122 207 207 207 F F F
135862024-02-12T01:40:25.072Z 2 040-059 123 123 123 208 208 208 F F F
135872024-02-12T01:40:25.072Z 3 060-079 123 123 123 208 208 208 F F F
135882024-02-12T01:40:25.072Z 4 080-099 123 123 123 208 208 208 F F F
135892024-02-12T01:40:25.075Z 5 100-119 123 123 123 208 208 208 F F F
135902024-02-12T01:40:25.075Z 6 120-139 123 123 123 208 208 208 F F F
135912024-02-12T01:40:25.075Z 7 140-159 123 123 123 208 208 208 F F F
135922024-02-12T01:40:25.075Z 8 160-179 122 122 122 207 207 207 F F F
135932024-02-12T01:40:25.075Z 9 180-199 120 120 120 205 205 205 F F F
135942024-02-12T01:40:25.075Z 10 200-219 123 123 123 208 208 208 F F F
135952024-02-12T01:40:25.075Z 11 220-239 116 116 116 200 200 200 F F F
135962024-02-12T01:40:25.076Z 12 240-259 122 122 122 206 206 206 F F F
135972024-02-12T01:40:25.076Z 13 260-279 123 123 123 208 208 208 F F F
135982024-02-12T01:40:25.076Z 14 280-299 123 123 123 208 208 208 F F F
135992024-02-12T01:40:25.076Z 15 300-319 120 120 120 205 205 205 F F F
136002024-02-12T01:40:25.076Z 16 320-339 120 120 120 205 205 205 F F F
136012024-02-12T01:40:25.076Z 17 340-359 120 120 120 205 205 205 F F F
136022024-02-12T01:40:25.076Z 18 360-379 113 113 113 197 197 197 F F F
136032024-02-12T01:40:25.076Z 19 380-399 122 122 122 207 207 207 F F F
136042024-02-12T01:40:25.076Z 20 400-419 123 123 123 208 208 208 F F F
136052024-02-12T01:40:25.076Z 21 420-439 123 123 123 208 208 208 F F F
136062024-02-12T01:40:25.076Z 22 440-459 118 118 118 202 202 202 F F F
136072024-02-12T01:40:25.076Z 23 460-479 118 118 118 203 203 203 F F F
136082024-02-12T01:40:25.076Z 24 480-499 120 120 120 205 205 205 F F F
136092024-02-12T01:40:25.076Z 25 500-519 123 123 123 208 208 208 F F F
136102024-02-12T01:40:25.076Z 26 520-539 122 122 122 207 207 207 F F F
136112024-02-12T01:40:25.076Z 27 540-559 120 120 120 205 205 205 F F F
136122024-02-12T01:40:25.076Z 28 560-579 120 120 120 205 205 205 F F F
136132024-02-12T01:40:25.076Z 29 580-599 109 109 109 193 193 193 F F F
136142024-02-12T01:40:25.076ZMax gen: 123, Max flush: 208
136152024-02-12T01:40:25.076Z
136162024-02-12T01:40:25.076ZBegin loop 62
136172024-02-12T01:40:25.076ZDownstairs to restart: 0
136182024-02-12T01:40:25.096Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.974578219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
136192024-02-12T01:40:25.096Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
136202024-02-12T01:40:25.096Z,"v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.974900154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
136212024-02-12T01:40:25.101Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.974946202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
136222024-02-12T01:40:25.101Z{"msg":"Crucible fcc24bfc-04d6-461b-85a4-9f1945746bc4 has session id: 26547342-40a5-48d8-878d-e517df378c7a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97497996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136232024-02-12T01:40:25.101Z{"msg":"Upstairs opts: Upstairs UUID: fcc24bfc-04d6-461b-85a4-9f1945746bc4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975016968Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136242024-02-12T01:40:25.101Z{"msg":"Crucible stats registered with UUID: fcc24bfc-04d6-461b-85a4-9f1945746bc4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975055706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136252024-02-12T01:40:25.101Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975071366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
136262024-02-12T01:40:25.102Z{"msg":"fcc24bfc-04d6-461b-85a4-9f1945746bc4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975091305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136272024-02-12T01:40:25.102Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975104094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136282024-02-12T01:40:25.102Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975130593Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136292024-02-12T01:40:25.102Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975152102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136302024-02-12T01:40:25.102Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.9751801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136312024-02-12T01:40:25.102Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97520057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136322024-02-12T01:40:25.102Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975235628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136332024-02-12T01:40:25.102Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975271666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136342024-02-12T01:40:25.102Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975311854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136352024-02-12T01:40:25.102Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975347632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136362024-02-12T01:40:25.102Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975433538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136372024-02-12T01:40:25.102Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975467997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136382024-02-12T01:40:25.102Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.975495556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136392024-02-12T01:40:25.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976156004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136402024-02-12T01:40:25.102Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976222031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136412024-02-12T01:40:25.102Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976261299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136422024-02-12T01:40:25.102Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976298937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136432024-02-12T01:40:25.102Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976698618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136442024-02-12T01:40:25.102Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.976843861Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136452024-02-12T01:40:25.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977210184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136462024-02-12T01:40:25.102Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977238742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136472024-02-12T01:40:25.102Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977254052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136482024-02-12T01:40:25.102Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97729189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136492024-02-12T01:40:25.103Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977314909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136502024-02-12T01:40:25.103Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977328948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136512024-02-12T01:40:25.103Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977538678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136522024-02-12T01:40:25.103Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.977574876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136532024-02-12T01:40:25.103Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978056293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136542024-02-12T01:40:25.103Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978453634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136552024-02-12T01:40:25.103Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978500312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136562024-02-12T01:40:25.103Z{"msg":"[0]R flush_numbers[0..12]: [203, 207, 208, 208, 208, 208, 208, 208, 207, 205, 208, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978528871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136572024-02-12T01:40:25.103Z{"msg":"[0]R generation[0..12]: [118, 122, 123, 123, 123, 123, 123, 123, 122, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97854415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136582024-02-12T01:40:25.103Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978565669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136592024-02-12T01:40:25.103Z{"msg":"[1]R flush_numbers[0..12]: [203, 207, 208, 208, 208, 208, 208, 208, 207, 205, 208, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978580228Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136602024-02-12T01:40:25.103Z{"msg":"[1]R generation[0..12]: [118, 122, 123, 123, 123, 123, 123, 123, 122, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978600117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136612024-02-12T01:40:25.103Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978614367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136622024-02-12T01:40:25.103ZWait for a query_work_queue command to finish before sending IO
136632024-02-12T01:40:25.103Z{"msg":"[2]R flush_numbers[0..12]: [203, 207, 208, 208, 208, 208, 208, 208, 207, 205, 208, 200]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978640625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136642024-02-12T01:40:25.103Z{"msg":"[2]R generation[0..12]: [118, 122, 123, 123, 123, 123, 123, 123, 122, 120, 123, 116]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978655585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136652024-02-12T01:40:25.103Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978688973Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136662024-02-12T01:40:25.103Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978721271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136672024-02-12T01:40:25.103Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97875709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136682024-02-12T01:40:25.103Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978775869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136692024-02-12T01:40:25.103Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978808997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136702024-02-12T01:40:25.104Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978847436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136712024-02-12T01:40:25.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978885754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136722024-02-12T01:40:25.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978923822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136732024-02-12T01:40:25.104Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.97896034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136742024-02-12T01:40:25.104Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.978996908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136752024-02-12T01:40:25.104ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
136762024-02-12T01:40:25.104Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.979036007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136772024-02-12T01:40:25.104Z{"msg":"fcc24bfc-04d6-461b-85a4-9f1945746bc4 is now active with session: 26547342-40a5-48d8-878d-e517df378c7a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.979062455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136782024-02-12T01:40:25.104Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.979075575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631,"session_id":"26547342-40a5-48d8-878d-e517df378c7a"}
136792024-02-12T01:40:25.104Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:23.979095024Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
136802024-02-12T01:40:25.104ZLoading write count information from file "/tmp/test_repair_verify.data"
136812024-02-12T01:40:25.104ZRun Repair workload
136822024-02-12T01:40:25.104Z01/30 Write block 69 len 8192 data: 12 11
136832024-02-12T01:40:25.104Z02/30 Flush
136842024-02-12T01:40:25.107Z03/30 Write block 475 len 40960 data: 10 7 9 9 14 14 13 13 14 14
136852024-02-12T01:40:25.107Z04/30 Write block 79 len 16384 data: 12 14 14 16
136862024-02-12T01:40:25.108Z05/30 Read block 87 len 36864
136872024-02-12T01:40:25.108Z06/30 Read block 88 len 16384
136882024-02-12T01:40:25.108Z07/30 Write block 147 len 4096 data: 11
136892024-02-12T01:40:25.108Z08/30 Read block 304 len 36864
136902024-02-12T01:40:25.108Z09/30 Read block 263 len 24576
136912024-02-12T01:40:25.108Z10/30 Flush
136922024-02-12T01:40:25.111Z11/30 Write block 526 len 40960 data: 6 4 6 7 11 11 12 14 13 14
136932024-02-12T01:40:25.114Z12/30 Write block 573 len 12288 data: 15 14 13
136942024-02-12T01:40:25.114Z13/30 Read block 68 len 16384
136952024-02-12T01:40:25.114Z14/30 Flush
136962024-02-12T01:40:25.116Z15/30 Write block 74 len 8192 data: 7 9
136972024-02-12T01:40:25.117Z16/30 Read block 464 len 24576
136982024-02-12T01:40:25.117Z17/30 Read block 231 len 24576
136992024-02-12T01:40:25.120Z18/30 Write block 441 len 4096 data: 9
137002024-02-12T01:40:25.120Z19/30 Read block 310 len 20480
137012024-02-12T01:40:25.120Z20/30 Read block 177 len 36864
137022024-02-12T01:40:25.123Z21/30 Write block 247 len 40960 data: 8 7 6 5 7 6 8 8 10 10
137032024-02-12T01:40:25.123Z22/30 Write block 221 len 20480 data: 10 10 10 8 7
137042024-02-12T01:40:25.123Z23/30 Write block 270 len 20480 data: 10 11 10 11 10
137052024-02-12T01:40:25.123Z24/30 Read block 266 len 4096
137062024-02-12T01:40:25.123Z25/30 Read block 298 len 20480
137072024-02-12T01:40:25.127Z26/30 Read block 32 len 24576
137082024-02-12T01:40:25.127Z27/30 Read block 439 len 24576
137092024-02-12T01:40:25.127Z28/30 Read block 403 len 24576
137102024-02-12T01:40:25.127Z29/30 Write block 52 len 20480 data: 12 9 9 9 9
137112024-02-12T01:40:25.130Z30/30 Read block 432 len 8192
137122024-02-12T01:40:25.130Z----------------------------------------------------------------
137132024-02-12T01:40:25.130Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:28
137142024-02-12T01:40:25.130ZStates: Active Active Active
137152024-02-12T01:40:25.130ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137162024-02-12T01:40:25.130Z 3 Acked 1002 Write 10 Done Done Done false
137172024-02-12T01:40:25.130Z 4 Acked 1003 Write 4 Done Done Done false
137182024-02-12T01:40:25.130Z 5 Acked 1004 Read 9 Sent Done Done false
137192024-02-12T01:40:25.130Z 6 Acked 1005 Read 4 Sent Done Done false
137202024-02-12T01:40:25.131Z 7 Acked 1006 Write 1 Sent Done Done false
137212024-02-12T01:40:25.131Z 8 Acked 1007 Read 9 Sent Done Done false
137222024-02-12T01:40:25.131Z 9 Acked 1008 Read 6 Sent Done Done false
137232024-02-12T01:40:25.131Z 10 Acked 1009 Flush 0 Sent Done Done false
137242024-02-12T01:40:25.131Z 11 Acked 1010 Write 10 Sent Done Done false
137252024-02-12T01:40:25.131Z 12 Acked 1011 Write 3 Sent Done Done false
137262024-02-12T01:40:25.131Z 13 Acked 1012 Read 4 Sent Done Done false
137272024-02-12T01:40:25.131Z 14 Acked 1013 Flush 0 Sent Done Done false
137282024-02-12T01:40:25.131Z 15 Acked 1014 Write 2 Sent Done Done false
137292024-02-12T01:40:25.131Z 16 Acked 1015 Read 6 Sent Done Done false
137302024-02-12T01:40:25.131Z 17 Acked 1016 Read 6 Sent Done Done false
137312024-02-12T01:40:25.131Z 18 Acked 1017 Write 1 Sent Done Done false
137322024-02-12T01:40:25.131Z 19 Acked 1018 Read 5 Sent Done Done false
137332024-02-12T01:40:25.131Z 20 Acked 1019 Read 9 Sent Done Done false
137342024-02-12T01:40:25.131Z 21 Acked 1020 Write 10 Sent Done Done false
137352024-02-12T01:40:25.131Z 22 Acked 1021 Write 5 Sent Done Done false
137362024-02-12T01:40:25.131Z 23 Acked 1022 Write 5 Sent Done Done false
137372024-02-12T01:40:25.131Z 24 Acked 1023 Read 1 Sent Done Done false
137382024-02-12T01:40:25.131Z 25 Acked 1024 Read 5 Sent Done Done false
137392024-02-12T01:40:25.131Z 26 Acked 1025 Read 6 Sent Done Done false
137402024-02-12T01:40:25.131Z 27 Acked 1026 Read 6 Sent Done Done false
137412024-02-12T01:40:25.131Z 28 Acked 1027 Read 6 Sent Done Done false
137422024-02-12T01:40:25.131Z 29 Acked 1028 Write 5 Sent Done Done false
137432024-02-12T01:40:25.132Z 30 Acked 1029 Read 2 Sent Sent Done false
137442024-02-12T01:40:25.132Z STATES DS:0 DS:1 DS:2 TOTAL
137452024-02-12T01:40:25.132Z New 0 0 0 0
137462024-02-12T01:40:25.132Z Sent 26 1 0 27
137472024-02-12T01:40:25.132Z Done 2 27 28 57
137482024-02-12T01:40:25.132Z Skipped 0 0 0 0
137492024-02-12T01:40:25.132Z Error 0 0 0 0
137502024-02-12T01:40:25.132ZLast Flush: 1001 1013 1013
137512024-02-12T01:40:25.132ZDownstairs last five completed: 1001 1000
137522024-02-12T01:40:25.132ZUpstairs last five completed: 30 29 28 27 26
137532024-02-12T01:40:25.132Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:24.009858916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1631}
137542024-02-12T01:40:25.132ZWrote out file "/tmp/test_repair_verify.data"
137552024-02-12T01:40:25.135Z
137562024-02-12T01:40:25.150ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
137572024-02-12T01:40:25.155ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137582024-02-12T01:40:25.155Z 0 000-019 118 118 118 203 203 203 F F F
137592024-02-12T01:40:25.155Z 1 020-039 122 122 122 207 207 207 F F F
137602024-02-12T01:40:25.155Z 2 040-059 123 123 123 208 208 208 F T T <---
137612024-02-12T01:40:25.155Z 3 060-079 124 124 124 209 210 210 T T T <---
137622024-02-12T01:40:25.155Z 4 080-099 123 124 124 208 210 210 T F F <---
137632024-02-12T01:40:25.155Z 5 100-119 123 123 123 208 208 208 F F F
137642024-02-12T01:40:25.155Z 6 120-139 123 123 123 208 208 208 F F F
137652024-02-12T01:40:25.155Z 7 140-159 123 124 124 208 210 210 F F F <---
137662024-02-12T01:40:25.155Z 8 160-179 122 122 122 207 207 207 F F F
137672024-02-12T01:40:25.155Z 9 180-199 120 120 120 205 205 205 F F F
137682024-02-12T01:40:25.155Z 10 200-219 123 123 123 208 208 208 F F F
137692024-02-12T01:40:25.155Z 11 220-239 116 116 116 200 200 200 F T T <---
137702024-02-12T01:40:25.155Z 12 240-259 122 122 122 206 206 206 F T T <---
137712024-02-12T01:40:25.155Z 13 260-279 123 123 123 208 208 208 F T T <---
137722024-02-12T01:40:25.155Z 14 280-299 123 123 123 208 208 208 F F F
137732024-02-12T01:40:25.155Z 15 300-319 120 120 120 205 205 205 F F F
137742024-02-12T01:40:25.155Z 16 320-339 120 120 120 205 205 205 F F F
137752024-02-12T01:40:25.155Z 17 340-359 120 120 120 205 205 205 F F F
137762024-02-12T01:40:25.155Z 18 360-379 113 113 113 197 197 197 F F F
137772024-02-12T01:40:25.155Z 19 380-399 122 122 122 207 207 207 F F F
137782024-02-12T01:40:25.155Z 20 400-419 123 123 123 208 208 208 F F F
137792024-02-12T01:40:25.156Z 21 420-439 123 123 123 208 208 208 F F F
137802024-02-12T01:40:25.156Z 22 440-459 118 118 118 202 202 202 F T T <---
137812024-02-12T01:40:25.156Z 23 460-479 118 124 124 203 210 210 T F F <---
137822024-02-12T01:40:25.156Z 24 480-499 120 124 124 205 210 210 T F F <---
137832024-02-12T01:40:25.156Z 25 500-519 123 123 123 208 208 208 F F F
137842024-02-12T01:40:25.156Z 26 520-539 122 124 124 207 211 211 F F F <---
137852024-02-12T01:40:25.156Z 27 540-559 120 120 120 205 205 205 F F F
137862024-02-12T01:40:25.156Z 28 560-579 120 124 124 205 211 211 F F F <---
137872024-02-12T01:40:25.156Z 29 580-599 109 109 109 193 193 193 F F F
137882024-02-12T01:40:25.156ZMax gen: 124, Max flush: 211
137892024-02-12T01:40:25.159ZError: Difference in extent metadata found!
137902024-02-12T01:40:25.159ZA Difference in extent metadata is expected here
137912024-02-12T01:40:25.159ZOn loop 62
137922024-02-12T01:40:25.159Z
137932024-02-12T01:40:25.165ZVerifying data now
137942024-02-12T01:40:30.762ZNo change after verify
137952024-02-12T01:40:30.762ZLoop: 62 Downstairs dump after verify (and repair):
137962024-02-12T01:40:30.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137972024-02-12T01:40:30.777Z 0 000-019 118 118 118 203 203 203 F F F
137982024-02-12T01:40:30.777Z 1 020-039 122 122 122 207 207 207 F F F
137992024-02-12T01:40:30.777Z 2 040-059 125 125 125 212 212 212 F F F
138002024-02-12T01:40:30.777Z 3 060-079 125 125 125 212 212 212 F F F
138012024-02-12T01:40:30.778Z 4 080-099 124 124 124 210 210 210 F F F
138022024-02-12T01:40:30.778Z 5 100-119 123 123 123 208 208 208 F F F
138032024-02-12T01:40:30.778Z 6 120-139 123 123 123 208 208 208 F F F
138042024-02-12T01:40:30.778Z 7 140-159 124 124 124 210 210 210 F F F
138052024-02-12T01:40:30.778Z 8 160-179 122 122 122 207 207 207 F F F
138062024-02-12T01:40:30.778Z 9 180-199 120 120 120 205 205 205 F F F
138072024-02-12T01:40:30.781Z 10 200-219 123 123 123 208 208 208 F F F
138082024-02-12T01:40:30.781Z 11 220-239 125 125 125 212 212 212 F F F
138092024-02-12T01:40:30.781Z 12 240-259 125 125 125 212 212 212 F F F
138102024-02-12T01:40:30.781Z 13 260-279 125 125 125 212 212 212 F F F
138112024-02-12T01:40:30.781Z 14 280-299 123 123 123 208 208 208 F F F
138122024-02-12T01:40:30.781Z 15 300-319 120 120 120 205 205 205 F F F
138132024-02-12T01:40:30.781Z 16 320-339 120 120 120 205 205 205 F F F
138142024-02-12T01:40:30.781Z 17 340-359 120 120 120 205 205 205 F F F
138152024-02-12T01:40:30.781Z 18 360-379 113 113 113 197 197 197 F F F
138162024-02-12T01:40:30.781Z 19 380-399 122 122 122 207 207 207 F F F
138172024-02-12T01:40:30.781Z 20 400-419 123 123 123 208 208 208 F F F
138182024-02-12T01:40:30.782Z 21 420-439 123 123 123 208 208 208 F F F
138192024-02-12T01:40:30.782Z 22 440-459 125 125 125 212 212 212 F F F
138202024-02-12T01:40:30.782Z 23 460-479 124 124 124 210 210 210 F F F
138212024-02-12T01:40:30.782Z 24 480-499 124 124 124 210 210 210 F F F
138222024-02-12T01:40:30.782Z 25 500-519 123 123 123 208 208 208 F F F
138232024-02-12T01:40:30.782Z 26 520-539 124 124 124 211 211 211 F F F
138242024-02-12T01:40:30.782Z 27 540-559 120 120 120 205 205 205 F F F
138252024-02-12T01:40:30.782Z 28 560-579 124 124 124 211 211 211 F F F
138262024-02-12T01:40:30.782Z 29 580-599 109 109 109 193 193 193 F F F
138272024-02-12T01:40:30.782ZMax gen: 125, Max flush: 212
138282024-02-12T01:40:30.782Z
138292024-02-12T01:40:30.782ZBegin loop 63
138302024-02-12T01:40:30.782ZDownstairs to restart: 1
138312024-02-12T01:40:30.799Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680151612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
138322024-02-12T01:40:30.799Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680436358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
138332024-02-12T01:40:30.799Z{"msg":"Crucible runtime is spawned
138342024-02-12T01:40:30.804ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680481866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
138352024-02-12T01:40:30.804Z{"msg":"Crucible 9b0ea12a-d3d3-47f0-80e9-36d70de8b89f has session id: 1738a428-6722-43f3-a1d9-adcc68839b9f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680533804Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138362024-02-12T01:40:30.804Z{"msg":"Upstairs opts: Upstairs UUID: 9b0ea12a-d3d3-47f0-80e9-36d70de8b89f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680586711Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138372024-02-12T01:40:30.804Z{"msg":"Crucible stats registered with UUID: 9b0ea12a-d3d3-47f0-80e9-36d70de8b89f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680628929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138382024-02-12T01:40:30.804Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680648838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
138392024-02-12T01:40:30.804Z{"msg":"9b0ea12a-d3d3-47f0-80e9-36d70de8b89f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680673387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138402024-02-12T01:40:30.804Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680693996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138412024-02-12T01:40:30.804Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680734994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138422024-02-12T01:40:30.804Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680770332Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138432024-02-12T01:40:30.804Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680797021Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138442024-02-12T01:40:30.804Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68082174Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138452024-02-12T01:40:30.804Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680897926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138462024-02-12T01:40:30.804Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.680936744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138472024-02-12T01:40:30.804Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681066718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138482024-02-12T01:40:30.804Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681109576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138492024-02-12T01:40:30.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681134285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138502024-02-12T01:40:30.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681168144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138512024-02-12T01:40:30.805Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681192512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138522024-02-12T01:40:30.805Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681715357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138532024-02-12T01:40:30.805Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681767945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138542024-02-12T01:40:30.805Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681802473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138552024-02-12T01:40:30.805Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.681837101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138562024-02-12T01:40:30.805Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.682418314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138572024-02-12T01:40:30.805Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.682458992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138582024-02-12T01:40:30.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68291387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138592024-02-12T01:40:30.805Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.682969458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138602024-02-12T01:40:30.805Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.682994846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138612024-02-12T01:40:30.805Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683013955Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138622024-02-12T01:40:30.805Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683038964Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138632024-02-12T01:40:30.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683082012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138642024-02-12T01:40:30.805Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683106361Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138652024-02-12T01:40:30.805Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68312085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138662024-02-12T01:40:30.805Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68355294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138672024-02-12T01:40:30.810Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683955031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138682024-02-12T01:40:30.810Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.683988049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138692024-02-12T01:40:30.810Z{"msg":"[0]R flush_numbers[0..12]: [203, 207, 212, 212, 210, 208, 208, 210, 207, 205, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684007168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138702024-02-12T01:40:30.810Z{"msg":"[0]R generation[0..12]: [118, 122, 125, 125, 124, 123, 123, 124, 122, 120, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684022727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138712024-02-12T01:40:30.810Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684044506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138722024-02-12T01:40:30.810Z{"msg":"[1]R flush_numbers[0..12]: [203, 207, 212, 212, 210, 208, 208, 210, 207, 205, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684058855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138732024-02-12T01:40:30.810Z{"msg":"[1]R generation[0..12]: [118, 122, 125, 125, 124, 123, 123, 124, 122, 120, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684079114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138742024-02-12T01:40:30.810Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684093484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138752024-02-12T01:40:30.810Z{"msg":"[2]R flush_numbers[0..12]: [203, 207, 212, 212, 210, 208, 208, 210, 207, 205, 208, 212]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684113993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138762024-02-12T01:40:30.810Z{"msg":"[2]R generation[0..12]: [118, 122, 125, 125, 124, 123, 123, 124, 122, 120, 123, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684128072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138772024-02-12T01:40:30.810Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684154051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138782024-02-12T01:40:30.810Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68416766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138792024-02-12T01:40:30.810ZWait for a query_work_queue command to finish before sending IO
138802024-02-12T01:40:30.810Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684213518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138812024-02-12T01:40:30.810Z{"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684227247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138822024-02-12T01:40:30.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684257676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138832024-02-12T01:40:30.810Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684277055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138842024-02-12T01:40:30.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684289864Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138852024-02-12T01:40:30.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684312143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138862024-02-12T01:40:30.810Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684336632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138872024-02-12T01:40:30.811Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68437236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138882024-02-12T01:40:30.811Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.68439571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138892024-02-12T01:40:30.811Z{"msg":"9b0ea12a-d3d3-47f0-80e9-36d70de8b89f is now active with session: 1738a428-6722-43f3-a1d9-adcc68839b9f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684431648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138902024-02-12T01:40:30.811Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684461016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639,"session_id":"1738a428-6722-43f3-a1d9-adcc68839b9f"}
138912024-02-12T01:40:30.811Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:29.684475596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
138922024-02-12T01:40:30.811ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138932024-02-12T01:40:30.811ZLoading write count information from file "/tmp/test_repair_verify.data"
138942024-02-12T01:40:30.811ZRun Repair workload
138952024-02-12T01:40:30.811Z01/30 Write block 336 len 4096 data: 5
138962024-02-12T01:40:30.811Z02/30 Read block 372 len 20480
138972024-02-12T01:40:30.811Z03/30 Read block 333 len 20480
138982024-02-12T01:40:30.811Z04/30 Write block 20 len 40960 data: 5 5 6 6 6 6 7 8 10 13
138992024-02-12T01:40:30.811Z05/30 Read block 7 len 16384
139002024-02-12T01:40:30.814Z06/30 Read block 400 len 32768
139012024-02-12T01:40:30.814Z07/30 Write block 375 len 24576 data: 16 13 12 11 12 10
139022024-02-12T01:40:30.815Z08/30 Flush
139032024-02-12T01:40:30.815Z09/30 Flush
139042024-02-12T01:40:30.815Z10/30 Write block 200 len 20480 data: 8 8 11 12 11
139052024-02-12T01:40:30.818Z11/30 Read block 488 len 20480
139062024-02-12T01:40:30.818Z12/30 Read block 241 len 8192
139072024-02-12T01:40:30.818Z13/30 Read block 191 len 36864
139082024-02-12T01:40:30.821Z14/30 Write block 149 len 8192 data: 14 15
139092024-02-12T01:40:30.821Z15/30 Read block 230 len 4096
139102024-02-12T01:40:30.821Z16/30 Flush
139112024-02-12T01:40:30.824Z17/30 Flush
139122024-02-12T01:40:30.824Z18/30 Flush
139132024-02-12T01:40:30.824Z19/30 Write block 497 len 20480 data: 10 11 11 9 10
139142024-02-12T01:40:30.824Z20/30 Write block 324 len 12288 data: 9 8 10
139152024-02-12T01:40:30.824Z21/30 Flush
139162024-02-12T01:40:30.827Z22/30 Flush
139172024-02-12T01:40:30.827Z23/30 Flush
139182024-02-12T01:40:30.829Z24/30 Write block 426 len 32768 data: 7 5 4 5 6 4 4 4
139192024-02-12T01:40:30.829Z25/30 Read block 181 len 20480
139202024-02-12T01:40:30.832Z26/30 Write block 134 len 28672 data: 10 9 12 11 13 14 14
139212024-02-12T01:40:30.832Z27/30 Write block 61 len 20480 data: 7 10 9 9 10
139222024-02-12T01:40:30.833Z28/30 Write block 223 len 36864 data: 11 9 8 6 7 8 7 5 6
139232024-02-12T01:40:30.833Z29/30 Write block 314 len 20480 data: 9 10 8 9 10
139242024-02-12T01:40:30.833Z30/30 Read block 417 len 20480
139252024-02-12T01:40:30.836Z----------------------------------------------------------------
139262024-02-12T01:40:30.836Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
139272024-02-12T01:40:30.836ZStates: Active Active Active
139282024-02-12T01:40:30.836ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
139292024-02-12T01:40:30.836Z 1 Acked 1000 Write 1 Done Done Done false
139302024-02-12T01:40:30.836Z 2 Acked 1001 Read 5 Done Done Done false
139312024-02-12T01:40:30.836Z 3 Acked 1002 Read 5 Done Done Done false
139322024-02-12T01:40:30.836Z 4 Acked 1003 Write 10 Done Done Done false
139332024-02-12T01:40:30.836Z 5 Acked 1004 Read 4 Done Done Done false
139342024-02-12T01:40:30.837Z 6 Acked 1005 Read 8 Done Sent Done false
139352024-02-12T01:40:30.837Z 7 Acked 1006 Write 6 Done Sent Done false
139362024-02-12T01:40:30.837Z 8 Acked 1007 Flush 0 Done Sent Done false
139372024-02-12T01:40:30.837Z 9 Acked 1008 Flush 0 Done Sent Done false
139382024-02-12T01:40:30.837Z 10 Acked 1009 Write 5 Done Sent Done false
139392024-02-12T01:40:30.837Z 11 Acked 1010 Read 5 Done Sent Done false
139402024-02-12T01:40:30.837Z 12 Acked 1011 Read 2 Done Sent Done false
139412024-02-12T01:40:30.837Z 13 Acked 1012 Read 9 Done Sent Done false
139422024-02-12T01:40:30.837Z 14 Acked 1013 Write 2 Done Sent Done false
139432024-02-12T01:40:30.837Z 15 Acked 1014 Read 1 Done Sent Done false
139442024-02-12T01:40:30.837Z 16 Acked 1015 Flush 0 Done Sent Done false
139452024-02-12T01:40:30.837Z 17 Acked 1016 Flush 0 Done Sent Done false
139462024-02-12T01:40:30.837Z 18 Acked 1017 Flush 0 Done Sent Done false
139472024-02-12T01:40:30.837Z 19 Acked 1018 Write 5 Done Sent Done false
139482024-02-12T01:40:30.837Z 20 Acked 1019 Write 3 Done Sent Done false
139492024-02-12T01:40:30.837Z 21 Acked 1020 Flush 0 Done Sent Done false
139502024-02-12T01:40:30.837Z 22 Acked 1021 Flush 0 Done Sent Done false
139512024-02-12T01:40:30.837Z 23 Acked 1022 Flush 0 Done Sent Done false
139522024-02-12T01:40:30.837Z 24 Acked 1023 Write 8 Done Sent Done false
139532024-02-12T01:40:30.837Z 25 Acked 1024 Read 5 Done Sent Done false
139542024-02-12T01:40:30.837Z 26 Acked 1025 Write 7 Done Sent Done false
139552024-02-12T01:40:30.837Z 27 Acked 1026 Write 5 Done Sent Done false
139562024-02-12T01:40:30.837Z 28 Acked 1027 Write 9 Done Sent Done false
139572024-02-12T01:40:30.837Z 29 Acked 1028 Write 5 Done Sent Done false
139582024-02-12T01:40:30.838Z 30 Acked 1029 Read 5 Done Sent Done false
139592024-02-12T01:40:30.838Z STATES DS:0 DS:1 DS:2 TOTAL
139602024-02-12T01:40:30.838Z New 0 0 0 0
139612024-02-12T01:40:30.838Z Sent 0 25 0 25
139622024-02-12T01:40:30.838Z Done 30 5 30 65
139632024-02-12T01:40:30.838Z Skipped 0 0 0 0
139642024-02-12T01:40:30.838Z Error 0 0 0 0
139652024-02-12T01:40:30.838ZLast Flush: 1022 0 1022
139662024-02-12T01:40:30.838ZDownstairs last five completed:
139672024-02-12T01:40:30.838ZUpstairs last five completed: 30 29 28 27 26
139682024-02-12T01:40:30.838Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:29.715329524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1639}
139692024-02-12T01:40:30.838ZWrote out file "/tmp/test_repair_verify.data"
139702024-02-12T01:40:30.841Z
139712024-02-12T01:40:30.844ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
139722024-02-12T01:40:30.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139732024-02-12T01:40:30.860Z 0 000-019 118 118 118 203 203 203 F F F
139742024-02-12T01:40:30.861Z 1 020-039 126 122 126 213 207 213 F T F <---
139752024-02-12T01:40:30.861Z 2 040-059 125 125 125 212 212 212 F F F
139762024-02-12T01:40:30.861Z 3 060-079 125 125 125 212 212 212 T F T <---
139772024-02-12T01:40:30.864Z 4 080-099 124 124 124 210 210 210 F F F
139782024-02-12T01:40:30.864Z 5 100-119 123 123 123 208 208 208 F F F
139792024-02-12T01:40:30.864Z 6 120-139 123 123 123 208 208 208 T F T <---
139802024-02-12T01:40:30.864Z 7 140-159 126 124 126 215 210 215 T F T <---
139812024-02-12T01:40:30.864Z 8 160-179 122 122 122 207 207 207 F F F
139822024-02-12T01:40:30.864Z 9 180-199 120 120 120 205 205 205 F F F
139832024-02-12T01:40:30.864Z 10 200-219 126 123 126 215 208 215 F F F <---
139842024-02-12T01:40:30.864Z 11 220-239 125 125 125 212 212 212 T F T <---
139852024-02-12T01:40:30.864Z 12 240-259 125 125 125 212 212 212 F F F
139862024-02-12T01:40:30.864Z 13 260-279 125 125 125 212 212 212 F F F
139872024-02-12T01:40:30.864Z 14 280-299 123 123 123 208 208 208 F F F
139882024-02-12T01:40:30.864Z 15 300-319 120 120 120 205 205 205 T F T <---
139892024-02-12T01:40:30.864Z 16 320-339 126 120 126 218 205 218 F T F <---
139902024-02-12T01:40:30.864Z 17 340-359 120 120 120 205 205 205 F F F
139912024-02-12T01:40:30.864Z 18 360-379 126 113 126 213 197 213 F F F <---
139922024-02-12T01:40:30.864Z 19 380-399 126 122 126 213 207 213 F F F <---
139932024-02-12T01:40:30.864Z 20 400-419 123 123 123 208 208 208 F F F
139942024-02-12T01:40:30.865Z 21 420-439 123 123 123 208 208 208 T F T <---
139952024-02-12T01:40:30.865Z 22 440-459 125 125 125 212 212 212 F F F
139962024-02-12T01:40:30.865Z 23 460-479 124 124 124 210 210 210 F F F
139972024-02-12T01:40:30.865Z 24 480-499 126 124 126 218 210 218 F F F <---
139982024-02-12T01:40:30.865Z 25 500-519 126 123 126 218 208 218 F F F <---
139992024-02-12T01:40:30.865Z 26 520-539 124 124 124 211 211 211 F F F
140002024-02-12T01:40:30.865Z 27 540-559 120 120 120 205 205 205 F F F
140012024-02-12T01:40:30.865Z 28 560-579 124 124 124 211 211 211 F F F
140022024-02-12T01:40:30.865Z 29 580-599 109 109 109 193 193 193 F F F
140032024-02-12T01:40:30.865ZMax gen: 126, Max flush: 218
140042024-02-12T01:40:30.865ZError: Difference in extent metadata found!
140052024-02-12T01:40:30.868ZA Difference in extent metadata is expected here
140062024-02-12T01:40:30.868ZOn loop 63
140072024-02-12T01:40:30.868Z
140082024-02-12T01:40:30.868ZVerifying data now
140092024-02-12T01:40:36.513ZNo change after verify
140102024-02-12T01:40:36.513ZLoop: 63 Downstairs dump after verify (and repair):
140112024-02-12T01:40:36.529ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140122024-02-12T01:40:36.529Z 0 000-019 118 118 118 203 203 203 F F F
140132024-02-12T01:40:36.529Z 1 020-039 126 126 126 213 213 213 F F F
140142024-02-12T01:40:36.529Z 2 040-059 125 125 125 212 212 212 F F F
140152024-02-12T01:40:36.529Z 3 060-079 127 127 127 219 219 219 F F F
140162024-02-12T01:40:36.529Z 4 080-099 124 124 124 210 210 210 F F F
140172024-02-12T01:40:36.529Z 5 100-119 123 123 123 208 208 208 F F F
140182024-02-12T01:40:36.529Z 6 120-139 127 127 127 219 219 219 F F F
140192024-02-12T01:40:36.533Z 7 140-159 127 127 127 219 219 219 F F F
140202024-02-12T01:40:36.533Z 8 160-179 122 122 122 207 207 207 F F F
140212024-02-12T01:40:36.533Z 9 180-199 120 120 120 205 205 205 F F F
140222024-02-12T01:40:36.533Z 10 200-219 126 126 126 215 215 215 F F F
140232024-02-12T01:40:36.533Z 11 220-239 127 127 127 219 219 219 F F F
140242024-02-12T01:40:36.533Z 12 240-259 125 125 125 212 212 212 F F F
140252024-02-12T01:40:36.533Z 13 260-279 125 125 125 212 212 212 F F F
140262024-02-12T01:40:36.533Z 14 280-299 123 123 123 208 208 208 F F F
140272024-02-12T01:40:36.533Z 15 300-319 127 127 127 219 219 219 F F F
140282024-02-12T01:40:36.533Z 16 320-339 126 126 126 218 218 218 F F F
140292024-02-12T01:40:36.533Z 17 340-359 120 120 120 205 205 205 F F F
140302024-02-12T01:40:36.533Z 18 360-379 126 126 126 213 213 213 F F F
140312024-02-12T01:40:36.533Z 19 380-399 126 126 126 213 213 213 F F F
140322024-02-12T01:40:36.533Z 20 400-419 123 123 123 208 208 208 F F F
140332024-02-12T01:40:36.533Z 21 420-439 127 127 127 219 219 219 F F F
140342024-02-12T01:40:36.534Z 22 440-459 125 125 125 212 212 212 F F F
140352024-02-12T01:40:36.534Z 23 460-479 124 124 124 210 210 210 F F F
140362024-02-12T01:40:36.534Z 24 480-499 126 126 126 218 218 218 F F F
140372024-02-12T01:40:36.534Z 25 500-519 126 126 126 218 218 218 F F F
140382024-02-12T01:40:36.534Z 26 520-539 124 124 124 211 211 211 F F F
140392024-02-12T01:40:36.534Z 27 540-559 120 120 120 205 205 205 F F F
140402024-02-12T01:40:36.534Z 28 560-579 124 124 124 211 211 211 F F F
140412024-02-12T01:40:36.534Z 29 580-599 109 109 109 193 193 193 F F F
140422024-02-12T01:40:36.534ZMax gen: 127, Max flush: 219
140432024-02-12T01:40:36.534Z
140442024-02-12T01:40:36.534ZBegin loop 64
140452024-02-12T01:40:36.534ZDownstairs to restart: 2
140462024-02-12T01:40:36.553Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432453675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
140472024-02-12T01:40:36.553Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432713892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
140482024-02-12T01:40:36.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432744531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
140492024-02-12T01:40:36.558ZCrucible runtime is spawned
140502024-02-12T01:40:36.558Z{"msg":"Crucible 3310c566-8e83-4333-8f1b-b415c3598149 has session id: 70d30a54-13ab-4311-9309-6a8db027c8c6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432793338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140512024-02-12T01:40:36.558Z{"msg":"Upstairs opts: Upstairs UUID: 3310c566-8e83-4333-8f1b-b415c3598149, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432841376Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140522024-02-12T01:40:36.558Z{"msg":"Crucible stats registered with UUID: 3310c566-8e83-4333-8f1b-b415c3598149","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432867045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140532024-02-12T01:40:36.558Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432894714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
140542024-02-12T01:40:36.558Z{"msg":"3310c566-8e83-4333-8f1b-b415c3598149 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432911143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140552024-02-12T01:40:36.558Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432936121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140562024-02-12T01:40:36.558Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432955971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140572024-02-12T01:40:36.558Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.432993989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140582024-02-12T01:40:36.558Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433025197Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140592024-02-12T01:40:36.558Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433070425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140602024-02-12T01:40:36.558Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433105693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140612024-02-12T01:40:36.558Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433139262Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140622024-02-12T01:40:36.558Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433206119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140632024-02-12T01:40:36.558Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433246737Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140642024-02-12T01:40:36.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433278055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140652024-02-12T01:40:36.558Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433320393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140662024-02-12T01:40:36.559Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:35.433357452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140672024-02-12T01:40:36.559Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:35.43339981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140682024-02-12T01:40:36.559Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:35.433496505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140692024-02-12T01:40:36.559Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433531233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140702024-02-12T01:40:36.559Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433545832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140712024-02-12T01:40:36.559Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.433613579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140722024-02-12T01:40:36.559Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434048939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140732024-02-12T01:40:36.559Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434084087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140742024-02-12T01:40:36.559Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434114135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140752024-02-12T01:40:36.559Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434148294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140762024-02-12T01:40:36.559Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434692338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140772024-02-12T01:40:36.559Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.434762145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140782024-02-12T01:40:36.559Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435245491Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140792024-02-12T01:40:36.559Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435292099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140802024-02-12T01:40:36.559Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435331607Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140812024-02-12T01:40:36.559Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435355886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140822024-02-12T01:40:36.559Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435379485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140832024-02-12T01:40:36.559Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:35.435393854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140842024-02-12T01:40:37.556Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.434917578Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140852024-02-12T01:40:37.556Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.434992855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140862024-02-12T01:40:37.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.435880752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140872024-02-12T01:40:37.561Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.43592987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140882024-02-12T01:40:37.561Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.436416037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140892024-02-12T01:40:37.561Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437050196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140902024-02-12T01:40:37.561Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437098624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140912024-02-12T01:40:37.561Z{"msg":"[0]R flush_numbers[0..12]: [203, 213, 212, 219, 210, 208, 219, 219, 207, 205, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437127813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140922024-02-12T01:40:37.561Z{"msg":"[0]R generation[0..12]: [118, 126, 125, 127, 124, 123, 127, 127, 122, 120, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437146922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140932024-02-12T01:40:37.561Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.43717349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140942024-02-12T01:40:37.561Z{"msg":"[1]R flush_numbers[0..12]: [203, 213, 212, 219, 210, 208, 219, 219, 207, 205, 215, 219]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.4371929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140952024-02-12T01:40:37.561Z{"msg":"[1]R generation[0..12]: [118, 126, 125, 127, 124, 123, 127, 127, 122, 120, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437218748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140962024-02-12T01:40:37.561Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437236987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140972024-02-12T01:40:37.561Z{"msg":"[2]R flush_numbers[0..12]: [203, 213, 212, 219, 210, 208, 219, 219, 207, 205, 215, 219]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
140982024-02-12T01:40:37.561Z,"time":"2024-02-12T01:40:36.437263356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
140992024-02-12T01:40:37.561Z{"msg":"[2]R generation[0..12]: [118, 126, 125, 127, 124, 123, 127, 127, 122, 120, 126, 127]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437284945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141002024-02-12T01:40:37.561Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437310834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141012024-02-12T01:40:37.561Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437327563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141022024-02-12T01:40:37.561Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437359252Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141032024-02-12T01:40:37.561Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437376451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141042024-02-12T01:40:37.561Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.43740169Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141052024-02-12T01:40:37.561Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437419039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141062024-02-12T01:40:37.561Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437435148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141072024-02-12T01:40:37.561Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437473256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141082024-02-12T01:40:37.561Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437508385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141092024-02-12T01:40:37.562Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437547863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141102024-02-12T01:40:37.562Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437574802Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141112024-02-12T01:40:37.562Z{"msg":"3310c566-8e83-4333-8f1b-b415c3598149 is now active with session: 70d30a54-13ab-4311-9309-6a8db027c8c6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437588341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141122024-02-12T01:40:37.562Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437618979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647,"session_id":"70d30a54-13ab-4311-9309-6a8db027c8c6"}
141132024-02-12T01:40:37.562Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:36.437654368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
141142024-02-12T01:40:37.562ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
141152024-02-12T01:40:37.562ZLoading write count information from file "/tmp/test_repair_verify.data"
141162024-02-12T01:40:37.562ZRun Repair workload
141172024-02-12T01:40:37.562Z01/30 Write block 35 len 4096 data: 13
141182024-02-12T01:40:37.565Z02/30 Read block 511 len 4096
141192024-02-12T01:40:37.565Z03/30 Read block 110 len 40960
141202024-02-12T01:40:37.565Z04/30 Read block 133 len 12288
141212024-02-12T01:40:37.565Z05/30 Flush
141222024-02-12T01:40:37.568Z06/30 Write block 65 len 24576 data: 11 10 12 11 13 12
141232024-02-12T01:40:37.568Z07/30 Read block 26 len 28672
141242024-02-12T01:40:37.569Z08/30 Read block 493 len 24576
141252024-02-12T01:40:37.569Z09/30 Read block 109 len 40960
141262024-02-12T01:40:37.572Z10/30 Flush
141272024-02-12T01:40:37.574Z11/30 Flush
141282024-02-12T01:40:37.574Z12/30 Write block 205 len 4096 data: 10
141292024-02-12T01:40:37.574Z13/30 Read block 446 len 16384
141302024-02-12T01:40:37.574Z14/30 Flush
141312024-02-12T01:40:37.577Z15/30 Write block 579 len 40960 data: 9 8 9 9 8 9 9 9 6 5
141322024-02-12T01:40:37.577Z16/30 Read block 593 len 16384
141332024-02-12T01:40:37.580Z17/30 Read block 450 len 36864
141342024-02-12T01:40:37.580Z18/30 Read block 538 len 12288
141352024-02-12T01:40:37.581Z19/30 Write block 224 len 8192 data: 10 9
141362024-02-12T01:40:37.581Z20/30 Flush
141372024-02-12T01:40:37.586Z21/30 Write block 97 len 24576 data: 7 8 6 5 7 6
141382024-02-12T01:40:37.586Z22/30 Write block 554 len 28672 data: 10 8 7 6 6 7 6
141392024-02-12T01:40:37.586Z23/30 Write block 457 len 24576 data: 8 10 7 6 6 7
141402024-02-12T01:40:37.589Z24/30 Read block 99 len 24576
141412024-02-12T01:40:37.590Z25/30 Write block 21 len 28672 data: 6 7 7 7 7 8 9
141422024-02-12T01:40:37.590Z26/30 Read block 74 len 24576
141432024-02-12T01:40:37.590Z27/30 Read block 77 len 28672
141442024-02-12T01:40:37.590Z28/30 Read block 252 len 20480
141452024-02-12T01:40:37.590Z29/30 Write block 20 len 24576 data: 6 7 8 8 8 8
141462024-02-12T01:40:37.594Z30/30 Read block 308 len 40960
141472024-02-12T01:40:37.594Z----------------------------------------------------------------
141482024-02-12T01:40:37.594Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
141492024-02-12T01:40:37.594ZStates: Active Active Active
141502024-02-12T01:40:37.594ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
141512024-02-12T01:40:37.594Z 1 Acked 1000 Write 1 Done Done Done false
141522024-02-12T01:40:37.594Z 2 Acked 1001 Read 1 Done Done Done false
141532024-02-12T01:40:37.594Z 3 Acked 1002 Read 10 Done Done Sent false
141542024-02-12T01:40:37.594Z 4 Acked 1003 Read 3 Done Done Sent false
141552024-02-12T01:40:37.594Z 5 Acked 1004 Flush 0 Done Done Sent false
141562024-02-12T01:40:37.594Z 6 Acked 1005 Write 6 Done Done Sent false
141572024-02-12T01:40:37.594Z 7 Acked 1006 Read 7 Done Done Sent false
141582024-02-12T01:40:37.594Z 8 Acked 1007 Read 6 Done Done Sent false
141592024-02-12T01:40:37.594Z 9 Acked 1008 Read 10 Done Done Sent false
141602024-02-12T01:40:37.594Z 10 Acked 1009 Flush 0 Done Done Sent false
141612024-02-12T01:40:37.594Z 11 Acked 1010 Flush 0 Done Done Sent false
141622024-02-12T01:40:37.594Z 12 Acked 1011 Write 1 Done Done Sent false
141632024-02-12T01:40:37.594Z 13 Acked 1012 Read 4 Done Done Sent false
141642024-02-12T01:40:37.594Z 14 Acked 1013 Flush 0 Done Done Sent false
141652024-02-12T01:40:37.594Z 15 Acked 1014 Write 10 Done Done Sent false
141662024-02-12T01:40:37.594Z 16 Acked 1015 Read 4 Done Done Sent false
141672024-02-12T01:40:37.594Z 17 Acked 1016 Read 9 Done Done Sent false
141682024-02-12T01:40:37.594Z 18 Acked 1017 Read 3 Done Done Sent false
141692024-02-12T01:40:37.595Z 19 Acked 1018 Write 2 Done Done Sent false
141702024-02-12T01:40:37.595Z 20 Acked 1019 Flush 0 Done Done Sent false
141712024-02-12T01:40:37.595Z 21 Acked 1020 Write 6 Done Done Sent false
141722024-02-12T01:40:37.595Z 22 Acked 1021 Write 7 Done Done Sent false
141732024-02-12T01:40:37.595Z 23 Acked 1022 Write 6 Done Done Sent false
141742024-02-12T01:40:37.595Z 24 Acked 1023 Read 6 Done Done Sent false
141752024-02-12T01:40:37.595Z 25 Acked 1024 Write 7 Done Done Sent false
141762024-02-12T01:40:37.595Z 26 Acked 1025 Read 6 Done Done Sent false
141772024-02-12T01:40:37.595Z 27 Acked 1026 Read 7 Done Done Sent false
141782024-02-12T01:40:37.595Z 28 Acked 1027 Read 5 Done Done Sent false
141792024-02-12T01:40:37.595Z 29 Acked 1028 Write 6 Done Done Sent false
141802024-02-12T01:40:37.595Z 30 Acked 1029 Read 10 Done Done Sent false
141812024-02-12T01:40:37.595Z STATES DS:0 DS:1 DS:2 TOTAL
141822024-02-12T01:40:37.595Z New 0 0 0 0
141832024-02-12T01:40:37.595Z Sent 0 0 28 28
141842024-02-12T01:40:37.595Z Done 30 30 2 62
141852024-02-12T01:40:37.595Z Skipped 0 0 0 0
141862024-02-12T01:40:37.595Z Error 0 0 0 0
141872024-02-12T01:40:37.595ZLast Flush: 1019 1019 0
141882024-02-12T01:40:37.595ZDownstairs last five completed:
141892024-02-12T01:40:37.595ZUpstairs last five completed: 30 29 28 27 26
141902024-02-12T01:40:37.595Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:36.472105084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1647}
141912024-02-12T01:40:37.595ZWrote out file "/tmp/test_repair_verify.data"
141922024-02-12T01:40:37.599Z
141932024-02-12T01:40:37.602ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141942024-02-12T01:40:37.618ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141952024-02-12T01:40:37.618Z 0 000-019 118 118 118 203 203 203 F F F
141962024-02-12T01:40:37.618Z 1 020-039 128 128 126 220 220 213 T T T <---
141972024-02-12T01:40:37.618Z 2 040-059 125 125 125 212 212 212 F F F
141982024-02-12T01:40:37.618Z 3 060-079 128 128 127 221 221 219 F F F <---
141992024-02-12T01:40:37.618Z 4 080-099 124 124 124 210 210 210 T T F <---
142002024-02-12T01:40:37.618Z 5 100-119 123 123 123 208 208 208 T T F <---
142012024-02-12T01:40:37.618Z 6 120-139 127 127 127 219 219 219 F F F
142022024-02-12T01:40:37.619Z 7 140-159 127 127 127 219 219 219 F F F
142032024-02-12T01:40:37.619Z 8 160-179 122 122 122 207 207 207 F F F
142042024-02-12T01:40:37.619Z 9 180-199 120 120 120 205 205 205 F F F
142052024-02-12T01:40:37.619Z 10 200-219 128 128 126 223 223 215 F F F <---
142062024-02-12T01:40:37.619Z 11 220-239 128 128 127 224 224 219 F F F <---
142072024-02-12T01:40:37.619Z 12 240-259 125 125 125 212 212 212 F F F
142082024-02-12T01:40:37.622Z 13 260-279 125 125 125 212 212 212 F F F
142092024-02-12T01:40:37.622Z 14 280-299 123 123 123 208 208 208 F F F
142102024-02-12T01:40:37.622Z 15 300-319 127 127 127 219 219 219 F F F
142112024-02-12T01:40:37.622Z 16 320-339 126 126 126 218 218 218 F F F
142122024-02-12T01:40:37.622Z 17 340-359 120 120 120 205 205 205 F F F
142132024-02-12T01:40:37.623Z 18 360-379 126 126 126 213 213 213 F F F
142142024-02-12T01:40:37.623Z 19 380-399 126 126 126 213 213 213 F F F
142152024-02-12T01:40:37.623Z 20 400-419 123 123 123 208 208 208 F F F
142162024-02-12T01:40:37.623Z 21 420-439 127 127 127 219 219 219 F F F
142172024-02-12T01:40:37.623Z 22 440-459 125 125 125 212 212 212 T T F <---
142182024-02-12T01:40:37.623Z 23 460-479 124 124 124 210 210 210 T T F <---
142192024-02-12T01:40:37.623Z 24 480-499 126 126 126 218 218 218 F F F
142202024-02-12T01:40:37.623Z 25 500-519 126 126 126 218 218 218 F F F
142212024-02-12T01:40:37.623Z 26 520-539 124 124 124 211 211 211 F F F
142222024-02-12T01:40:37.623Z 27 540-559 120 120 120 205 205 205 T T F <---
142232024-02-12T01:40:37.623Z 28 560-579 128 128 124 224 224 211 T T F <---
142242024-02-12T01:40:37.623Z 29 580-599 128 128 109 224 224 193 F F F <---
142252024-02-12T01:40:37.623ZMax gen: 128, Max flush: 224
142262024-02-12T01:40:37.623ZError: Difference in extent metadata found!
142272024-02-12T01:40:37.623ZA Difference in extent metadata is expected here
142282024-02-12T01:40:37.623ZOn loop 64
142292024-02-12T01:40:37.623Z
142302024-02-12T01:40:37.626ZVerifying data now
142312024-02-12T01:40:42.201ZNo change after verify
142322024-02-12T01:40:42.201ZLoop: 64 Downstairs dump after verify (and repair):
142332024-02-12T01:40:42.216ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142342024-02-12T01:40:42.216Z 0 000-019 118 118 118 203 203 203 F F F
142352024-02-12T01:40:42.217Z 1 020-039 129 129 129 225 225 225 F F F
142362024-02-12T01:40:42.217Z 2 040-059 125 125 125 212 212 212 F F F
142372024-02-12T01:40:42.217Z 3 060-079 128 128 128 221 221 221 F F F
142382024-02-12T01:40:42.217Z 4 080-099 129 129 129 225 225 225 F F F
142392024-02-12T01:40:42.217Z 5 100-119 129 129 129 225 225 225 F F F
142402024-02-12T01:40:42.217Z 6 120-139 127 127 127 219 219 219 F F F
142412024-02-12T01:40:42.217Z 7 140-159 127 127 127 219 219 219 F F F
142422024-02-12T01:40:42.217Z 8 160-179 122 122 122 207 207 207 F F F
142432024-02-12T01:40:42.217Z 9 180-199 120 120 120 205 205 205 F F F
142442024-02-12T01:40:42.217Z 10 200-219 128 128 128 223 223 223 F F F
142452024-02-12T01:40:42.217Z 11 220-239 128 128 128 224 224 224 F F F
142462024-02-12T01:40:42.217Z 12 240-259 125 125 125 212 212 212 F F F
142472024-02-12T01:40:42.221Z 13 260-279 125 125 125 212 212 212 F F F
142482024-02-12T01:40:42.222Z 14 280-299 123 123 123 208 208 208 F F F
142492024-02-12T01:40:42.222Z 15 300-319 127 127 127 219 219 219 F F F
142502024-02-12T01:40:42.222Z 16 320-339 126 126 126 218 218 218 F F F
142512024-02-12T01:40:42.222Z 17 340-359 120 120 120 205 205 205 F F F
142522024-02-12T01:40:42.222Z 18 360-379 126 126 126 213 213 213 F F F
142532024-02-12T01:40:42.222Z 19 380-399 126 126 126 213 213 213 F F F
142542024-02-12T01:40:42.222Z 20 400-419 123 123 123 208 208 208 F F F
142552024-02-12T01:40:42.222Z 21 420-439 127 127 127 219 219 219 F F F
142562024-02-12T01:40:42.222Z 22 440-459 129 129 129 225 225 225 F F F
142572024-02-12T01:40:42.222Z 23 460-479 129 129 129 225 225 225 F F F
142582024-02-12T01:40:42.222Z 24 480-499 126 126 126 218 218 218 F F F
142592024-02-12T01:40:42.222Z 25 500-519 126 126 126 218 218 218 F F F
142602024-02-12T01:40:42.222Z 26 520-539 124 124 124 211 211 211 F F F
142612024-02-12T01:40:42.223Z 27 540-559 129 129 129 225 225 225 F F F
142622024-02-12T01:40:42.223Z 28 560-579 129 129 129 225 225 225 F F F
142632024-02-12T01:40:42.223Z 29 580-599 128 128 128 224 224 224 F F F
142642024-02-12T01:40:42.223ZMax gen: 129, Max flush: 225
142652024-02-12T01:40:42.223Z
142662024-02-12T01:40:42.223ZBegin loop 65
142672024-02-12T01:40:42.223ZDownstairs to restart: 0
142682024-02-12T01:40:42.238Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.117564933Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
142692024-02-12T01:40:42.238Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.11784117Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
142702024-02-12T01:40:42.243Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.117863739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
142712024-02-12T01:40:42.243Z{"msg":"Crucible caca0d30-fea9-4771-8e30-465e78d30748 has session id: 43e09ed7-5076-441e-b69c-35150a139f69","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.117897688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142722024-02-12T01:40:42.243Z{"msg":"Upstairs opts: Upstairs UUID: caca0d30-fea9-4771-8e30-465e78d30748, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.117932846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142732024-02-12T01:40:42.243Z{"msg":"Crucible stats registered with UUID: caca0d30-fea9-4771-8e30-465e78d30748","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.117975124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142742024-02-12T01:40:42.243ZCrucible runtime is spawned
142752024-02-12T01:40:42.243Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118190524Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
142762024-02-12T01:40:42.243Z{"msg":"caca0d30-fea9-4771-8e30-465e78d30748 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118221112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142772024-02-12T01:40:42.244Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118236201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142782024-02-12T01:40:42.244Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.11826101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142792024-02-12T01:40:42.244Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118285549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142802024-02-12T01:40:42.244Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118312118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142812024-02-12T01:40:42.244Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118335837Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142822024-02-12T01:40:42.244Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118377735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142832024-02-12T01:40:42.244Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118404384Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142842024-02-12T01:40:42.244Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118432462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142852024-02-12T01:40:42.244Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.11846733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142862024-02-12T01:40:42.244Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118580075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142872024-02-12T01:40:42.244Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118624803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142882024-02-12T01:40:42.244Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.118650632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142892024-02-12T01:40:42.244Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.119375207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142902024-02-12T01:40:42.244Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.119411176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142912024-02-12T01:40:42.244Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.119427055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142922024-02-12T01:40:42.244Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.119441144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142932024-02-12T01:40:42.244Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120049455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142942024-02-12T01:40:42.245Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120101192Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142952024-02-12T01:40:42.245Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120544651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142962024-02-12T01:40:42.245Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120586989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142972024-02-12T01:40:42.245Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120630237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142982024-02-12T01:40:42.245Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120665306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
142992024-02-12T01:40:42.245Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120691364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143002024-02-12T01:40:42.245Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120726053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143012024-02-12T01:40:42.245Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120798869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143022024-02-12T01:40:42.245Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.120825958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143032024-02-12T01:40:42.245Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121332344Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143042024-02-12T01:40:42.252Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121794412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143052024-02-12T01:40:42.252Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121852029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143062024-02-12T01:40:42.252Z{"msg":"[0]R flush_numbers[0..12]: [203, 225, 212, 221, 225, 225, 219, 219, 207, 205, 223, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121888667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143072024-02-12T01:40:42.252Z{"msg":"[0]R generation[0..12]: [118, 129, 125, 128, 129, 129, 127, 127, 122, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121905406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143082024-02-12T01:40:42.252Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121927995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143092024-02-12T01:40:42.253Z{"msg":"[1]R flush_numbers[0..12]: [203, 225, 212, 221, 225, 225, 219, 219, 207, 205, 223, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121943185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143102024-02-12T01:40:42.253Z{"msg":"[1]R generation[0..12]: [118, 129, 125, 128, 129, 129, 127, 127, 122, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.121980333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143112024-02-12T01:40:42.253Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122003332Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143122024-02-12T01:40:42.253Z{"msg":"[2]R flush_numbers[0..12]: [203, 225, 212, 221, 225, 225, 219, 219, 207, 205, 223, 224]","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO
143132024-02-12T01:40:42.253Z":30,"time":"2024-02-12T01:40:41.12203225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143142024-02-12T01:40:42.253Z{"msg":"[2]R generation[0..12]: [118, 129, 125, 128, 129, 129, 127, 127, 122, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.12204852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143152024-02-12T01:40:42.253Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122069179Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143162024-02-12T01:40:42.253Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122083008Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143172024-02-12T01:40:42.253Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122103397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143182024-02-12T01:40:42.253Z{"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122116966Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143192024-02-12T01:40:42.253Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122137505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143202024-02-12T01:40:42.253Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122152095Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143212024-02-12T01:40:42.253Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122173654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143222024-02-12T01:40:42.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122209012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143232024-02-12T01:40:42.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.12224203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143242024-02-12T01:40:42.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.12226188Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143252024-02-12T01:40:42.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122344945Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143262024-02-12T01:40:42.254Z{"msg":"caca0d30-fea9-4771-8e30-465e78d30748 is now active with session: 43e09ed7-5076-441e-b69c-35150a139f69","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122378854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143272024-02-12T01:40:42.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122400023Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655,"session_id":"43e09ed7-5076-441e-b69c-35150a139f69"}
143282024-02-12T01:40:42.254Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:41.122434941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
143292024-02-12T01:40:42.254ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
143302024-02-12T01:40:42.254ZLoading write count information from file "/tmp/test_repair_verify.data"
143312024-02-12T01:40:42.254ZRun Repair workload
143322024-02-12T01:40:42.254Z01/30 Write block 590 len 32768 data: 9 7 8 5 5 7 7 6
143332024-02-12T01:40:42.254Z02/30 Write block 23 len 28672 data: 9 9 9 9 10 11 14
143342024-02-12T01:40:42.254Z03/30 Read block 76 len 20480
143352024-02-12T01:40:42.254Z04/30 Read block 7 len 8192
143362024-02-12T01:40:42.254Z05/30 Write block 64 len 40960 data: 10 12 11 13 12 14 13 12 8 7
143372024-02-12T01:40:42.261Z06/30 Read block 352 len 28672
143382024-02-12T01:40:42.261Z07/30 Write block 411 len 40960 data: 11 11 13 12 13 11 11 8 7 8
143392024-02-12T01:40:42.261Z08/30 Read block 442 len 32768
143402024-02-12T01:40:42.261Z09/30 Flush
143412024-02-12T01:40:42.261Z10/30 Write block 258 len 12288 data: 11 10 13
143422024-02-12T01:40:42.261Z11/30 Write block 17 len 24576 data: 7 7 6 7 8 9
143432024-02-12T01:40:42.261Z12/30 Read block 34 len 28672
143442024-02-12T01:40:42.261Z13/30 Read block 237 len 8192
143452024-02-12T01:40:42.262Z14/30 Write block 463 len 4096 data: 7
143462024-02-12T01:40:42.262Z15/30 Write block 154 len 36864 data: 10 6 5 6 7 8 11 10 12
143472024-02-12T01:40:42.262Z16/30 Read block 13 len 20480
143482024-02-12T01:40:42.265Z17/30 Read block 193 len 24576
143492024-02-12T01:40:42.265Z18/30 Flush
143502024-02-12T01:40:42.268Z19/30 Write block 120 len 4096 data: 13
143512024-02-12T01:40:42.269Z20/30 Read block 284 len 4096
143522024-02-12T01:40:42.269Z21/30 Write block 369 len 4096 data: 12
143532024-02-12T01:40:42.269Z22/30 Write block 502 len 36864 data: 10 8 7 8 6 6 6 6 9
143542024-02-12T01:40:42.269Z23/30 Read block 384 len 12288
143552024-02-12T01:40:42.269Z24/30 Flush
143562024-02-12T01:40:42.273Z25/30 Write block 383 len 32768 data: 8 10 12 14 13 13 12 16
143572024-02-12T01:40:42.273Z26/30 Write block 440 len 20480 data: 9 10 9 7 8
143582024-02-12T01:40:42.273Z27/30 Read block 317 len 8192
143592024-02-12T01:40:42.277Z28/30 Write block 243 len 36864 data: 13 10 8 8 9 8 7 6 8
143602024-02-12T01:40:42.277Z29/30 Read block 1 len 20480
143612024-02-12T01:40:42.277Z30/30 Read block 140 len 40960
143622024-02-12T01:40:42.277Z----------------------------------------------------------------
143632024-02-12T01:40:42.277Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
143642024-02-12T01:40:42.277ZStates: Active Active Active
143652024-02-12T01:40:42.277ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143662024-02-12T01:40:42.277Z 1 Acked 1000 Write 8 Sent Done Done false
143672024-02-12T01:40:42.277Z 2 Acked 1001 Write 7 Sent Done Done false
143682024-02-12T01:40:42.277Z 3 Acked 1002 Read 5 Sent Done Done false
143692024-02-12T01:40:42.278Z 4 Acked 1003 Read 2 Sent Done Done false
143702024-02-12T01:40:42.278Z 5 Acked 1004 Write 10 Sent Done Done false
143712024-02-12T01:40:42.278Z 6 Acked 1005 Read 7 Sent Done Done false
143722024-02-12T01:40:42.278Z 7 Acked 1006 Write 10 Sent Done Done false
143732024-02-12T01:40:42.278Z 8 Acked 1007 Read 8 Sent Done Done false
143742024-02-12T01:40:42.278Z 9 Acked 1008 Flush 0 Sent Done Done false
143752024-02-12T01:40:42.278Z 10 Acked 1009 Write 3 Sent Done Done false
143762024-02-12T01:40:42.278Z 11 Acked 1010 Write 6 Sent Done Done false
143772024-02-12T01:40:42.278Z 12 Acked 1011 Read 7 Sent Done Done false
143782024-02-12T01:40:42.278Z 13 Acked 1012 Read 2 Sent Done Done false
143792024-02-12T01:40:42.278Z 14 Acked 1013 Write 1 Sent Done Done false
143802024-02-12T01:40:42.278Z 15 Acked 1014 Write 9 Sent Done Done false
143812024-02-12T01:40:42.278Z 16 Acked 1015 Read 5 Sent Done Done false
143822024-02-12T01:40:42.278Z 17 Acked 1016 Read 6 Sent Done Done false
143832024-02-12T01:40:42.279Z 18 Acked 1017 Flush 0 Sent Done Done false
143842024-02-12T01:40:42.279Z 19 Acked 1018 Write 1 Sent Done Done false
143852024-02-12T01:40:42.279Z 20 Acked 1019 Read 1 Sent Done Done false
143862024-02-12T01:40:42.279Z 21 Acked 1020 Write 1 Sent Done Done false
143872024-02-12T01:40:42.279Z 22 Acked 1021 Write 9 Sent Done Done false
143882024-02-12T01:40:42.279Z 23 Acked 1022 Read 3 Sent Done Done false
143892024-02-12T01:40:42.279Z 24 Acked 1023 Flush 0 Sent Done Done false
143902024-02-12T01:40:42.279Z 25 Acked 1024 Write 8 Sent Done Done false
143912024-02-12T01:40:42.279Z 26 Acked 1025 Write 5 Sent Done Done false
143922024-02-12T01:40:42.279Z 27 Acked 1026 Read 2 Sent Done Done false
143932024-02-12T01:40:42.279Z 28 Acked 1027 Write 9 Sent Done Done false
143942024-02-12T01:40:42.279Z 29 Acked 1028 Read 5 Sent Done Done false
143952024-02-12T01:40:42.280Z 30 Acked 1029 Read 10 Sent Done Sent false
143962024-02-12T01:40:42.280Z STATES DS:0 DS:1 DS:2 TOTAL
143972024-02-12T01:40:42.280Z New 0 0 0 0
143982024-02-12T01:40:42.280Z Sent 30 0 1 31
143992024-02-12T01:40:42.280Z Done 0 30 29 59
144002024-02-12T01:40:42.280Z Skipped 0 0 0 0
144012024-02-12T01:40:42.280Z Error 0 0 0 0
144022024-02-12T01:40:42.280ZLast Flush: 0 1023 1023
144032024-02-12T01:40:42.280ZDownstairs last five completed:
144042024-02-12T01:40:42.280ZUpstairs last five completed: 30 29 28 27 26
144052024-02-12T01:40:42.284Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:41.156426339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1655}
144062024-02-12T01:40:42.284ZWrote out file "/tmp/test_repair_verify.data"
144072024-02-12T01:40:42.284Z
144082024-02-12T01:40:42.288ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
144092024-02-12T01:40:42.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144102024-02-12T01:40:42.302Z 0 000-019 118 130 130 203 227 227 F F F <---
144112024-02-12T01:40:42.302Z 1 020-039 129 130 130 225 227 227 F F F <---
144122024-02-12T01:40:42.302Z 2 040-059 125 125 125 212 212 212 F F F
144132024-02-12T01:40:42.302Z 3 060-079 128 130 130 221 226 226 F F F <---
144142024-02-12T01:40:42.302Z 4 080-099 129 129 129 225 225 225 F F F
144152024-02-12T01:40:42.306Z 5 100-119 129 129 129 225 225 225 F F F
144162024-02-12T01:40:42.306Z 6 120-139 127 130 130 219 228 228 F F F <---
144172024-02-12T01:40:42.306Z 7 140-159 127 130 130 219 227 227 F F F <---
144182024-02-12T01:40:42.306Z 8 160-179 122 130 130 207 227 227 F F F <---
144192024-02-12T01:40:42.306Z 9 180-199 120 120 120 205 205 205 F F F
144202024-02-12T01:40:42.306Z 10 200-219 128 128 128 223 223 223 F F F
144212024-02-12T01:40:42.307Z 11 220-239 128 128 128 224 224 224 F F F
144222024-02-12T01:40:42.307Z 12 240-259 125 130 130 212 227 227 F T T <---
144232024-02-12T01:40:42.307Z 13 260-279 125 130 130 212 227 227 F F F <---
144242024-02-12T01:40:42.307Z 14 280-299 123 123 123 208 208 208 F F F
144252024-02-12T01:40:42.307Z 15 300-319 127 127 127 219 219 219 F F F
144262024-02-12T01:40:42.307Z 16 320-339 126 126 126 218 218 218 F F F
144272024-02-12T01:40:42.307Z 17 340-359 120 120 120 205 205 205 F F F
144282024-02-12T01:40:42.307Z 18 360-379 126 130 130 213 228 228 F F F <---
144292024-02-12T01:40:42.307Z 19 380-399 126 126 126 213 213 213 F T T <---
144302024-02-12T01:40:42.307Z 20 400-419 123 130 130 208 226 226 F F F <---
144312024-02-12T01:40:42.307Z 21 420-439 127 130 130 219 226 226 F F F <---
144322024-02-12T01:40:42.307Z 22 440-459 129 129 129 225 225 225 F T T <---
144332024-02-12T01:40:42.307Z 23 460-479 129 130 130 225 227 227 F F F <---
144342024-02-12T01:40:42.308Z 24 480-499 126 126 126 218 218 218 F F F
144352024-02-12T01:40:42.308Z 25 500-519 126 130 130 218 228 228 F F F <---
144362024-02-12T01:40:42.308Z 26 520-539 124 124 124 211 211 211 F F F
144372024-02-12T01:40:42.308Z 27 540-559 129 129 129 225 225 225 F F F
144382024-02-12T01:40:42.308Z 28 560-579 129 129 129 225 225 225 F F F
144392024-02-12T01:40:42.308Z 29 580-599 128 130 130 224 226 226 F F F <---
144402024-02-12T01:40:42.308ZMax gen: 130, Max flush: 228
144412024-02-12T01:40:42.308ZError: Difference in extent metadata found!
144422024-02-12T01:40:42.308ZA Difference in extent metadata is expected here
144432024-02-12T01:40:42.308ZOn loop 65
144442024-02-12T01:40:42.308Z
144452024-02-12T01:40:42.308ZVerifying data now
144462024-02-12T01:40:47.070ZNo change after verify
144472024-02-12T01:40:47.070ZLoop: 65 Downstairs dump after verify (and repair):
144482024-02-12T01:40:47.085ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144492024-02-12T01:40:47.085Z 0 000-019 130 130 130 227 227 227 F F F
144502024-02-12T01:40:47.085Z 1 020-039 130 130 130 227 227 227 F F F
144512024-02-12T01:40:47.085Z 2 040-059 125 125 125 212 212 212 F F F
144522024-02-12T01:40:47.085Z 3 060-079 130 130 130 226 226 226 F F F
144532024-02-12T01:40:47.085Z 4 080-099 129 129 129 225 225 225 F F F
144542024-02-12T01:40:47.085Z 5 100-119 129 129 129 225 225 225 F F F
144552024-02-12T01:40:47.085Z 6 120-139 130 130 130 228 228 228 F F F
144562024-02-12T01:40:47.085Z 7 140-159 130 130 130 227 227 227 F F F
144572024-02-12T01:40:47.085Z 8 160-179 130 130 130 227 227 227 F F F
144582024-02-12T01:40:47.089Z 9 180-199 120 120 120 205 205 205 F F F
144592024-02-12T01:40:47.089Z 10 200-219 128 128 128 223 223 223 F F F
144602024-02-12T01:40:47.089Z 11 220-239 128 128 128 224 224 224 F F F
144612024-02-12T01:40:47.089Z 12 240-259 131 131 131 229 229 229 F F F
144622024-02-12T01:40:47.089Z 13 260-279 130 130 130 227 227 227 F F F
144632024-02-12T01:40:47.089Z 14 280-299 123 123 123 208 208 208 F F F
144642024-02-12T01:40:47.089Z 15 300-319 127 127 127 219 219 219 F F F
144652024-02-12T01:40:47.089Z 16 320-339 126 126 126 218 218 218 F F F
144662024-02-12T01:40:47.089Z 17 340-359 120 120 120 205 205 205 F F F
144672024-02-12T01:40:47.089Z 18 360-379 130 130 130 228 228 228 F F F
144682024-02-12T01:40:47.089Z 19 380-399 131 131 131 229 229 229 F F F
144692024-02-12T01:40:47.089Z 20 400-419 130 130 130 226 226 226 F F F
144702024-02-12T01:40:47.089Z 21 420-439 130 130 130 226 226 226 F F F
144712024-02-12T01:40:47.089Z 22 440-459 131 131 131 229 229 229 F F F
144722024-02-12T01:40:47.089Z 23 460-479 130 130 130 227 227 227 F F F
144732024-02-12T01:40:47.089Z 24 480-499 126 126 126 218 218 218 F F F
144742024-02-12T01:40:47.089Z 25 500-519 130 130 130 228 228 228 F F F
144752024-02-12T01:40:47.089Z 26 520-539 124 124 124 211 211 211 F F F
144762024-02-12T01:40:47.089Z 27 540-559 129 129 129 225 225 225 F F F
144772024-02-12T01:40:47.089Z 28 560-579 129 129 129 225 225 225 F F F
144782024-02-12T01:40:47.090Z 29 580-599 130 130 130 226 226 226 F F F
144792024-02-12T01:40:47.090ZMax gen: 131, Max flush: 229
144802024-02-12T01:40:47.090Z
144812024-02-12T01:40:47.090ZBegin loop 66
144822024-02-12T01:40:47.090ZDownstairs to restart: 0
144832024-02-12T01:40:47.107Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986055502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":Crucible runtime is spawned
144842024-02-12T01:40:47.107Z1663}
144852024-02-12T01:40:47.111Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986388416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663}
144862024-02-12T01:40:47.111Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986435984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663}
144872024-02-12T01:40:47.111Z{"msg":"Crucible 12667c94-24cc-490b-99bb-f15df0d013c1 has session id: 815718b0-b95e-4395-9564-b6f2b33d908e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986485641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144882024-02-12T01:40:47.111Z{"msg":"Upstairs opts: Upstairs UUID: 12667c94-24cc-490b-99bb-f15df0d013c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986547838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144892024-02-12T01:40:47.111Z{"msg":"Crucible stats registered with UUID: 12667c94-24cc-490b-99bb-f15df0d013c1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986595616Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144902024-02-12T01:40:47.111Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986641514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663}
144912024-02-12T01:40:47.111Z{"msg":"12667c94-24cc-490b-99bb-f15df0d013c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986669623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144922024-02-12T01:40:47.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986713141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144932024-02-12T01:40:47.111Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986777527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144942024-02-12T01:40:47.111Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986819225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144952024-02-12T01:40:47.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986864443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144962024-02-12T01:40:47.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986900961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144972024-02-12T01:40:47.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.9869334Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144982024-02-12T01:40:47.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.986977198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
144992024-02-12T01:40:47.112Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987018296Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145002024-02-12T01:40:47.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987063624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145012024-02-12T01:40:47.112Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:45.987118601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145022024-02-12T01:40:47.112Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:45.987170829Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145032024-02-12T01:40:47.112Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:45.987226176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145042024-02-12T01:40:47.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987266044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145052024-02-12T01:40:47.112Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987290713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145062024-02-12T01:40:47.112Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987323431Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145072024-02-12T01:40:47.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.9873499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145082024-02-12T01:40:47.112Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987376769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145092024-02-12T01:40:47.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987670325Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145102024-02-12T01:40:47.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987725792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145112024-02-12T01:40:47.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.987754761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145122024-02-12T01:40:47.112Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.98777815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145132024-02-12T01:40:47.112Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.988325644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145142024-02-12T01:40:47.112Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.988364612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145152024-02-12T01:40:47.118Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.988928205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145162024-02-12T01:40:47.118Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.988964653Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145172024-02-12T01:40:47.119Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.988980572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145182024-02-12T01:40:47.119Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.989001711Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145192024-02-12T01:40:47.119Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.98902355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145202024-02-12T01:40:47.119Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:45.989045029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145212024-02-12T01:40:48.110Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.988557363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145222024-02-12T01:40:48.110Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.988714926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145232024-02-12T01:40:48.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.989605883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145242024-02-12T01:40:48.115Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.989644181Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145252024-02-12T01:40:48.115Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.9900893Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145262024-02-12T01:40:48.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990593036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145272024-02-12T01:40:48.115Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990626465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145282024-02-12T01:40:48.115Z{"msg":"[0]R flush_numbers[0..12]: [227, 227, 212, 226, 225, 225, 228, 227, 227, 205, 223, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990642124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145292024-02-12T01:40:48.115Z{"msg":"[0]R generation[0..12]: [130, 130, 125, 130, 129, 129, 130, 130, 130, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990666543Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145302024-02-12T01:40:48.115Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990757238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145312024-02-12T01:40:48.115Z{"msg":"[1]R flush_numbers[0..12]: [227, 227, 212, 226, 225, 225, 228, 227, 227, 205, 223, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990785657Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145322024-02-12T01:40:48.115Z{"msg":"[1]R generation[0..12]: [130, 130, 125, 130, 129, 129, 130, 130, 130, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990801246Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145332024-02-12T01:40:48.115Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990815916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145342024-02-12T01:40:48.115Z{"msg":"[2]R flush_numbers[0..12]: [227, 227, 212, 226, 225, 225, 228, 227, 227, 205, 223, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990837295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145352024-02-12T01:40:48.115Z{"msg":"[2]R generation[0..12]: [130, 130, 125, 130, 129, 129, 130, 130, 130, 120, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990852584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145362024-02-12T01:40:48.115Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990866743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145372024-02-12T01:40:48.115Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990886762Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145382024-02-12T01:40:48.115Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990900992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145392024-02-12T01:40:48.115Z{"msg":"Next flush: 230","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
145402024-02-12T01:40:48.115Z,"time":"2024-02-12T01:40:46.990914001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145412024-02-12T01:40:48.115Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.99093019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145422024-02-12T01:40:48.115Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990952489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145432024-02-12T01:40:48.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990965109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145442024-02-12T01:40:48.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990978358Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145452024-02-12T01:40:48.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.990991977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145462024-02-12T01:40:48.116Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.991005416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145472024-02-12T01:40:48.116Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.991018146Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145482024-02-12T01:40:48.116Z{"msg":"12667c94-24cc-490b-99bb-f15df0d013c1 is now active with session: 815718b0-b95e-4395-9564-b6f2b33d908e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.991038865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145492024-02-12T01:40:48.116Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.991051074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663,"session_id":"815718b0-b95e-4395-9564-b6f2b33d908e"}
145502024-02-12T01:40:48.116Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:46.991063484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663}
145512024-02-12T01:40:48.116ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145522024-02-12T01:40:48.116ZLoading write count information from file "/tmp/test_repair_verify.data"
145532024-02-12T01:40:48.116ZRun Repair workload
145542024-02-12T01:40:48.116Z01/30 Write block 416 len 36864 data: 12 12 9 8 9 10 9 7 10
145552024-02-12T01:40:48.119Z02/30 Write block 533 len 16384 data: 15 14 15 14
145562024-02-12T01:40:48.119Z03/30 Read block 502 len 40960
145572024-02-12T01:40:48.119Z04/30 Write block 21 len 12288 data: 9 10 10
145582024-02-12T01:40:48.119Z05/30 Read block 270 len 16384
145592024-02-12T01:40:48.120Z06/30 Flush
145602024-02-12T01:40:48.122Z07/30 Write block 195 len 4096 data: 5
145612024-02-12T01:40:48.122Z08/30 Write block 520 len 36864 data: 10 9 9 10 9 6 7 5 7
145622024-02-12T01:40:48.123Z09/30 Read block 1 len 8192
145632024-02-12T01:40:48.125Z10/30 Read block 499 len 32768
145642024-02-12T01:40:48.125Z11/30 Flush
145652024-02-12T01:40:48.128Z12/30 Write block 345 len 12288 data: 6 7 7
145662024-02-12T01:40:48.128Z13/30 Read block 132 len 12288
145672024-02-12T01:40:48.128Z14/30 Read block 343 len 12288
145682024-02-12T01:40:48.131Z15/30 Read block 294 len 4096
145692024-02-12T01:40:48.131Z16/30 Flush
145702024-02-12T01:40:48.138Z17/30 Write block 391 len 20480 data: 15 15 16 14 10
145712024-02-12T01:40:48.138Z18/30 Read block 45 len 24576
145722024-02-12T01:40:48.138Z19/30 Read block 456 len 28672
145732024-02-12T01:40:48.141Z20/30 Write block 105 len 36864 data: 11 10 11 11 10 9 9 6 9
145742024-02-12T01:40:48.141Z21/30 Write block 71 len 32768 data: 13 9 8 8 10 10 9 9
145752024-02-12T01:40:48.141Z22/30 Write block 191 len 40960 data: 10 10 8 6 6 6 8 6 9 9
145762024-02-12T01:40:48.141Z23/30 Read block 592 len 16384
145772024-02-12T01:40:48.141Z24/30 Read block 151 len 12288
145782024-02-12T01:40:48.141Z25/30 Read block 49 len 12288
145792024-02-12T01:40:48.141Z26/30 Flush
145802024-02-12T01:40:48.144Z27/30 Write block 364 len 20480 data: 10 10 9 14 11
145812024-02-12T01:40:48.144Z28/30 Write block 458 len 16384 data: 11 8 7 7
145822024-02-12T01:40:48.144Z29/30 Read block 239 len 4096
145832024-02-12T01:40:48.148Z30/30 Write block 496 len 16384 data: 11 11 12 12
145842024-02-12T01:40:48.148Z----------------------------------------------------------------
145852024-02-12T01:40:48.148Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
145862024-02-12T01:40:48.148ZStates: Active Active Active
145872024-02-12T01:40:48.148ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145882024-02-12T01:40:48.148Z 1 Acked 1000 Write 9 Done Done Done false
145892024-02-12T01:40:48.148Z 2 Acked 1001 Write 4 Done Done Done false
145902024-02-12T01:40:48.148Z 3 Acked 1002 Read 10 Sent Done Done false
145912024-02-12T01:40:48.148Z 4 Acked 1003 Write 3 Sent Done Done false
145922024-02-12T01:40:48.148Z 5 Acked 1004 Read 4 Sent Done Done false
145932024-02-12T01:40:48.148Z 6 Acked 1005 Flush 0 Sent Done Done false
145942024-02-12T01:40:48.148Z 7 Acked 1006 Write 1 Sent Done Done false
145952024-02-12T01:40:48.148Z 8 Acked 1007 Write 9 Sent Done Done false
145962024-02-12T01:40:48.148Z 9 Acked 1008 Read 2 Sent Done Done false
145972024-02-12T01:40:48.148Z 10 Acked 1009 Read 8 Sent Done Done false
145982024-02-12T01:40:48.148Z 11 Acked 1010 Flush 0 Sent Done Done false
145992024-02-12T01:40:48.148Z 12 Acked 1011 Write 3 Sent Done Done false
146002024-02-12T01:40:48.148Z 13 Acked 1012 Read 3 Sent Done Done false
146012024-02-12T01:40:48.148Z 14 Acked 1013 Read 3 Sent Done Done false
146022024-02-12T01:40:48.148Z 15 Acked 1014 Read 1 Sent Done Done false
146032024-02-12T01:40:48.148Z 16 Acked 1015 Flush 0 Sent Done Done false
146042024-02-12T01:40:48.149Z 17 Acked 1016 Write 5 Sent Done Done false
146052024-02-12T01:40:48.149Z 18 Acked 1017 Read 6 Sent Done Done false
146062024-02-12T01:40:48.149Z 19 Acked 1018 Read 7 Sent Done Done false
146072024-02-12T01:40:48.149Z 20 Acked 1019 Write 9 Sent Done Done false
146082024-02-12T01:40:48.149Z 21 Acked 1020 Write 8 Sent Done Done false
146092024-02-12T01:40:48.149Z 22 Acked 1021 Write 10 Sent Done Done false
146102024-02-12T01:40:48.149Z 23 Acked 1022 Read 4 Sent Done Done false
146112024-02-12T01:40:48.149Z 24 Acked 1023 Read 3 Sent Done Done false
146122024-02-12T01:40:48.149Z 25 Acked 1024 Read 3 Sent Done Done false
146132024-02-12T01:40:48.149Z 26 Acked 1025 Flush 0 Sent Done Done false
146142024-02-12T01:40:48.149Z 27 Acked 1026 Write 5 Sent Done Done false
146152024-02-12T01:40:48.149Z 28 Acked 1027 Write 4 Sent Done Done false
146162024-02-12T01:40:48.149Z 29 Acked 1028 Read 1 Sent Done Done false
146172024-02-12T01:40:48.149Z 30 Acked 1029 Write 4 Sent Sent Sent false
146182024-02-12T01:40:48.149Z STATES DS:0 DS:1 DS:2 TOTAL
146192024-02-12T01:40:48.149Z New 0 0 0 0
146202024-02-12T01:40:48.149Z Sent 28 1 1 30
146212024-02-12T01:40:48.149Z Done 2 29 29 60
146222024-02-12T01:40:48.149Z Skipped 0 0 0 0
146232024-02-12T01:40:48.149Z Error 0 0 0 0
146242024-02-12T01:40:48.149ZLast Flush: 0 1025 1025
146252024-02-12T01:40:48.149ZDownstairs last five completed:
146262024-02-12T01:40:48.149ZUpstairs last five completed: 30 29 28 27 26
146272024-02-12T01:40:48.149Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:47.024851042Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1663}
146282024-02-12T01:40:48.149ZWrote out file "/tmp/test_repair_verify.data"
146292024-02-12T01:40:48.153Z
146302024-02-12T01:40:48.155ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146312024-02-12T01:40:48.171ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146322024-02-12T01:40:48.171Z 0 000-019 130 130 130 227 227 227 F F F
146332024-02-12T01:40:48.171Z 1 020-039 130 132 132 227 230 230 F F F <---
146342024-02-12T01:40:48.171Z 2 040-059 125 125 125 212 212 212 F F F
146352024-02-12T01:40:48.171Z 3 060-079 130 132 132 226 233 233 F F F <---
146362024-02-12T01:40:48.171Z 4 080-099 129 129 129 225 225 225 F F F
146372024-02-12T01:40:48.171Z 5 100-119 129 132 132 225 233 233 F F F <---
146382024-02-12T01:40:48.171Z 6 120-139 130 130 130 228 228 228 F F F
146392024-02-12T01:40:48.171Z 7 140-159 130 130 130 227 227 227 F F F
146402024-02-12T01:40:48.175Z 8 160-179 130 130 130 227 227 227 F F F
146412024-02-12T01:40:48.175Z 9 180-199 120 132 132 205 233 233 F F F <---
146422024-02-12T01:40:48.175Z 10 200-219 128 132 132 223 233 233 F F F <---
146432024-02-12T01:40:48.175Z 11 220-239 128 128 128 224 224 224 F F F
146442024-02-12T01:40:48.175Z 12 240-259 131 131 131 229 229 229 F F F
146452024-02-12T01:40:48.175Z 13 260-279 130 130 130 227 227 227 F F F
146462024-02-12T01:40:48.175Z 14 280-299 123 123 123 208 208 208 F F F
146472024-02-12T01:40:48.175Z 15 300-319 127 127 127 219 219 219 F F F
146482024-02-12T01:40:48.175Z 16 320-339 126 126 126 218 218 218 F F F
146492024-02-12T01:40:48.175Z 17 340-359 120 132 132 205 232 232 F F F <---
146502024-02-12T01:40:48.175Z 18 360-379 130 130 130 228 228 228 F T T <---
146512024-02-12T01:40:48.175Z 19 380-399 131 132 132 229 233 233 F F F <---
146522024-02-12T01:40:48.175Z 20 400-419 130 132 132 226 230 230 T F F <---
146532024-02-12T01:40:48.175Z 21 420-439 130 132 132 226 230 230 T F F <---
146542024-02-12T01:40:48.175Z 22 440-459 131 131 131 229 229 229 F T T <---
146552024-02-12T01:40:48.175Z 23 460-479 130 130 130 227 227 227 F T T <---
146562024-02-12T01:40:48.175Z 24 480-499 126 126 126 218 218 218 F T T <---
146572024-02-12T01:40:48.175Z 25 500-519 130 130 130 228 228 228 F F F
146582024-02-12T01:40:48.175Z 26 520-539 124 132 132 211 231 231 T F F <---
146592024-02-12T01:40:48.176Z 27 540-559 129 129 129 225 225 225 F F F
146602024-02-12T01:40:48.176Z 28 560-579 129 129 129 225 225 225 F F F
146612024-02-12T01:40:48.176Z 29 580-599 130 130 130 226 226 226 F F F
146622024-02-12T01:40:48.176ZMax gen: 132, Max flush: 233
146632024-02-12T01:40:48.176ZError: Difference in extent metadata found!
146642024-02-12T01:40:48.176ZA Difference in extent metadata is expected here
146652024-02-12T01:40:48.176ZOn loop 66
146662024-02-12T01:40:48.176Z
146672024-02-12T01:40:48.179ZVerifying data now
146682024-02-12T01:40:53.867ZNo change after verify
146692024-02-12T01:40:53.867ZLoop: 66 Downstairs dump after verify (and repair):
146702024-02-12T01:40:53.882ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146712024-02-12T01:40:53.882Z 0 000-019 130 130 130 227 227 227 F F F
146722024-02-12T01:40:53.882Z 1 020-039 132 132 132 230 230 230 F F F
146732024-02-12T01:40:53.882Z 2 040-059 125 125 125 212 212 212 F F F
146742024-02-12T01:40:53.882Z 3 060-079 132 132 132 233 233 233 F F F
146752024-02-12T01:40:53.882Z 4 080-099 129 129 129 225 225 225 F F F
146762024-02-12T01:40:53.882Z 5 100-119 132 132 132 233 233 233 F F F
146772024-02-12T01:40:53.882Z 6 120-139 130 130 130 228 228 228 F F F
146782024-02-12T01:40:53.882Z 7 140-159 130 130 130 227 227 227 F F F
146792024-02-12T01:40:53.882Z 8 160-179 130 130 130 227 227 227 F F F
146802024-02-12T01:40:53.882Z 9 180-199 132 132 132 233 233 233 F F F
146812024-02-12T01:40:53.882Z 10 200-219 132 132 132 233 233 233 F F F
146822024-02-12T01:40:53.882Z 11 220-239 128 128 128 224 224 224 F F F
146832024-02-12T01:40:53.882Z 12 240-259 131 131 131 229 229 229 F F F
146842024-02-12T01:40:53.886Z 13 260-279 130 130 130 227 227 227 F F F
146852024-02-12T01:40:53.886Z 14 280-299 123 123 123 208 208 208 F F F
146862024-02-12T01:40:53.886Z 15 300-319 127 127 127 219 219 219 F F F
146872024-02-12T01:40:53.886Z 16 320-339 126 126 126 218 218 218 F F F
146882024-02-12T01:40:53.886Z 17 340-359 132 132 132 232 232 232 F F F
146892024-02-12T01:40:53.886Z 18 360-379 133 133 133 234 234 234 F F F
146902024-02-12T01:40:53.886Z 19 380-399 132 132 132 233 233 233 F F F
146912024-02-12T01:40:53.886Z 20 400-419 132 132 132 230 230 230 F F F
146922024-02-12T01:40:53.886Z 21 420-439 132 132 132 230 230 230 F F F
146932024-02-12T01:40:53.886Z 22 440-459 133 133 133 234 234 234 F F F
146942024-02-12T01:40:53.886Z 23 460-479 133 133 133 234 234 234 F F F
146952024-02-12T01:40:53.886Z 24 480-499 133 133 133 234 234 234 F F F
146962024-02-12T01:40:53.886Z 25 500-519 130 130 130 228 228 228 F F F
146972024-02-12T01:40:53.886Z 26 520-539 132 132 132 231 231 231 F F F
146982024-02-12T01:40:53.886Z 27 540-559 129 129 129 225 225 225 F F F
146992024-02-12T01:40:53.886Z 28 560-579 129 129 129 225 225 225 F F F
147002024-02-12T01:40:53.886Z 29 580-599 130 130 130 226 226 226 F F F
147012024-02-12T01:40:53.886ZMax gen: 133, Max flush: 234
147022024-02-12T01:40:53.887Z
147032024-02-12T01:40:53.887ZBegin loop 67
147042024-02-12T01:40:53.887ZDownstairs to restart: 2
147052024-02-12T01:40:53.904Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.782707889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
147062024-02-12T01:40:53.904Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.782935728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
147072024-02-12T01:40:53.909Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.782973606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
147082024-02-12T01:40:53.909Z{"msg":"Crucible 483e0ac8-b617-42d6-9180-2514d8b5a96f has session id: ba5c09e6-b869-4e38-b80c-2e8d38b47cdb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783006575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147092024-02-12T01:40:53.909Z{"msg":"Upstairs opts: Upstairs UUID: 483e0ac8-b617-42d6-9180-2514d8b5a96f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783042503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147102024-02-12T01:40:53.909Z{"msg":"Crucible stats registered with UUID: 483e0ac8-b617-42d6-9180-2514d8b5a96f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783081001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147112024-02-12T01:40:53.909ZCrucible runtime is spawned
147122024-02-12T01:40:53.909Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783229384Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
147132024-02-12T01:40:53.909Z{"msg":"483e0ac8-b617-42d6-9180-2514d8b5a96f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.78332466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147142024-02-12T01:40:53.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783351248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147152024-02-12T01:40:53.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783366898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147162024-02-12T01:40:53.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783381237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147172024-02-12T01:40:53.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783403926Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147182024-02-12T01:40:53.909Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783429055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147192024-02-12T01:40:53.909Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783468403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147202024-02-12T01:40:53.909Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783493952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147212024-02-12T01:40:53.909Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.78353256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147222024-02-12T01:40:53.909Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783556519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147232024-02-12T01:40:53.909Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783719321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147242024-02-12T01:40:53.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783769619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147252024-02-12T01:40:53.910Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.783872534Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147262024-02-12T01:40:53.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.784461586Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147272024-02-12T01:40:53.910Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.784500924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147282024-02-12T01:40:53.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.784517963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147292024-02-12T01:40:53.910Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.784532892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147302024-02-12T01:40:53.910Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785014479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147312024-02-12T01:40:53.910Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785141473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147322024-02-12T01:40:53.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785457838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147332024-02-12T01:40:53.910Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785484287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147342024-02-12T01:40:53.910Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785501506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147352024-02-12T01:40:53.910Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785538794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147362024-02-12T01:40:53.910Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785558193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147372024-02-12T01:40:53.910Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785583742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147382024-02-12T01:40:53.910Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785639879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147392024-02-12T01:40:53.910Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.785666328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147402024-02-12T01:40:53.910Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786185453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147412024-02-12T01:40:53.914Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786605313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147422024-02-12T01:40:53.914Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786641771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147432024-02-12T01:40:53.914Z{"msg":"[0]R flush_numbers[0..12]: [227, 230, 212, 233, 225, 233, 228, 227, 227, 233, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.78667303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147442024-02-12T01:40:53.914Z{"msg":"[0]R generation[0..12]: [130, 132, 125, 132, 129, 132, 130, 130, 130, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786695369Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147452024-02-12T01:40:53.914Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786710318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147462024-02-12T01:40:53.914Z{"msg":"[1]R flush_numbers[0..12]: [227, 230, 212, 233, 225, 233, 228, 227, 227, 233, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786724568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147472024-02-12T01:40:53.914Z{"msg":"[1]R generation[0..12]: [130, 132, 125, 132, 129, 132, 130, 130, 130, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786738567Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147482024-02-12T01:40:53.914Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786752636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147492024-02-12T01:40:53.914Z{"msg":"[2]R flush_numbers[0..12]: [227, 230, 212, 233, 225, 233, 228, 227, 227, 233, 233, 224]","v":Wait for a query_work_queue command to finish before sending IO
147502024-02-12T01:40:53.914Z0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786777055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147512024-02-12T01:40:53.914Z{"msg":"[2]R generation[0..12]: [130, 132, 125, 132, 129, 132, 130, 130, 130, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786799934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147522024-02-12T01:40:53.915Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786814493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147532024-02-12T01:40:53.915Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786834552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147542024-02-12T01:40:53.915Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786848302Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147552024-02-12T01:40:53.915Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786867661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147562024-02-12T01:40:53.915Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.78688084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147572024-02-12T01:40:53.915Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786901259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147582024-02-12T01:40:53.915Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786913499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147592024-02-12T01:40:53.915Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786933158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147602024-02-12T01:40:53.915Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786946397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147612024-02-12T01:40:53.915Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.786977106Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147622024-02-12T01:40:53.915Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.787004644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147632024-02-12T01:40:53.915Z{"msg":"483e0ac8-b617-42d6-9180-2514d8b5a96f is now active with session: ba5c09e6-b869-4e38-b80c-2e8d38b47cdb","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.787019844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147642024-02-12T01:40:53.915Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.787038523Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671,"session_id":"ba5c09e6-b869-4e38-b80c-2e8d38b47cdb"}
147652024-02-12T01:40:53.915Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:52.787051312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
147662024-02-12T01:40:53.915ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147672024-02-12T01:40:53.915ZLoading write count information from file "/tmp/test_repair_verify.data"
147682024-02-12T01:40:53.915ZRun Repair workload
147692024-02-12T01:40:53.915Z01/30 Write block 267 len 36864 data: 13 11 10 11 12 11 12 11 9
147702024-02-12T01:40:53.915Z02/30 Read block 176 len 16384
147712024-02-12T01:40:53.915Z03/30 Read block 558 len 16384
147722024-02-12T01:40:53.915Z04/30 Read block 282 len 4096
147732024-02-12T01:40:53.915Z05/30 Read block 487 len 4096
147742024-02-12T01:40:53.919Z06/30 Read block 223 len 32768
147752024-02-12T01:40:53.919Z07/30 Read block 81 len 24576
147762024-02-12T01:40:53.919Z08/30 Write block 257 len 8192 data: 11 12
147772024-02-12T01:40:53.919Z09/30 Write block 507 len 4096 data: 7
147782024-02-12T01:40:53.919Z10/30 Flush
147792024-02-12T01:40:53.922Z11/30 Write block 368 len 28672 data: 12 13 13 11 13 14 14
147802024-02-12T01:40:53.922Z12/30 Write block 85 len 12288 data: 15 12 15
147812024-02-12T01:40:53.922Z13/30 Read block 3 len 20480
147822024-02-12T01:40:53.922Z14/30 Flush
147832024-02-12T01:40:53.925Z15/30 Write block 414 len 12288 data: 13 14 13
147842024-02-12T01:40:53.925Z16/30 Flush
147852024-02-12T01:40:53.928Z17/30 Write block 26 len 20480 data: 10 11 12 15 14
147862024-02-12T01:40:53.928Z18/30 Read block 156 len 28672
147872024-02-12T01:40:53.933Z19/30 Read block 439 len 32768
147882024-02-12T01:40:53.934Z20/30 Write block 401 len 16384 data: 8 10 8 9
147892024-02-12T01:40:53.934Z21/30 Write block 394 len 28672 data: 15 11 9 9 9 7 8
147902024-02-12T01:40:53.934Z22/30 Write block 171 len 36864 data: 13 10 10 9 12 13 12 13 13
147912024-02-12T01:40:53.936Z23/30 Write block 332 len 4096 data: 7
147922024-02-12T01:40:53.936Z24/30 Read block 410 len 40960
147932024-02-12T01:40:53.936Z25/30 Read block 232 len 24576
147942024-02-12T01:40:53.937Z26/30 Write block 359 len 8192 data: 7 8
147952024-02-12T01:40:53.937Z27/30 Write block 272 len 12288 data: 12 13 12
147962024-02-12T01:40:53.937Z28/30 Read block 288 len 24576
147972024-02-12T01:40:53.937Z29/30 Read block 404 len 4096
147982024-02-12T01:40:53.940Z30/30 Write block 579 len 32768 data: 10 9 10 10 9 10 10 10
147992024-02-12T01:40:53.940Z----------------------------------------------------------------
148002024-02-12T01:40:53.940Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
148012024-02-12T01:40:53.940ZStates: Active Active Active
148022024-02-12T01:40:53.941ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148032024-02-12T01:40:53.941Z 1 Acked 1000 Write 9 Done Done Done false
148042024-02-12T01:40:53.941Z 2 Acked 1001 Read 4 Done Done Done false
148052024-02-12T01:40:53.941Z 3 Acked 1002 Read 4 Done Done Sent false
148062024-02-12T01:40:53.941Z 4 Acked 1003 Read 1 Done Done Sent false
148072024-02-12T01:40:53.941Z 5 Acked 1004 Read 1 Done Done Sent false
148082024-02-12T01:40:53.941Z 6 Acked 1005 Read 8 Done Done Sent false
148092024-02-12T01:40:53.941Z 7 Acked 1006 Read 6 Done Done Sent false
148102024-02-12T01:40:53.941Z 8 Acked 1007 Write 2 Done Done Sent false
148112024-02-12T01:40:53.941Z 9 Acked 1008 Write 1 Done Done Sent false
148122024-02-12T01:40:53.941Z 10 Acked 1009 Flush 0 Done Done Sent false
148132024-02-12T01:40:53.941Z 11 Acked 1010 Write 7 Done Done Sent false
148142024-02-12T01:40:53.941Z 12 Acked 1011 Write 3 Done Done Sent false
148152024-02-12T01:40:53.941Z 13 Acked 1012 Read 5 Done Done Sent false
148162024-02-12T01:40:53.941Z 14 Acked 1013 Flush 0 Done Done Sent false
148172024-02-12T01:40:53.941Z 15 Acked 1014 Write 3 Done Done Sent false
148182024-02-12T01:40:53.941Z 16 Acked 1015 Flush 0 Done Done Sent false
148192024-02-12T01:40:53.941Z 17 Acked 1016 Write 5 Done Done Sent false
148202024-02-12T01:40:53.941Z 18 Acked 1017 Read 7 Done Done Sent false
148212024-02-12T01:40:53.941Z 19 Acked 1018 Read 8 Done Done Sent false
148222024-02-12T01:40:53.941Z 20 Acked 1019 Write 4 Done Done Sent false
148232024-02-12T01:40:53.941Z 21 Acked 1020 Write 7 Done Done Sent false
148242024-02-12T01:40:53.941Z 22 Acked 1021 Write 9 Done Done Sent false
148252024-02-12T01:40:53.942Z 23 Acked 1022 Write 1 Done Done Sent false
148262024-02-12T01:40:53.942Z 24 Acked 1023 Read 10 Done Done Sent false
148272024-02-12T01:40:53.942Z 25 Acked 1024 Read 6 Done Done Sent false
148282024-02-12T01:40:53.942Z 26 Acked 1025 Write 2 Done Done Sent false
148292024-02-12T01:40:53.942Z 27 Acked 1026 Write 3 Done Done Sent false
148302024-02-12T01:40:53.942Z 28 Acked 1027 Read 6 Done Done Sent false
148312024-02-12T01:40:53.942Z 29 Acked 1028 Read 1 Done Done Sent false
148322024-02-12T01:40:53.942Z 30 Acked 1029 Write 8 Sent Sent Sent false
148332024-02-12T01:40:53.942Z STATES DS:0 DS:1 DS:2 TOTAL
148342024-02-12T01:40:53.942Z New 0 0 0 0
148352024-02-12T01:40:53.942Z Sent 1 1 28 30
148362024-02-12T01:40:53.942Z Done 29 29 2 60
148372024-02-12T01:40:53.942Z Skipped 0 0 0 0
148382024-02-12T01:40:53.942Z Error 0 0 0 0
148392024-02-12T01:40:53.942ZLast Flush: 1015 1015 0
148402024-02-12T01:40:53.942ZDownstairs last five completed:
148412024-02-12T01:40:53.942ZUpstairs last five completed: 30 29 28 27 26
148422024-02-12T01:40:53.942Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:52.816622561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1671}
148432024-02-12T01:40:53.942ZWrote out file "/tmp/test_repair_verify.data"
148442024-02-12T01:40:53.945Z
148452024-02-12T01:40:53.946ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148462024-02-12T01:40:53.964ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148472024-02-12T01:40:53.964Z 0 000-019 130 130 130 227 227 227 F F F
148482024-02-12T01:40:53.964Z 1 020-039 132 132 132 230 230 230 T T F <---
148492024-02-12T01:40:53.964Z 2 040-059 125 125 125 212 212 212 F F F
148502024-02-12T01:40:53.964Z 3 060-079 132 132 132 233 233 233 F F F
148512024-02-12T01:40:53.964Z 4 080-099 134 134 129 236 236 225 F F F <---
148522024-02-12T01:40:53.964Z 5 100-119 132 132 132 233 233 233 F F F
148532024-02-12T01:40:53.964Z 6 120-139 130 130 130 228 228 228 F F F
148542024-02-12T01:40:53.964Z 7 140-159 130 130 130 227 227 227 F F F
148552024-02-12T01:40:53.964Z 8 160-179 130 130 130 227 227 227 T T F <---
148562024-02-12T01:40:53.964Z 9 180-199 132 132 132 233 233 233 F F F
148572024-02-12T01:40:53.964Z 10 200-219 132 132 132 233 233 233 F F F
148582024-02-12T01:40:53.968Z 11 220-239 128 128 128 224 224 224 F F F
148592024-02-12T01:40:53.968Z 12 240-259 134 134 131 235 235 229 F F F <---
148602024-02-12T01:40:53.968Z 13 260-279 134 134 130 235 235 227 T T T <---
148612024-02-12T01:40:53.968Z 14 280-299 123 123 123 208 208 208 F F F
148622024-02-12T01:40:53.968Z 15 300-319 127 127 127 219 219 219 F F F
148632024-02-12T01:40:53.968Z 16 320-339 126 126 126 218 218 218 T T F <---
148642024-02-12T01:40:53.968Z 17 340-359 132 132 132 232 232 232 T T F <---
148652024-02-12T01:40:53.968Z 18 360-379 134 134 133 236 236 234 T T F <---
148662024-02-12T01:40:53.968Z 19 380-399 132 132 132 233 233 233 T T F <---
148672024-02-12T01:40:53.968Z 20 400-419 134 134 132 237 237 230 T T F <---
148682024-02-12T01:40:53.968Z 21 420-439 132 132 132 230 230 230 F F F
148692024-02-12T01:40:53.968Z 22 440-459 133 133 133 234 234 234 F F F
148702024-02-12T01:40:53.968Z 23 460-479 133 133 133 234 234 234 F F F
148712024-02-12T01:40:53.968Z 24 480-499 133 133 133 234 234 234 F F F
148722024-02-12T01:40:53.968Z 25 500-519 134 134 130 235 235 228 F F F <---
148732024-02-12T01:40:53.968Z 26 520-539 132 132 132 231 231 231 F F F
148742024-02-12T01:40:53.968Z 27 540-559 129 129 129 225 225 225 F F F
148752024-02-12T01:40:53.968Z 28 560-579 129 129 129 225 225 225 T T F <---
148762024-02-12T01:40:53.968Z 29 580-599 130 130 130 226 226 226 T T F <---
148772024-02-12T01:40:53.968ZMax gen: 134, Max flush: 237
148782024-02-12T01:40:53.968ZError: Difference in extent metadata found!
148792024-02-12T01:40:53.968ZA Difference in extent metadata is expected here
148802024-02-12T01:40:53.968ZOn loop 67
148812024-02-12T01:40:53.969Z
148822024-02-12T01:40:53.972ZVerifying data now
148832024-02-12T01:40:59.624ZNo change after verify
148842024-02-12T01:40:59.624ZLoop: 67 Downstairs dump after verify (and repair):
148852024-02-12T01:40:59.640ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148862024-02-12T01:40:59.640Z 0 000-019 130 130 130 227 227 227 F F F
148872024-02-12T01:40:59.640Z 1 020-039 135 135 135 238 238 238 F F F
148882024-02-12T01:40:59.640Z 2 040-059 125 125 125 212 212 212 F F F
148892024-02-12T01:40:59.640Z 3 060-079 132 132 132 233 233 233 F F F
148902024-02-12T01:40:59.640Z 4 080-099 134 134 134 236 236 236 F F F
148912024-02-12T01:40:59.640Z 5 100-119 132 132 132 233 233 233 F F F
148922024-02-12T01:40:59.640Z 6 120-139 130 130 130 228 228 228 F F F
148932024-02-12T01:40:59.640Z 7 140-159 130 130 130 227 227 227 F F F
148942024-02-12T01:40:59.640Z 8 160-179 135 135 135 238 238 238 F F F
148952024-02-12T01:40:59.641Z 9 180-199 132 132 132 233 233 233 F F F
148962024-02-12T01:40:59.641Z 10 200-219 132 132 132 233 233 233 F F F
148972024-02-12T01:40:59.641Z 11 220-239 128 128 128 224 224 224 F F F
148982024-02-12T01:40:59.641Z 12 240-259 134 134 134 235 235 235 F F F
148992024-02-12T01:40:59.644Z 13 260-279 135 135 135 238 238 238 F F F
149002024-02-12T01:40:59.644Z 14 280-299 123 123 123 208 208 208 F F F
149012024-02-12T01:40:59.644Z 15 300-319 127 127 127 219 219 219 F F F
149022024-02-12T01:40:59.644Z 16 320-339 135 135 135 238 238 238 F F F
149032024-02-12T01:40:59.644Z 17 340-359 135 135 135 238 238 238 F F F
149042024-02-12T01:40:59.644Z 18 360-379 135 135 135 238 238 238 F F F
149052024-02-12T01:40:59.644Z 19 380-399 135 135 135 238 238 238 F F F
149062024-02-12T01:40:59.644Z 20 400-419 135 135 135 238 238 238 F F F
149072024-02-12T01:40:59.644Z 21 420-439 132 132 132 230 230 230 F F F
149082024-02-12T01:40:59.645Z 22 440-459 133 133 133 234 234 234 F F F
149092024-02-12T01:40:59.645Z 23 460-479 133 133 133 234 234 234 F F F
149102024-02-12T01:40:59.645Z 24 480-499 133 133 133 234 234 234 F F F
149112024-02-12T01:40:59.645Z 25 500-519 134 134 134 235 235 235 F F F
149122024-02-12T01:40:59.645Z 26 520-539 132 132 132 231 231 231 F F F
149132024-02-12T01:40:59.645Z 27 540-559 129 129 129 225 225 225 F F F
149142024-02-12T01:40:59.645Z 28 560-579 135 135 135 238 238 238 F F F
149152024-02-12T01:40:59.645Z 29 580-599 135 135 135 238 238 238 F F F
149162024-02-12T01:40:59.645ZMax gen: 135, Max flush: 238
149172024-02-12T01:40:59.645Z
149182024-02-12T01:40:59.645ZBegin loop 68
149192024-02-12T01:40:59.645ZDownstairs to restart: 1
149202024-02-12T01:40:59.664Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.541698783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
149212024-02-12T01:40:59.664Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.541945282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
149222024-02-12T01:40:59.664Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54197124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
149232024-02-12T01:40:59.668Z{"msg":"Crucible 9eec5617-c0c0-495c-8e69-bbffeae48b73 has session id: 8ea1a5eb-30c7-48f2-849f-3f144644a571","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.541998099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149242024-02-12T01:40:59.668ZCrucible runtime is spawned
149252024-02-12T01:40:59.668Z{"msg":"Upstairs opts: Upstairs UUID: 9eec5617-c0c0-495c-8e69-bbffeae48b73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542056046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149262024-02-12T01:40:59.669Z{"msg":"Crucible stats registered with UUID: 9eec5617-c0c0-495c-8e69-bbffeae48b73","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542092015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149272024-02-12T01:40:59.669Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542111194Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
149282024-02-12T01:40:59.669Z{"msg":"9eec5617-c0c0-495c-8e69-bbffeae48b73 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542295185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149292024-02-12T01:40:59.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542321574Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149302024-02-12T01:40:59.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542338213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149312024-02-12T01:40:59.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542355462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149322024-02-12T01:40:59.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54239989Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149332024-02-12T01:40:59.669Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542440898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149342024-02-12T01:40:59.669Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542461807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149352024-02-12T01:40:59.669Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542498625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149362024-02-12T01:40:59.669Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542540913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149372024-02-12T01:40:59.669Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542592251Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149382024-02-12T01:40:59.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542630109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149392024-02-12T01:40:59.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542667957Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149402024-02-12T01:40:59.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.542697276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149412024-02-12T01:40:59.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.543402022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149422024-02-12T01:40:59.669Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54343854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149432024-02-12T01:40:59.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54345471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149442024-02-12T01:40:59.669Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.543473579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149452024-02-12T01:40:59.669Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544027602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149462024-02-12T01:40:59.669Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54408055Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149472024-02-12T01:40:59.669Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544576716Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149482024-02-12T01:40:59.669Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544617954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149492024-02-12T01:40:59.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544634623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149502024-02-12T01:40:59.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544661112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149512024-02-12T01:40:59.670Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544675791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149522024-02-12T01:40:59.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54469614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149532024-02-12T01:40:59.670Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544890061Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149542024-02-12T01:40:59.670Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.544922909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149552024-02-12T01:40:59.674Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545442245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149562024-02-12T01:40:59.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545877144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149572024-02-12T01:40:59.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545921352Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149582024-02-12T01:40:59.674Z{"msg":"[0]R flush_numbers[0..12]: [227, 238, 212, 233, 236, 233, 228, 227, 238, 233, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545944501Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149592024-02-12T01:40:59.674Z{"msg":"[0]R generation[0..12]: [130, 135, 125, 132, 134, 132, 130, 130, 135, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545971499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149602024-02-12T01:40:59.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.545989649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149612024-02-12T01:40:59.674Z{"msg":"[1]R flush_numbers[0..12]: [227, 238, 212, 233, 236, 233, 228, 227, 238, 233, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546025707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149622024-02-12T01:40:59.675Z{"msg":"[1]R generation[0..12]: [130, 135, 125, 132, 134, 132, 130, 130, 135, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546040676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149632024-02-12T01:40:59.675Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546062065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149642024-02-12T01:40:59.675Z{"msg":"[2]R flush_numbers[0..12]: [227, 238, 212, 233, 236, 233, 228, 227, 238, 233, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546076645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149652024-02-12T01:40:59.675Z{"msg":"[2]R generation[0..12]: [130, 135, 125, 132, 134, 132, 130, 130, 135, 132, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546097464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149662024-02-12T01:40:59.675Z{Wait for a query_work_queue command to finish before sending IO
149672024-02-12T01:40:59.675Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546114513Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149682024-02-12T01:40:59.675Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546134382Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149692024-02-12T01:40:59.675Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546148101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149702024-02-12T01:40:59.675Z{"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54616801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149712024-02-12T01:40:59.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546181589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149722024-02-12T01:40:59.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546196059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149732024-02-12T01:40:59.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546227717Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149742024-02-12T01:40:59.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546259076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149752024-02-12T01:40:59.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546314523Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149762024-02-12T01:40:59.675Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546345202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149772024-02-12T01:40:59.675Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.54637823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149782024-02-12T01:40:59.675Z{"msg":"9eec5617-c0c0-495c-8e69-bbffeae48b73 is now active with session: 8ea1a5eb-30c7-48f2-849f-3f144644a571","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546397209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149792024-02-12T01:40:59.675Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546430888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679,"session_id":"8ea1a5eb-30c7-48f2-849f-3f144644a571"}
149802024-02-12T01:40:59.675Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:40:58.546463526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
149812024-02-12T01:40:59.675ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
149822024-02-12T01:40:59.675ZLoading write count information from file "/tmp/test_repair_verify.data"
149832024-02-12T01:40:59.675ZRun Repair workload
149842024-02-12T01:40:59.675Z01/30 Write block 179 len 16384 data: 14 11 13 11
149852024-02-12T01:40:59.675Z02/30 Read block 502 len 40960
149862024-02-12T01:40:59.676Z03/30 Write block 487 len 8192 data: 13 14
149872024-02-12T01:40:59.676Z04/30 Write block 355 len 4096 data: 7
149882024-02-12T01:40:59.679Z05/30 Read block 276 len 16384
149892024-02-12T01:40:59.679Z06/30 Read block 353 len 4096
149902024-02-12T01:40:59.679Z07/30 Read block 332 len 24576
149912024-02-12T01:40:59.679Z08/30 Read block 274 len 12288
149922024-02-12T01:40:59.679Z09/30 Read block 428 len 28672
149932024-02-12T01:40:59.679Z10/30 Flush
149942024-02-12T01:40:59.682Z11/30 Write block 364 len 36864 data: 11 11 10 15 13 14 14 12 14
149952024-02-12T01:40:59.682Z12/30 Write block 581 len 36864 data: 11 11 10 11 11 11 7 6 6
149962024-02-12T01:40:59.682Z13/30 Read block 243 len 4096
149972024-02-12T01:40:59.688Z14/30 Read block 571 len 36864
149982024-02-12T01:40:59.688Z15/30 Write block 439 len 36864 data: 10 10 11 10 8 9 8 8 9
149992024-02-12T01:40:59.688Z16/30 Write block 522 len 4096 data: 10
150002024-02-12T01:40:59.691Z17/30 Write block 272 len 12288 data: 13 14 13
150012024-02-12T01:40:59.691Z18/30 Read block 469 len 28672
150022024-02-12T01:40:59.691Z19/30 Read block 70 len 8192
150032024-02-12T01:40:59.691Z20/30 Read block 179 len 32768
150042024-02-12T01:40:59.691Z21/30 Write block 101 len 20480 data: 8 7 7 10 12
150052024-02-12T01:40:59.691Z22/30 Write block 46 len 4096 data: 10
150062024-02-12T01:40:59.692Z23/30 Read block 42 len 24576
150072024-02-12T01:40:59.695Z24/30 Write block 287 len 28672 data: 9 10 9 9 10 8 9
150082024-02-12T01:40:59.695Z25/30 Read block 311 len 24576
150092024-02-12T01:40:59.695Z26/30 Write block 265 len 40960 data: 11 13 14 12 11 12 13 14 15 14
150102024-02-12T01:40:59.695Z27/30 Write block 140 len 32768 data: 15 17 19 17 16 16 12 12
150112024-02-12T01:40:59.698Z28/30 Write block 51 len 36864 data: 15 13 10 10 10 10 12 15 11
150122024-02-12T01:40:59.698Z29/30 Write block 536 len 12288 data: 15 14 13
150132024-02-12T01:40:59.698Z30/30 Write block 148 len 4096 data: 12
150142024-02-12T01:40:59.698Z----------------------------------------------------------------
150152024-02-12T01:40:59.698Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:20
150162024-02-12T01:40:59.699ZStates: Active Active Active
150172024-02-12T01:40:59.699ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150182024-02-12T01:40:59.699Z 11 Acked 1010 Write 9 Done Done Done false
150192024-02-12T01:40:59.699Z 12 Acked 1011 Write 9 Done Done Done false
150202024-02-12T01:40:59.699Z 13 Acked 1012 Read 1 Done Done Done false
150212024-02-12T01:40:59.699Z 14 Acked 1013 Read 9 Done Done Done false
150222024-02-12T01:40:59.699Z 15 Acked 1014 Write 9 Done Done Done false
150232024-02-12T01:40:59.699Z 16 Acked 1015 Write 1 Done Done Done false
150242024-02-12T01:40:59.699Z 17 Acked 1016 Write 3 Done Done Done false
150252024-02-12T01:40:59.699Z 18 Acked 1017 Read 7 Done Done Done false
150262024-02-12T01:40:59.699Z 19 Acked 1018 Read 2 Done Done Done false
150272024-02-12T01:40:59.699Z 20 Acked 1019 Read 8 Done Done Done false
150282024-02-12T01:40:59.699Z 21 Acked 1020 Write 5 Done Done Done false
150292024-02-12T01:40:59.699Z 22 Acked 1021 Write 1 Done Done Done false
150302024-02-12T01:40:59.699Z 23 Acked 1022 Read 6 Done Sent Done false
150312024-02-12T01:40:59.699Z 24 Acked 1023 Write 7 Done Sent Done false
150322024-02-12T01:40:59.699Z 25 Acked 1024 Read 6 Done Sent Done false
150332024-02-12T01:40:59.699Z 26 Acked 1025 Write 10 Done Sent Done false
150342024-02-12T01:40:59.699Z 27 Acked 1026 Write 8 Done Sent Done false
150352024-02-12T01:40:59.699Z 28 Acked 1027 Write 9 Sent Sent Sent false
150362024-02-12T01:40:59.699Z 29 Acked 1028 Write 3 Sent Sent Sent false
150372024-02-12T01:40:59.699Z 30 Acked 1029 Write 1 Sent Sent Sent false
150382024-02-12T01:40:59.699Z STATES DS:0 DS:1 DS:2 TOTAL
150392024-02-12T01:40:59.699Z New 0 0 0 0
150402024-02-12T01:40:59.700Z Sent 3 8 3 14
150412024-02-12T01:40:59.700Z Done 17 12 17 46
150422024-02-12T01:40:59.700Z Skipped 0 0 0 0
150432024-02-12T01:40:59.700Z Error 0 0 0 0
150442024-02-12T01:40:59.700ZLast Flush: 1009 1009 1009
150452024-02-12T01:40:59.700ZDownstairs last five completed: 1009 1008 1007 1006 1005
150462024-02-12T01:40:59.700ZUpstairs last five completed: 30 29 28 27 26
150472024-02-12T01:40:59.700Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:40:58.575124149Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1679}
150482024-02-12T01:40:59.700ZWrote out file "/tmp/test_repair_verify.data"
150492024-02-12T01:40:59.703Z
150502024-02-12T01:40:59.706ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150512024-02-12T01:40:59.727ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150522024-02-12T01:40:59.727Z 0 000-019 130 130 130 227 227 227 F F F
150532024-02-12T01:40:59.727Z 1 020-039 135 135 135 238 238 238 F F F
150542024-02-12T01:40:59.727Z 2 040-059 125 125 125 212 212 212 T T T
150552024-02-12T01:40:59.727Z 3 060-079 132 132 132 233 233 233 F F F
150562024-02-12T01:40:59.727Z 4 080-099 134 134 134 236 236 236 F F F
150572024-02-12T01:40:59.727Z 5 100-119 132 132 132 233 233 233 T T T
150582024-02-12T01:40:59.727Z 6 120-139 130 130 130 228 228 228 F F F
150592024-02-12T01:40:59.727Z 7 140-159 130 130 130 227 227 227 T F T <---
150602024-02-12T01:40:59.727Z 8 160-179 136 136 136 239 239 239 F F F
150612024-02-12T01:40:59.727Z 9 180-199 136 136 136 239 239 239 F F F
150622024-02-12T01:40:59.727Z 10 200-219 132 132 132 233 233 233 F F F
150632024-02-12T01:40:59.730Z 11 220-239 128 128 128 224 224 224 F F F
150642024-02-12T01:40:59.730Z 12 240-259 134 134 134 235 235 235 F F F
150652024-02-12T01:40:59.730Z 13 260-279 135 135 135 238 238 238 T T T
150662024-02-12T01:40:59.731Z 14 280-299 123 123 123 208 208 208 T F T <---
150672024-02-12T01:40:59.731Z 15 300-319 127 127 127 219 219 219 F F F
150682024-02-12T01:40:59.731Z 16 320-339 135 135 135 238 238 238 F F F
150692024-02-12T01:40:59.731Z 17 340-359 136 136 136 239 239 239 F F F
150702024-02-12T01:40:59.731Z 18 360-379 135 135 135 238 238 238 T T T
150712024-02-12T01:40:59.731Z 19 380-399 135 135 135 238 238 238 F F F
150722024-02-12T01:40:59.731Z 20 400-419 135 135 135 238 238 238 F F F
150732024-02-12T01:40:59.731Z 21 420-439 132 132 132 230 230 230 T T T
150742024-02-12T01:40:59.731Z 22 440-459 133 133 133 234 234 234 T T T
150752024-02-12T01:40:59.731Z 23 460-479 133 133 133 234 234 234 F F F
150762024-02-12T01:40:59.731Z 24 480-499 136 136 136 239 239 239 F F F
150772024-02-12T01:40:59.731Z 25 500-519 134 134 134 235 235 235 F F F
150782024-02-12T01:40:59.731Z 26 520-539 132 132 132 231 231 231 T T T
150792024-02-12T01:40:59.731Z 27 540-559 129 129 129 225 225 225 F F F
150802024-02-12T01:40:59.731Z 28 560-579 135 135 135 238 238 238 F F F
150812024-02-12T01:40:59.731Z 29 580-599 135 135 135 238 238 238 T T T
150822024-02-12T01:40:59.731ZMax gen: 136, Max flush: 239
150832024-02-12T01:40:59.731ZError: Difference in extent metadata found!
150842024-02-12T01:40:59.731ZA Difference in extent metadata is expected here
150852024-02-12T01:40:59.731ZOn loop 68
150862024-02-12T01:40:59.731Z
150872024-02-12T01:40:59.735ZVerifying data now
150882024-02-12T01:41:01.268ZNo change after verify
150892024-02-12T01:41:01.268ZLoop: 68 Downstairs dump after verify (and repair):
150902024-02-12T01:41:01.283ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150912024-02-12T01:41:01.283Z 0 000-019 130 130 130 227 227 227 F F F
150922024-02-12T01:41:01.283Z 1 020-039 135 135 135 238 238 238 F F F
150932024-02-12T01:41:01.283Z 2 040-059 137 137 137 240 240 240 F F F
150942024-02-12T01:41:01.283Z 3 060-079 132 132 132 233 233 233 F F F
150952024-02-12T01:41:01.283Z 4 080-099 134 134 134 236 236 236 F F F
150962024-02-12T01:41:01.283Z 5 100-119 137 137 137 240 240 240 F F F
150972024-02-12T01:41:01.283Z 6 120-139 130 130 130 228 228 228 F F F
150982024-02-12T01:41:01.283Z 7 140-159 137 137 137 240 240 240 F F F
150992024-02-12T01:41:01.283Z 8 160-179 136 136 136 239 239 239 F F F
151002024-02-12T01:41:01.286Z 9 180-199 136 136 136 239 239 239 F F F
151012024-02-12T01:41:01.286Z 10 200-219 132 132 132 233 233 233 F F F
151022024-02-12T01:41:01.287Z 11 220-239 128 128 128 224 224 224 F F F
151032024-02-12T01:41:01.287Z 12 240-259 134 134 134 235 235 235 F F F
151042024-02-12T01:41:01.287Z 13 260-279 137 137 137 240 240 240 F F F
151052024-02-12T01:41:01.287Z 14 280-299 137 137 137 240 240 240 F F F
151062024-02-12T01:41:01.287Z 15 300-319 127 127 127 219 219 219 F F F
151072024-02-12T01:41:01.287Z 16 320-339 135 135 135 238 238 238 F F F
151082024-02-12T01:41:01.287Z 17 340-359 136 136 136 239 239 239 F F F
151092024-02-12T01:41:01.287Z 18 360-379 137 137 137 240 240 240 F F F
151102024-02-12T01:41:01.287Z 19 380-399 135 135 135 238 238 238 F F F
151112024-02-12T01:41:01.287Z 20 400-419 135 135 135 238 238 238 F F F
151122024-02-12T01:41:01.287Z 21 420-439 137 137 137 240 240 240 F F F
151132024-02-12T01:41:01.287Z 22 440-459 137 137 137 240 240 240 F F F
151142024-02-12T01:41:01.287Z 23 460-479 133 133 133 234 234 234 F F F
151152024-02-12T01:41:01.287Z 24 480-499 136 136 136 239 239 239 F F F
151162024-02-12T01:41:01.287Z 25 500-519 134 134 134 235 235 235 F F F
151172024-02-12T01:41:01.287Z 26 520-539 137 137 137 240 240 240 F F F
151182024-02-12T01:41:01.287Z 27 540-559 129 129 129 225 225 225 F F F
151192024-02-12T01:41:01.287Z 28 560-579 135 135 135 238 238 238 F F F
151202024-02-12T01:41:01.287Z 29 580-599 137 137 137 240 240 240 F F F
151212024-02-12T01:41:01.287ZMax gen: 137, Max flush: 240
151222024-02-12T01:41:01.287Z
151232024-02-12T01:41:01.287ZBegin loop 69
151242024-02-12T01:41:01.287ZDownstairs to restart: 0
151252024-02-12T01:41:01.304Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.181748808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
151262024-02-12T01:41:01.304Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.181987107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
151272024-02-12T01:41:01.304Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182012985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
151282024-02-12T01:41:01.310Z{"msg":"Crucible 61f32160-b61c-44f3-ab25-21ab0a014cbf has session id: 323b9105-5f77-4e36-96c0-b94a276035cf","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182047054Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151292024-02-12T01:41:01.310Z{"msg":"Crucible runtime is spawned
151302024-02-12T01:41:01.310ZUpstairs opts: Upstairs UUID: 61f32160-b61c-44f3-ab25-21ab0a014cbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182100781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151312024-02-12T01:41:01.310Z{"msg":"Crucible stats registered with UUID: 61f32160-b61c-44f3-ab25-21ab0a014cbf","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.18212503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151322024-02-12T01:41:01.310Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182201916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
151332024-02-12T01:41:01.310Z{"msg":"61f32160-b61c-44f3-ab25-21ab0a014cbf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182294542Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151342024-02-12T01:41:01.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182352909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151352024-02-12T01:41:01.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182410857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151362024-02-12T01:41:01.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182458484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151372024-02-12T01:41:01.311Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182494013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151382024-02-12T01:41:01.311Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182536411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151392024-02-12T01:41:01.311Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182587828Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151402024-02-12T01:41:01.311Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182609027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151412024-02-12T01:41:01.311Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182652695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151422024-02-12T01:41:01.311Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182691533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151432024-02-12T01:41:01.311Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182717162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151442024-02-12T01:41:01.311Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182741171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151452024-02-12T01:41:01.311Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:00.182879304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151462024-02-12T01:41:01.311Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:00.182923002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151472024-02-12T01:41:01.311Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:00.182952001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151482024-02-12T01:41:01.311Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.18297479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151492024-02-12T01:41:01.311Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.182988489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151502024-02-12T01:41:01.311Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.183097294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151512024-02-12T01:41:01.311Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.183435068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151522024-02-12T01:41:01.311Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.183471316Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151532024-02-12T01:41:01.311Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.183636948Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151542024-02-12T01:41:01.311Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.183677636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151552024-02-12T01:41:01.311Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184081807Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151562024-02-12T01:41:01.311Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184240069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151572024-02-12T01:41:01.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184543835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151582024-02-12T01:41:01.311Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184600552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151592024-02-12T01:41:01.312Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184660449Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151602024-02-12T01:41:01.312Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184694008Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151612024-02-12T01:41:01.312Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184709697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151622024-02-12T01:41:01.312Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:00.184730666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151632024-02-12T01:41:02.307Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.184622872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151642024-02-12T01:41:02.307Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.184707528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151652024-02-12T01:41:02.313Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.185565397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151662024-02-12T01:41:02.314Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.185607965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151672024-02-12T01:41:02.314Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.18613076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151682024-02-12T01:41:02.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186625606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151692024-02-12T01:41:02.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186664865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151702024-02-12T01:41:02.314Z{"msg":"[0]R flush_numbers[0..12]: [227, 238, 240, 233, 236, 240, 228, 240, 239, 239, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186684113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151712024-02-12T01:41:02.314Z{"msg":"[0]R generation[0..12]: [130, 135, 137, 132, 134, 137, 130, 137, 136, 136, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186703093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151722024-02-12T01:41:02.314Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186729991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151732024-02-12T01:41:02.314Z{"msg":"[1]R flush_numbers[0..12]: [227, 238, 240, 233, 236, 240, 228, 240, 239, 239, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.18674931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151742024-02-12T01:41:02.314Z{"msg":"[1]R generation[0..12]: [130, 135, 137, 132, 134, 137, 130, 137, 136, 136, 132, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186775569Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151752024-02-12T01:41:02.314Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186794888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151762024-02-12T01:41:02.314Z{"msg":"[2]R flush_numbers[0..12]: [227, 238, 240, 233, 236, 240, 228, 240, 239, 239, 233, 224]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186820667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151772024-02-12T01:41:02.314Z{"msg":"[2]R generation[0..12]: [130, 135, 137, 132, 134, 137, 130, 137, 136, 136, 132, 128]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
151782024-02-12T01:41:02.314Z2024-02-12T01:41:01.186840156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151792024-02-12T01:41:02.314Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186873584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151802024-02-12T01:41:02.314Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186898493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151812024-02-12T01:41:02.314Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186916373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151822024-02-12T01:41:02.314Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186940221Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151832024-02-12T01:41:02.314Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186957611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151842024-02-12T01:41:02.314Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186981729Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151852024-02-12T01:41:02.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.186997409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151862024-02-12T01:41:02.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187013908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151872024-02-12T01:41:02.314Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187031037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151882024-02-12T01:41:02.314Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187047696Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151892024-02-12T01:41:02.315Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187071405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151902024-02-12T01:41:02.315Z{"msg":"61f32160-b61c-44f3-ab25-21ab0a014cbf is now active with session: 323b9105-5f77-4e36-96c0-b94a276035cf","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187087554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151912024-02-12T01:41:02.315Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187109813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687,"session_id":"323b9105-5f77-4e36-96c0-b94a276035cf"}
151922024-02-12T01:41:02.315Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:01.187125263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
151932024-02-12T01:41:02.315ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
151942024-02-12T01:41:02.315ZLoading write count information from file "/tmp/test_repair_verify.data"
151952024-02-12T01:41:02.315ZRun Repair workload
151962024-02-12T01:41:02.315Z01/30 Write block 502 len 12288 data: 11 9 8
151972024-02-12T01:41:02.318Z02/30 Read block 124 len 40960
151982024-02-12T01:41:02.319Z03/30 Read block 551 len 28672
151992024-02-12T01:41:02.319Z04/30 Read block 487 len 36864
152002024-02-12T01:41:02.319Z05/30 Write block 91 len 8192 data: 10 8
152012024-02-12T01:41:02.319Z06/30 Read block 98 len 36864
152022024-02-12T01:41:02.319Z07/30 Write block 570 len 28672 data: 13 11 13 16 15 14 10
152032024-02-12T01:41:02.319Z08/30 Write block 565 len 4096 data: 10
152042024-02-12T01:41:02.319Z09/30 Read block 111 len 36864
152052024-02-12T01:41:02.322Z10/30 Read block 94 len 36864
152062024-02-12T01:41:02.322Z11/30 Read block 455 len 8192
152072024-02-12T01:41:02.322Z12/30 Read block 351 len 12288
152082024-02-12T01:41:02.322Z13/30 Write block 199 len 8192 data: 10 10
152092024-02-12T01:41:02.322Z14/30 Flush
152102024-02-12T01:41:02.327Z15/30 Write block 535 len 36864 data: 16 16 15 14 11 11 9 9 6
152112024-02-12T01:41:02.330Z16/30 Write block 193 len 16384 data: 9 7 7 7
152122024-02-12T01:41:02.330Z17/30 Read block 32 len 28672
152132024-02-12T01:41:02.330Z18/30 Write block 234 len 24576 data: 9 9 12 15 16 15
152142024-02-12T01:41:02.330Z19/30 Write block 305 len 20480 data: 8 10 12 13 13
152152024-02-12T01:41:02.330Z20/30 Write block 506 len 12288 data: 7 8 7
152162024-02-12T01:41:02.330Z21/30 Write block 302 len 16384 data: 4 5 7 9
152172024-02-12T01:41:02.333Z22/30 Write block 470 len 8192 data: 9 8
152182024-02-12T01:41:02.333Z23/30 Read block 54 len 12288
152192024-02-12T01:41:02.333Z24/30 Write block 408 len 32768 data: 12 12 13 12 12 14 14 15
152202024-02-12T01:41:02.333Z25/30 Write block 527 len 16384 data: 6 8 8 12
152212024-02-12T01:41:02.334Z26/30 Flush
152222024-02-12T01:41:02.338Z27/30 Write block 330 len 32768 data: 7 8 8 6 6 6 6 6
152232024-02-12T01:41:02.345Z28/30 Write block 396 len 20480 data: 10 10 10 8 9
152242024-02-12T01:41:02.345Z29/30 Read block 73 len 20480
152252024-02-12T01:41:02.345Z30/30 Write block 261 len 4096 data: 9
152262024-02-12T01:41:02.345Z----------------------------------------------------------------
152272024-02-12T01:41:02.345Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
152282024-02-12T01:41:02.345ZStates: Active Active Active
152292024-02-12T01:41:02.345ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152302024-02-12T01:41:02.345Z 1 Acked 1000 Write 3 Done Done Done false
152312024-02-12T01:41:02.345Z 2 Acked 1001 Read 10 Sent Done Done false
152322024-02-12T01:41:02.345Z 3 Acked 1002 Read 7 Sent Done Done false
152332024-02-12T01:41:02.345Z 4 Acked 1003 Read 9 Sent Done Done false
152342024-02-12T01:41:02.345Z 5 Acked 1004 Write 2 Sent Done Done false
152352024-02-12T01:41:02.345Z 6 Acked 1005 Read 9 Sent Done Done false
152362024-02-12T01:41:02.345Z 7 Acked 1006 Write 7 Sent Done Done false
152372024-02-12T01:41:02.345Z 8 Acked 1007 Write 1 Sent Done Done false
152382024-02-12T01:41:02.345Z 9 Acked 1008 Read 9 Sent Done Done false
152392024-02-12T01:41:02.345Z 10 Acked 1009 Read 9 Sent Done Done false
152402024-02-12T01:41:02.345Z 11 Acked 1010 Read 2 Sent Done Done false
152412024-02-12T01:41:02.345Z 12 Acked 1011 Read 3 Sent Done Done false
152422024-02-12T01:41:02.345Z 13 Acked 1012 Write 2 Sent Done Done false
152432024-02-12T01:41:02.345Z 14 Acked 1013 Flush 0 Sent Done Done false
152442024-02-12T01:41:02.345Z 15 Acked 1014 Write 9 Sent Done Done false
152452024-02-12T01:41:02.346Z 16 Acked 1015 Write 4 Sent Done Done false
152462024-02-12T01:41:02.346Z 17 Acked 1016 Read 7 Sent Done Done false
152472024-02-12T01:41:02.346Z 18 Acked 1017 Write 6 Sent Done Done false
152482024-02-12T01:41:02.346Z 19 Acked 1018 Write 5 Sent Done Done false
152492024-02-12T01:41:02.346Z 20 Acked 1019 Write 3 Sent Done Done false
152502024-02-12T01:41:02.346Z 21 Acked 1020 Write 4 Sent Done Done false
152512024-02-12T01:41:02.346Z 22 Acked 1021 Write 2 Sent Done Done false
152522024-02-12T01:41:02.346Z 23 Acked 1022 Read 3 Sent Done Done false
152532024-02-12T01:41:02.346Z 24 Acked 1023 Write 8 Sent Done Done false
152542024-02-12T01:41:02.346Z 25 Acked 1024 Write 4 Sent Done Done false
152552024-02-12T01:41:02.346Z 26 Acked 1025 Flush 0 Sent Done Done false
152562024-02-12T01:41:02.346Z 27 Acked 1026 Write 8 Sent Done Done false
152572024-02-12T01:41:02.346Z 28 Acked 1027 Write 5 Sent Done Done false
152582024-02-12T01:41:02.346Z 29 Acked 1028 Read 5 Sent Done Done false
152592024-02-12T01:41:02.346Z 30 Acked 1029 Write 1 Sent Sent Sent false
152602024-02-12T01:41:02.346Z STATES DS:0 DS:1 DS:2 TOTAL
152612024-02-12T01:41:02.346Z New 0 0 0 0
152622024-02-12T01:41:02.346Z Sent 29 1 1 31
152632024-02-12T01:41:02.346Z Done 1 29 29 59
152642024-02-12T01:41:02.346Z Skipped 0 0 0 0
152652024-02-12T01:41:02.346Z Error 0 0 0 0
152662024-02-12T01:41:02.346ZLast Flush: 0 1025 1025
152672024-02-12T01:41:02.346ZDownstairs last five completed:
152682024-02-12T01:41:02.346ZUpstairs last five completed: 30 29 28 27 26
152692024-02-12T01:41:02.347Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:01.218185911Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1687}
152702024-02-12T01:41:02.347ZWrote out file "/tmp/test_repair_verify.data"
152712024-02-12T01:41:02.352Z
152722024-02-12T01:41:02.352ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
152732024-02-12T01:41:02.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152742024-02-12T01:41:02.366Z 0 000-019 130 130 130 227 227 227 F F F
152752024-02-12T01:41:02.366Z 1 020-039 135 135 135 238 238 238 F F F
152762024-02-12T01:41:02.366Z 2 040-059 137 137 137 240 240 240 F F F
152772024-02-12T01:41:02.366Z 3 060-079 132 132 132 233 233 233 F F F
152782024-02-12T01:41:02.366Z 4 080-099 134 138 138 236 241 241 F F F <---
152792024-02-12T01:41:02.366Z 5 100-119 137 137 137 240 240 240 F F F
152802024-02-12T01:41:02.366Z 6 120-139 130 130 130 228 228 228 F F F
152812024-02-12T01:41:02.366Z 7 140-159 137 137 137 240 240 240 F F F
152822024-02-12T01:41:02.370Z 8 160-179 136 136 136 239 239 239 F F F
152832024-02-12T01:41:02.370Z 9 180-199 136 138 138 239 242 242 F F F <---
152842024-02-12T01:41:02.371Z 10 200-219 132 138 138 233 241 241 F F F <---
152852024-02-12T01:41:02.371Z 11 220-239 128 138 138 224 242 242 F F F <---
152862024-02-12T01:41:02.371Z 12 240-259 134 134 134 235 235 235 F F F
152872024-02-12T01:41:02.371Z 13 260-279 137 137 137 240 240 240 F T T <---
152882024-02-12T01:41:02.371Z 14 280-299 137 137 137 240 240 240 F F F
152892024-02-12T01:41:02.371Z 15 300-319 127 138 138 219 242 242 F F F <---
152902024-02-12T01:41:02.371Z 16 320-339 135 135 135 238 238 238 F T T <---
152912024-02-12T01:41:02.371Z 17 340-359 136 136 136 239 239 239 F F F
152922024-02-12T01:41:02.371Z 18 360-379 137 137 137 240 240 240 F F F
152932024-02-12T01:41:02.371Z 19 380-399 135 135 135 238 238 238 F T T <---
152942024-02-12T01:41:02.371Z 20 400-419 135 138 138 238 242 242 F T T <---
152952024-02-12T01:41:02.371Z 21 420-439 137 137 137 240 240 240 F F F
152962024-02-12T01:41:02.371Z 22 440-459 137 137 137 240 240 240 F F F
152972024-02-12T01:41:02.371Z 23 460-479 133 138 138 234 242 242 F F F <---
152982024-02-12T01:41:02.371Z 24 480-499 136 136 136 239 239 239 F F F
152992024-02-12T01:41:02.371Z 25 500-519 134 138 138 235 242 242 T F F <---
153002024-02-12T01:41:02.371Z 26 520-539 137 138 138 240 242 242 F F F <---
153012024-02-12T01:41:02.371Z 27 540-559 129 138 138 225 242 242 F F F <---
153022024-02-12T01:41:02.371Z 28 560-579 135 138 138 238 241 241 F F F <---
153032024-02-12T01:41:02.371Z 29 580-599 137 137 137 240 240 240 F F F
153042024-02-12T01:41:02.371ZMax gen: 138, Max flush: 242
153052024-02-12T01:41:02.371ZError: Difference in extent metadata found!
153062024-02-12T01:41:02.371ZA Difference in extent metadata is expected here
153072024-02-12T01:41:02.372ZOn loop 69
153082024-02-12T01:41:02.372Z
153092024-02-12T01:41:02.372ZVerifying data now
153102024-02-12T01:41:08.055ZNo change after verify
153112024-02-12T01:41:08.055ZLoop: 69 Downstairs dump after verify (and repair):
153122024-02-12T01:41:08.070ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153132024-02-12T01:41:08.070Z 0 000-019 130 130 130 227 227 227 F F F
153142024-02-12T01:41:08.070Z 1 020-039 135 135 135 238 238 238 F F F
153152024-02-12T01:41:08.070Z 2 040-059 137 137 137 240 240 240 F F F
153162024-02-12T01:41:08.070Z 3 060-079 132 132 132 233 233 233 F F F
153172024-02-12T01:41:08.070Z 4 080-099 138 138 138 241 241 241 F F F
153182024-02-12T01:41:08.074Z 5 100-119 137 137 137 240 240 240 F F F
153192024-02-12T01:41:08.074Z 6 120-139 130 130 130 228 228 228 F F F
153202024-02-12T01:41:08.074Z 7 140-159 137 137 137 240 240 240 F F F
153212024-02-12T01:41:08.074Z 8 160-179 136 136 136 239 239 239 F F F
153222024-02-12T01:41:08.074Z 9 180-199 138 138 138 242 242 242 F F F
153232024-02-12T01:41:08.074Z 10 200-219 138 138 138 241 241 241 F F F
153242024-02-12T01:41:08.074Z 11 220-239 138 138 138 242 242 242 F F F
153252024-02-12T01:41:08.074Z 12 240-259 134 134 134 235 235 235 F F F
153262024-02-12T01:41:08.074Z 13 260-279 139 139 139 243 243 243 F F F
153272024-02-12T01:41:08.074Z 14 280-299 137 137 137 240 240 240 F F F
153282024-02-12T01:41:08.074Z 15 300-319 138 138 138 242 242 242 F F F
153292024-02-12T01:41:08.074Z 16 320-339 139 139 139 243 243 243 F F F
153302024-02-12T01:41:08.074Z 17 340-359 136 136 136 239 239 239 F F F
153312024-02-12T01:41:08.074Z 18 360-379 137 137 137 240 240 240 F F F
153322024-02-12T01:41:08.074Z 19 380-399 139 139 139 243 243 243 F F F
153332024-02-12T01:41:08.074Z 20 400-419 139 139 139 243 243 243 F F F
153342024-02-12T01:41:08.074Z 21 420-439 137 137 137 240 240 240 F F F
153352024-02-12T01:41:08.074Z 22 440-459 137 137 137 240 240 240 F F F
153362024-02-12T01:41:08.074Z 23 460-479 138 138 138 242 242 242 F F F
153372024-02-12T01:41:08.074Z 24 480-499 136 136 136 239 239 239 F F F
153382024-02-12T01:41:08.074Z 25 500-519 138 138 138 242 242 242 F F F
153392024-02-12T01:41:08.074Z 26 520-539 138 138 138 242 242 242 F F F
153402024-02-12T01:41:08.075Z 27 540-559 138 138 138 242 242 242 F F F
153412024-02-12T01:41:08.075Z 28 560-579 138 138 138 241 241 241 F F F
153422024-02-12T01:41:08.075Z 29 580-599 137 137 137 240 240 240 F F F
153432024-02-12T01:41:08.075ZMax gen: 139, Max flush: 243
153442024-02-12T01:41:08.075Z
153452024-02-12T01:41:08.075ZBegin loop 70
153462024-02-12T01:41:08.075ZDownstairs to restart: 2
153472024-02-12T01:41:08.091Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968153614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
153482024-02-12T01:41:08.096Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968475479Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
153492024-02-12T01:41:08.096Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name"Crucible runtime is spawned
153502024-02-12T01:41:08.096Z:"crucible","level":30,"time":"2024-02-12T01:41:06.968563785Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
153512024-02-12T01:41:08.096Z{"msg":"Crucible 15b88a79-7cd7-420a-adf2-078f2f2f737d has session id: 4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968611422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153522024-02-12T01:41:08.096Z{"msg":"Upstairs opts: Upstairs UUID: 15b88a79-7cd7-420a-adf2-078f2f2f737d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968644761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153532024-02-12T01:41:08.096Z{"msg":"Crucible stats registered with UUID: 15b88a79-7cd7-420a-adf2-078f2f2f737d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.96866971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153542024-02-12T01:41:08.096Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968683749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
153552024-02-12T01:41:08.096Z{"msg":"15b88a79-7cd7-420a-adf2-078f2f2f737d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968711918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153562024-02-12T01:41:08.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968733637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153572024-02-12T01:41:08.097Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968748686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153582024-02-12T01:41:08.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968762655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153592024-02-12T01:41:08.097Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968787954Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153602024-02-12T01:41:08.097Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968809733Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153612024-02-12T01:41:08.097Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968843931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153622024-02-12T01:41:08.097Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.96887939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153632024-02-12T01:41:08.097Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968922758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153642024-02-12T01:41:08.097Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.968950866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153652024-02-12T01:41:08.097Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:06.969130478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153662024-02-12T01:41:08.097Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:06.969208404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153672024-02-12T01:41:08.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969277901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153682024-02-12T01:41:08.097Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:06.969314929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153692024-02-12T01:41:08.097Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969345747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153702024-02-12T01:41:08.097Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969365586Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153712024-02-12T01:41:08.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969393045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153722024-02-12T01:41:08.097Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969407355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153732024-02-12T01:41:08.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.969945689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153742024-02-12T01:41:08.097Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970019955Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153752024-02-12T01:41:08.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.97012305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153762024-02-12T01:41:08.097Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970168098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153772024-02-12T01:41:08.097Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970465914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153782024-02-12T01:41:08.097Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970586138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153792024-02-12T01:41:08.098Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970860535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153802024-02-12T01:41:08.098Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970898273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153812024-02-12T01:41:08.098Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.970914723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153822024-02-12T01:41:08.098Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.971049646Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153832024-02-12T01:41:08.098Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.971086774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153842024-02-12T01:41:08.098Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:06.971103184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153852024-02-12T01:41:09.093Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.969770198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153862024-02-12T01:41:09.097Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.970034195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153872024-02-12T01:41:09.097Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.970961301Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153882024-02-12T01:41:09.097Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.970998719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153892024-02-12T01:41:09.097Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.971458207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153902024-02-12T01:41:09.098Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.971955604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153912024-02-12T01:41:09.098Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972016531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153922024-02-12T01:41:09.098Z{"msg":"[0]R flush_numbers[0..12]: [227, 238, 240, 233, 241, 240, 228, 240, 239, 242, 241, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972054339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153932024-02-12T01:41:09.098Z{"msg":"[0]R generation[0..12]: [130, 135, 137, 132, 138, 137, 130, 137, 136, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972091767Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153942024-02-12T01:41:09.098Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972128125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153952024-02-12T01:41:09.098Z{"msg":"[1]R flush_numbers[0..12]: [227, 238, 240, 233, 241, 240, 228, 240, 239, 242, 241, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972164063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153962024-02-12T01:41:09.098Z{"msg":"[1]R generation[0..12]: [130, 135, 137, 132, 138, 137, 130, 137, 136, 138, 138, 138]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
153972024-02-12T01:41:09.098Z,"time":"2024-02-12T01:41:07.972201082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153982024-02-12T01:41:09.098Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.97224435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
153992024-02-12T01:41:09.098Z{"msg":"[2]R flush_numbers[0..12]: [227, 238, 240, 233, 241, 240, 228, 240, 239, 242, 241, 242]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972284888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154002024-02-12T01:41:09.098Z{"msg":"[2]R generation[0..12]: [130, 135, 137, 132, 138, 137, 130, 137, 136, 138, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972321816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154012024-02-12T01:41:09.098Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972350035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154022024-02-12T01:41:09.098Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972385173Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154032024-02-12T01:41:09.098Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972426571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154042024-02-12T01:41:09.098Z{"msg":"Next flush: 244","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972463059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154052024-02-12T01:41:09.098Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972498678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154062024-02-12T01:41:09.098Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972535916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154072024-02-12T01:41:09.098Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972564844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154082024-02-12T01:41:09.098Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972580604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154092024-02-12T01:41:09.098Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972601123Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154102024-02-12T01:41:09.098Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972615222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154112024-02-12T01:41:09.098Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972634501Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154122024-02-12T01:41:09.098Z{"msg":"15b88a79-7cd7-420a-adf2-078f2f2f737d is now active with session: 4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972647871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154132024-02-12T01:41:09.098Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972667599Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695,"session_id":"4eaf63c1-4caa-4d96-8ce4-b6b7afe15e5e"}
154142024-02-12T01:41:09.099Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:07.972680429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
154152024-02-12T01:41:09.099ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154162024-02-12T01:41:09.102ZLoading write count information from file "/tmp/test_repair_verify.data"
154172024-02-12T01:41:09.102ZRun Repair workload
154182024-02-12T01:41:09.102Z01/30 Write block 198 len 8192 data: 7 11
154192024-02-12T01:41:09.102Z02/30 Write block 45 len 40960 data: 10 11 13 14 15 14 16 14 11 11
154202024-02-12T01:41:09.102Z03/30 Write block 229 len 32768 data: 8 6 7 8 8 10 10 13
154212024-02-12T01:41:09.102Z04/30 Write block 239 len 28672 data: 16 13 10 12 14 11 9
154222024-02-12T01:41:09.102Z05/30 Read block 232 len 16384
154232024-02-12T01:41:09.102Z06/30 Write block 288 len 16384 data: 11 10 10 11
154242024-02-12T01:41:09.105Z07/30 Read block 238 len 32768
154252024-02-12T01:41:09.105Z08/30 Write block 461 len 32768 data: 8 8 8 8 8 10 10 10
154262024-02-12T01:41:09.106Z09/30 Flush
154272024-02-12T01:41:09.108Z10/30 Flush
154282024-02-12T01:41:09.111Z11/30 Write block 565 len 24576 data: 11 11 12 14 15 14
154292024-02-12T01:41:09.111Z12/30 Read block 274 len 4096
154302024-02-12T01:41:09.112Z13/30 Write block 182 len 20480 data: 12 12 13 10 9
154312024-02-12T01:41:09.112Z14/30 Read block 389 len 16384
154322024-02-12T01:41:09.114Z15/30 Write block 480 len 12288 data: 15 14 14
154332024-02-12T01:41:09.115Z16/30 Write block 587 len 16384 data: 8 7 7 10
154342024-02-12T01:41:09.115Z17/30 Write block 571 len 28672 data: 12 14 17 16 15 11 11
154352024-02-12T01:41:09.115Z18/30 Write block 196 len 20480 data: 8 9 8 12 11
154362024-02-12T01:41:09.118Z19/30 Write block 588 len 36864 data: 8 8 11 8 9 6 6 8 8
154372024-02-12T01:41:09.118Z20/30 Read block 296 len 24576
154382024-02-12T01:41:09.118Z21/30 Read block 425 len 28672
154392024-02-12T01:41:09.120Z22/30 Read block 197 len 12288
154402024-02-12T01:41:09.121Z23/30 Read block 152 len 16384
154412024-02-12T01:41:09.121Z24/30 Write block 468 len 40960 data: 11 10 10 9 10 13 13 11 8 10
154422024-02-12T01:41:09.121Z25/30 Flush
154432024-02-12T01:41:09.124Z26/30 Write block 487 len 8192 data: 14 15
154442024-02-12T01:41:09.128Z27/30 Read block 11 len 24576
154452024-02-12T01:41:09.128Z28/30 Write block 274 len 36864 data: 15 10 9 11 13 14 13 11 14
154462024-02-12T01:41:09.128Z29/30 Read block 290 len 28672
154472024-02-12T01:41:09.132Z30/30 Write block 368 len 4096 data: 14
154482024-02-12T01:41:09.132Z----------------------------------------------------------------
154492024-02-12T01:41:09.132Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
154502024-02-12T01:41:09.132ZStates: Active Active Active
154512024-02-12T01:41:09.132ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
154522024-02-12T01:41:09.132Z 1 Acked 1000 Write 2 Done Done Sent false
154532024-02-12T01:41:09.132Z 2 Acked 1001 Write 10 Done Done Sent false
154542024-02-12T01:41:09.132Z 3 Acked 1002 Write 8 Done Done Sent false
154552024-02-12T01:41:09.132Z 4 Acked 1003 Write 7 Done Done Sent false
154562024-02-12T01:41:09.132Z 5 Acked 1004 Read 4 Done Done Sent false
154572024-02-12T01:41:09.132Z 6 Acked 1005 Write 4 Done Done Sent false
154582024-02-12T01:41:09.132Z 7 Acked 1006 Read 8 Done Done Sent false
154592024-02-12T01:41:09.133Z 8 Acked 1007 Write 8 Done Done Sent false
154602024-02-12T01:41:09.133Z 9 Acked 1008 Flush 0 Done Done Sent false
154612024-02-12T01:41:09.133Z 10 Acked 1009 Flush 0 Done Done Sent false
154622024-02-12T01:41:09.133Z 11 Acked 1010 Write 6 Done Done Sent false
154632024-02-12T01:41:09.133Z 12 Acked 1011 Read 1 Done Done Sent false
154642024-02-12T01:41:09.133Z 13 Acked 1012 Write 5 Done Done Sent false
154652024-02-12T01:41:09.133Z 14 Acked 1013 Read 4 Done Done Sent false
154662024-02-12T01:41:09.133Z 15 Acked 1014 Write 3 Done Done Sent false
154672024-02-12T01:41:09.133Z 16 Acked 1015 Write 4 Done Done Sent false
154682024-02-12T01:41:09.134Z 17 Acked 1016 Write 7 Done Done Sent false
154692024-02-12T01:41:09.134Z 18 Acked 1017 Write 5 Done Done Sent false
154702024-02-12T01:41:09.134Z 19 Acked 1018 Write 9 Done Done Sent false
154712024-02-12T01:41:09.134Z 20 Acked 1019 Read 6 Done Done Sent false
154722024-02-12T01:41:09.134Z 21 Acked 1020 Read 7 Done Done Sent false
154732024-02-12T01:41:09.134Z 22 Acked 1021 Read 3 Done Done Sent false
154742024-02-12T01:41:09.135Z 23 Acked 1022 Read 4 Done Done Sent false
154752024-02-12T01:41:09.135Z 24 Acked 1023 Write 10 Done Done Sent false
154762024-02-12T01:41:09.135Z 25 Acked 1024 Flush 0 Done Done Sent false
154772024-02-12T01:41:09.135Z 26 Acked 1025 Write 2 Done Done Sent false
154782024-02-12T01:41:09.135Z 27 Acked 1026 Read 6 Done Done Sent false
154792024-02-12T01:41:09.135Z 28 Acked 1027 Write 9 Done Done Sent false
154802024-02-12T01:41:09.135Z 29 Acked 1028 Read 7 Done Done Sent false
154812024-02-12T01:41:09.135Z 30 Acked 1029 Write 1 Sent Sent Sent false
154822024-02-12T01:41:09.135Z STATES DS:0 DS:1 DS:2 TOTAL
154832024-02-12T01:41:09.135Z New 0 0 0 0
154842024-02-12T01:41:09.135Z Sent 1 1 30 32
154852024-02-12T01:41:09.135Z Done 29 29 0 58
154862024-02-12T01:41:09.135Z Skipped 0 0 0 0
154872024-02-12T01:41:09.136Z Error 0 0 0 0
154882024-02-12T01:41:09.136ZLast Flush: 1024 1024 0
154892024-02-12T01:41:09.136ZDownstairs last five completed:
154902024-02-12T01:41:09.136ZUpstairs last five completed: 30 29 28 27 26
154912024-02-12T01:41:09.136Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:08.006159162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1695}
154922024-02-12T01:41:09.136ZWrote out file "/tmp/test_repair_verify.data"
154932024-02-12T01:41:09.140Z
154942024-02-12T01:41:09.140ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154952024-02-12T01:41:09.154ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154962024-02-12T01:41:09.154Z 0 000-019 130 130 130 227 227 227 F F F
154972024-02-12T01:41:09.154Z 1 020-039 135 135 135 238 238 238 F F F
154982024-02-12T01:41:09.154Z 2 040-059 140 140 137 244 244 240 F F F <---
154992024-02-12T01:41:09.154Z 3 060-079 132 132 132 233 233 233 F F F
155002024-02-12T01:41:09.154Z 4 080-099 138 138 138 241 241 241 F F F
155012024-02-12T01:41:09.154Z 5 100-119 137 137 137 240 240 240 F F F
155022024-02-12T01:41:09.154Z 6 120-139 130 130 130 228 228 228 F F F
155032024-02-12T01:41:09.154Z 7 140-159 137 137 137 240 240 240 F F F
155042024-02-12T01:41:09.154Z 8 160-179 136 136 136 239 239 239 F F F
155052024-02-12T01:41:09.154Z 9 180-199 140 140 138 246 246 242 F F F <---
155062024-02-12T01:41:09.154Z 10 200-219 140 140 138 246 246 241 F F F <---
155072024-02-12T01:41:09.154Z 11 220-239 140 140 138 244 244 242 F F F <---
155082024-02-12T01:41:09.154Z 12 240-259 140 140 134 244 244 235 F F F <---
155092024-02-12T01:41:09.154Z 13 260-279 139 139 139 243 243 243 T T F <---
155102024-02-12T01:41:09.158Z 14 280-299 140 140 137 244 244 240 T T F <---
155112024-02-12T01:41:09.158Z 15 300-319 138 138 138 242 242 242 F F F
155122024-02-12T01:41:09.158Z 16 320-339 139 139 139 243 243 243 F F F
155132024-02-12T01:41:09.158Z 17 340-359 136 136 136 239 239 239 F F F
155142024-02-12T01:41:09.158Z 18 360-379 137 137 137 240 240 240 T T F <---
155152024-02-12T01:41:09.158Z 19 380-399 139 139 139 243 243 243 F F F
155162024-02-12T01:41:09.158Z 20 400-419 139 139 139 243 243 243 F F F
155172024-02-12T01:41:09.158Z 21 420-439 137 137 137 240 240 240 F F F
155182024-02-12T01:41:09.158Z 22 440-459 137 137 137 240 240 240 F F F
155192024-02-12T01:41:09.158Z 23 460-479 140 140 138 246 246 242 F F F <---
155202024-02-12T01:41:09.158Z 24 480-499 140 140 136 246 246 239 T T F <---
155212024-02-12T01:41:09.159Z 25 500-519 138 138 138 242 242 242 F F F
155222024-02-12T01:41:09.159Z 26 520-539 138 138 138 242 242 242 F F F
155232024-02-12T01:41:09.159Z 27 540-559 138 138 138 242 242 242 F F F
155242024-02-12T01:41:09.159Z 28 560-579 140 140 138 246 246 241 F F F <---
155252024-02-12T01:41:09.159Z 29 580-599 140 140 137 246 246 240 F F F <---
155262024-02-12T01:41:09.159ZMax gen: 140, Max flush: 246
155272024-02-12T01:41:09.159ZError: Difference in extent metadata found!
155282024-02-12T01:41:09.159ZA Difference in extent metadata is expected here
155292024-02-12T01:41:09.159ZOn loop 70
155302024-02-12T01:41:09.159Z
155312024-02-12T01:41:09.162ZVerifying data now
155322024-02-12T01:41:13.755ZNo change after verify
155332024-02-12T01:41:13.755ZLoop: 70 Downstairs dump after verify (and repair):
155342024-02-12T01:41:13.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155352024-02-12T01:41:13.770Z 0 000-019 130 130 130 227 227 227 F F F
155362024-02-12T01:41:13.770Z 1 020-039 135 135 135 238 238 238 F F F
155372024-02-12T01:41:13.770Z 2 040-059 140 140 140 244 244 244 F F F
155382024-02-12T01:41:13.770Z 3 060-079 132 132 132 233 233 233 F F F
155392024-02-12T01:41:13.770Z 4 080-099 138 138 138 241 241 241 F F F
155402024-02-12T01:41:13.770Z 5 100-119 137 137 137 240 240 240 F F F
155412024-02-12T01:41:13.770Z 6 120-139 130 130 130 228 228 228 F F F
155422024-02-12T01:41:13.770Z 7 140-159 137 137 137 240 240 240 F F F
155432024-02-12T01:41:13.770Z 8 160-179 136 136 136 239 239 239 F F F
155442024-02-12T01:41:13.770Z 9 180-199 140 140 140 246 246 246 F F F
155452024-02-12T01:41:13.770Z 10 200-219 140 140 140 246 246 246 F F F
155462024-02-12T01:41:13.770Z 11 220-239 140 140 140 244 244 244 F F F
155472024-02-12T01:41:13.770Z 12 240-259 140 140 140 244 244 244 F F F
155482024-02-12T01:41:13.774Z 13 260-279 141 141 141 247 247 247 F F F
155492024-02-12T01:41:13.774Z 14 280-299 141 141 141 247 247 247 F F F
155502024-02-12T01:41:13.774Z 15 300-319 138 138 138 242 242 242 F F F
155512024-02-12T01:41:13.774Z 16 320-339 139 139 139 243 243 243 F F F
155522024-02-12T01:41:13.774Z 17 340-359 136 136 136 239 239 239 F F F
155532024-02-12T01:41:13.774Z 18 360-379 141 141 141 247 247 247 F F F
155542024-02-12T01:41:13.774Z 19 380-399 139 139 139 243 243 243 F F F
155552024-02-12T01:41:13.774Z 20 400-419 139 139 139 243 243 243 F F F
155562024-02-12T01:41:13.774Z 21 420-439 137 137 137 240 240 240 F F F
155572024-02-12T01:41:13.774Z 22 440-459 137 137 137 240 240 240 F F F
155582024-02-12T01:41:13.774Z 23 460-479 140 140 140 246 246 246 F F F
155592024-02-12T01:41:13.774Z 24 480-499 141 141 141 247 247 247 F F F
155602024-02-12T01:41:13.774Z 25 500-519 138 138 138 242 242 242 F F F
155612024-02-12T01:41:13.774Z 26 520-539 138 138 138 242 242 242 F F F
155622024-02-12T01:41:13.774Z 27 540-559 138 138 138 242 242 242 F F F
155632024-02-12T01:41:13.774Z 28 560-579 140 140 140 246 246 246 F F F
155642024-02-12T01:41:13.774Z 29 580-599 140 140 140 246 246 246 F F F
155652024-02-12T01:41:13.774ZMax gen: 141, Max flush: 247
155662024-02-12T01:41:13.774Z
155672024-02-12T01:41:13.775ZBegin loop 71
155682024-02-12T01:41:13.775ZDownstairs to restart: 1
155692024-02-12T01:41:13.792Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668159471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
155702024-02-12T01:41:13.792Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.66839625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
155712024-02-12T01:41:13.792Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668413069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
155722024-02-12T01:41:13.792Z{"msg":"Crucible 4b3462fb-5494-4aff-b673-49444fae5732 has session id: f306fce9-5737-4eb7-9424-64cdf459dafc","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668428168Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155732024-02-12T01:41:13.792Z{"msg":"Upstairs opts: Upstairs UUID: 4b3462fb-5494-4aff-b673-49444fae5732, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668461757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155742024-02-12T01:41:13.796Z{"msg":"Crucible stats registered with UUID: 4b3462fb-5494-4aff-b673-49444fae5732","v":0,"name":"crucible","level":30Crucible runtime is spawned
155752024-02-12T01:41:13.797Z,"time":"2024-02-12T01:41:12.668486675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155762024-02-12T01:41:13.797Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.66859221Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
155772024-02-12T01:41:13.797Z{"msg":"4b3462fb-5494-4aff-b673-49444fae5732 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668624269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155782024-02-12T01:41:13.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668638838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155792024-02-12T01:41:13.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668653928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155802024-02-12T01:41:13.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668678556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155812024-02-12T01:41:13.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668695575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155822024-02-12T01:41:13.797Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668714365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155832024-02-12T01:41:13.797Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668743613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155842024-02-12T01:41:13.797Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668768472Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155852024-02-12T01:41:13.797Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.66880986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155862024-02-12T01:41:13.797Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.668831959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155872024-02-12T01:41:13.797Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:12.668963103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155882024-02-12T01:41:13.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669003291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155892024-02-12T01:41:13.797Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669031969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155902024-02-12T01:41:13.797Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:12.669124465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155912024-02-12T01:41:13.797Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:12.669171863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155922024-02-12T01:41:13.797Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669189082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155932024-02-12T01:41:13.797Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669202851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155942024-02-12T01:41:13.797Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669274748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155952024-02-12T01:41:13.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669689588Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155962024-02-12T01:41:13.797Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669725686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155972024-02-12T01:41:13.797Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669742756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155982024-02-12T01:41:13.798Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.669766114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
155992024-02-12T01:41:13.798Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670291919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156002024-02-12T01:41:13.798Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670438012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156012024-02-12T01:41:13.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.67069475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156022024-02-12T01:41:13.798Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670739748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156032024-02-12T01:41:13.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670765747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156042024-02-12T01:41:13.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670920259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156052024-02-12T01:41:13.798Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670954338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156062024-02-12T01:41:13.798Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:12.670969267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156072024-02-12T01:41:14.793Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.669829592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156082024-02-12T01:41:14.798Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.670124388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156092024-02-12T01:41:14.798Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.670839744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156102024-02-12T01:41:14.798Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.670876592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156112024-02-12T01:41:14.798Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.67134267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156122024-02-12T01:41:14.798Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.671914483Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156132024-02-12T01:41:14.798Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.671948801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156142024-02-12T01:41:14.798Z{"msg":"[0]R flush_numbers[0..12]: [227, 238, 244, 233, 241, 240, 228, 240, 239, 246, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.67196449Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156152024-02-12T01:41:14.798Z{"msg":"[0]R generation[0..12]: [130, 135, 140, 132, 138, 137, 130, 137, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.67197898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156162024-02-12T01:41:14.798Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672039967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156172024-02-12T01:41:14.799Z{"msg":"[1]R flush_numbers[0..12]: [227, 238, 244, 233, 241, 240, 228, 240, 239, 246, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672056776Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156182024-02-12T01:41:14.799Z{"msg":"[1]R generation[0..12]: [130, 135, 140, 132, 138, 137, 130, 137, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672071115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156192024-02-12T01:41:14.799Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672094364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156202024-02-12T01:41:14.799Z{"msg":"[2]R flush_numbers[0..12]: [227, 238, 244, 233, 241, 240, 228, 240, 239, 246, 246, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672108934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156212024-02-12T01:41:14.799Z{"msg":"[2]R generation[0..12]: [130, 135, 140, 132, 138, 137, 130, 137, 136, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672129702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":Wait for a query_work_queue command to finish before sending IO
156222024-02-12T01:41:14.799Z1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156232024-02-12T01:41:14.799Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672148222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156242024-02-12T01:41:14.799Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672168561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156252024-02-12T01:41:14.799Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.67218262Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156262024-02-12T01:41:14.799Z{"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672202259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156272024-02-12T01:41:14.799Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672215789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156282024-02-12T01:41:14.799Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672229898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156292024-02-12T01:41:14.799Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672242527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156302024-02-12T01:41:14.799Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672263066Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156312024-02-12T01:41:14.799Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672276535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156322024-02-12T01:41:14.799Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672290295Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156332024-02-12T01:41:14.799Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672309114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156342024-02-12T01:41:14.799Z{"msg":"4b3462fb-5494-4aff-b673-49444fae5732 is now active with session: f306fce9-5737-4eb7-9424-64cdf459dafc","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672322303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156352024-02-12T01:41:14.799Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672341203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703,"session_id":"f306fce9-5737-4eb7-9424-64cdf459dafc"}
156362024-02-12T01:41:14.799Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:13.672353842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
156372024-02-12T01:41:14.799ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
156382024-02-12T01:41:14.799ZLoading write count information from file "/tmp/test_repair_verify.data"
156392024-02-12T01:41:14.799ZRun Repair workload
156402024-02-12T01:41:14.803Z01/30 Write block 194 len 40960 data: 8 8 9 10 9 13 12 9 12 13
156412024-02-12T01:41:14.803Z02/30 Write block 55 len 36864 data: 11 11 13 16 12 9 8 11 10
156422024-02-12T01:41:14.803Z03/30 Read block 571 len 32768
156432024-02-12T01:41:14.803Z04/30 Read block 377 len 12288
156442024-02-12T01:41:14.803Z05/30 Write block 351 len 20480 data: 10 13 11 7 8
156452024-02-12T01:41:14.803Z06/30 Write block 120 len 12288 data: 14 13 12
156462024-02-12T01:41:14.806Z07/30 Read block 348 len 28672
156472024-02-12T01:41:14.806Z08/30 Read block 54 len 24576
156482024-02-12T01:41:14.807Z09/30 Read block 62 len 40960
156492024-02-12T01:41:14.810Z10/30 Read block 168 len 12288
156502024-02-12T01:41:14.810Z11/30 Flush
156512024-02-12T01:41:14.813Z12/30 Write block 50 len 12288 data: 15 17 15
156522024-02-12T01:41:14.813Z13/30 Read block 44 len 32768
156532024-02-12T01:41:14.813Z14/30 Read block 293 len 20480
156542024-02-12T01:41:14.813Z15/30 Read block 235 len 4096
156552024-02-12T01:41:14.817Z16/30 Write block 458 len 24576 data: 12 9 8 9 9 9
156562024-02-12T01:41:14.817Z17/30 Write block 224 len 24576 data: 11 10 7 8 9 9
156572024-02-12T01:41:14.817Z18/30 Read block 298 len 8192
156582024-02-12T01:41:14.817Z19/30 Write block 25 len 20480 data: 10 11 12 13 16
156592024-02-12T01:41:14.817Z20/30 Read block 520 len 8192
156602024-02-12T01:41:14.820Z21/30 Read block 420 len 32768
156612024-02-12T01:41:14.820Z22/30 Write block 372 len 36864 data: 15 15 15 17 14 13 12 13 11
156622024-02-12T01:41:14.820Z23/30 Read block 147 len 4096
156632024-02-12T01:41:14.820Z24/30 Read block 109 len 24576
156642024-02-12T01:41:14.823Z25/30 Flush
156652024-02-12T01:41:14.826Z26/30 Write block 521 len 16384 data: 10 11 11 10
156662024-02-12T01:41:14.826Z27/30 Read block 36 len 24576
156672024-02-12T01:41:14.826Z28/30 Write block 431 len 36864 data: 5 5 5 6 6 7 8 8 11
156682024-02-12T01:41:14.830Z29/30 Write block 432 len 12288 data: 6 6 7
156692024-02-12T01:41:14.830Z30/30 Read block 30 len 24576
156702024-02-12T01:41:14.830Z----------------------------------------------------------------
156712024-02-12T01:41:14.830Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
156722024-02-12T01:41:14.830ZStates: Active Active Active
156732024-02-12T01:41:14.830ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156742024-02-12T01:41:14.830Z 1 Acked 1000 Write 10 Done Done Done false
156752024-02-12T01:41:14.830Z 2 Acked 1001 Write 9 Done Done Done false
156762024-02-12T01:41:14.830Z 3 Acked 1002 Read 8 Done Done Done false
156772024-02-12T01:41:14.830Z 4 Acked 1003 Read 3 Done Done Done false
156782024-02-12T01:41:14.830Z 5 Acked 1004 Write 5 Done Done Done false
156792024-02-12T01:41:14.830Z 6 Acked 1005 Write 3 Done Done Done false
156802024-02-12T01:41:14.830Z 7 Acked 1006 Read 7 Done Sent Done false
156812024-02-12T01:41:14.830Z 8 Acked 1007 Read 6 Done Sent Done false
156822024-02-12T01:41:14.830Z 9 Acked 1008 Read 10 Done Sent Done false
156832024-02-12T01:41:14.830Z 10 Acked 1009 Read 3 Done Sent Done false
156842024-02-12T01:41:14.830Z 11 Acked 1010 Flush 0 Done Sent Done false
156852024-02-12T01:41:14.830Z 12 Acked 1011 Write 3 Done Sent Done false
156862024-02-12T01:41:14.831Z 13 Acked 1012 Read 8 Done Sent Done false
156872024-02-12T01:41:14.831Z 14 Acked 1013 Read 5 Done Sent Done false
156882024-02-12T01:41:14.831Z 15 Acked 1014 Read 1 Done Sent Done false
156892024-02-12T01:41:14.831Z 16 Acked 1015 Write 6 Done Sent Done false
156902024-02-12T01:41:14.831Z 17 Acked 1016 Write 6 Done Sent Done false
156912024-02-12T01:41:14.831Z 18 Acked 1017 Read 2 Done Sent Done false
156922024-02-12T01:41:14.831Z 19 Acked 1018 Write 5 Done Sent Done false
156932024-02-12T01:41:14.831Z 20 Acked 1019 Read 2 Done Sent Done false
156942024-02-12T01:41:14.831Z 21 Acked 1020 Read 8 Done Sent Done false
156952024-02-12T01:41:14.831Z 22 Acked 1021 Write 9 Done Sent Done false
156962024-02-12T01:41:14.831Z 23 Acked 1022 Read 1 Done Sent Done false
156972024-02-12T01:41:14.831Z 24 Acked 1023 Read 6 Done Sent Done false
156982024-02-12T01:41:14.831Z 25 Acked 1024 Flush 0 Done Sent Done false
156992024-02-12T01:41:14.831Z 26 Acked 1025 Write 4 Done Sent Done false
157002024-02-12T01:41:14.831Z 27 Acked 1026 Read 6 Done Sent Done false
157012024-02-12T01:41:14.831Z 28 Acked 1027 Write 9 Done Sent Done false
157022024-02-12T01:41:14.831Z 29 Acked 1028 Write 3 Done Sent Done false
157032024-02-12T01:41:14.831Z 30 Acked 1029 Read 6 Sent Sent Done false
157042024-02-12T01:41:14.831Z STATES DS:0 DS:1 DS:2 TOTAL
157052024-02-12T01:41:14.831Z New 0 0 0 0
157062024-02-12T01:41:14.831Z Sent 1 24 0 25
157072024-02-12T01:41:14.831Z Done 29 6 30 65
157082024-02-12T01:41:14.831Z Skipped 0 0 0 0
157092024-02-12T01:41:14.831Z Error 0 0 0 0
157102024-02-12T01:41:14.831ZLast Flush: 1024 0 1024
157112024-02-12T01:41:14.832ZDownstairs last five completed:
157122024-02-12T01:41:14.832ZUpstairs last five completed: 30 29 28 27 26
157132024-02-12T01:41:14.832Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:13.705453953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1703}
157142024-02-12T01:41:14.832ZWrote out file "/tmp/test_repair_verify.data"
157152024-02-12T01:41:14.835Z
157162024-02-12T01:41:14.841ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
157172024-02-12T01:41:14.854ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157182024-02-12T01:41:14.854Z 0 000-019 130 130 130 227 227 227 F F F
157192024-02-12T01:41:14.854Z 1 020-039 142 135 142 249 238 249 F F F <---
157202024-02-12T01:41:14.854Z 2 040-059 142 140 142 249 244 249 F T F <---
157212024-02-12T01:41:14.857Z 3 060-079 142 132 142 248 233 248 F T F <---
157222024-02-12T01:41:14.857Z 4 080-099 138 138 138 241 241 241 F F F
157232024-02-12T01:41:14.858Z 5 100-119 137 137 137 240 240 240 F F F
157242024-02-12T01:41:14.858Z 6 120-139 142 130 142 248 228 248 F T F <---
157252024-02-12T01:41:14.858Z 7 140-159 137 137 137 240 240 240 F F F
157262024-02-12T01:41:14.858Z 8 160-179 136 136 136 239 239 239 F F F
157272024-02-12T01:41:14.858Z 9 180-199 142 140 142 248 246 248 F T F <---
157282024-02-12T01:41:14.858Z 10 200-219 142 140 142 248 246 248 F T F <---
157292024-02-12T01:41:14.858Z 11 220-239 142 140 142 249 244 249 F F F <---
157302024-02-12T01:41:14.858Z 12 240-259 140 140 140 244 244 244 F F F
157312024-02-12T01:41:14.858Z 13 260-279 141 141 141 247 247 247 F F F
157322024-02-12T01:41:14.858Z 14 280-299 141 141 141 247 247 247 F F F
157332024-02-12T01:41:14.858Z 15 300-319 138 138 138 242 242 242 F F F
157342024-02-12T01:41:14.858Z 16 320-339 139 139 139 243 243 243 F F F
157352024-02-12T01:41:14.858Z 17 340-359 142 136 142 248 239 248 F T F <---
157362024-02-12T01:41:14.858Z 18 360-379 142 141 142 249 247 249 F F F <---
157372024-02-12T01:41:14.858Z 19 380-399 142 139 142 249 243 249 F F F <---
157382024-02-12T01:41:14.858Z 20 400-419 139 139 139 243 243 243 F F F
157392024-02-12T01:41:14.858Z 21 420-439 137 137 137 240 240 240 T F T <---
157402024-02-12T01:41:14.858Z 22 440-459 142 137 142 249 240 249 F F F <---
157412024-02-12T01:41:14.858Z 23 460-479 142 140 142 249 246 249 F F F <---
157422024-02-12T01:41:14.858Z 24 480-499 141 141 141 247 247 247 F F F
157432024-02-12T01:41:14.858Z 25 500-519 138 138 138 242 242 242 F F F
157442024-02-12T01:41:14.858Z 26 520-539 138 138 138 242 242 242 T F T <---
157452024-02-12T01:41:14.858Z 27 540-559 138 138 138 242 242 242 F F F
157462024-02-12T01:41:14.859Z 28 560-579 140 140 140 246 246 246 F F F
157472024-02-12T01:41:14.859Z 29 580-599 140 140 140 246 246 246 F F F
157482024-02-12T01:41:14.859ZMax gen: 142, Max flush: 249
157492024-02-12T01:41:14.859ZError: Difference in extent metadata found!
157502024-02-12T01:41:14.859ZA Difference in extent metadata is expected here
157512024-02-12T01:41:14.859ZOn loop 71
157522024-02-12T01:41:14.859Z
157532024-02-12T01:41:14.862ZVerifying data now
157542024-02-12T01:41:20.540ZNo change after verify
157552024-02-12T01:41:20.541ZLoop: 71 Downstairs dump after verify (and repair):
157562024-02-12T01:41:20.556ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157572024-02-12T01:41:20.556Z 0 000-019 130 130 130 227 227 227 F F F
157582024-02-12T01:41:20.556Z 1 020-039 142 142 142 249 249 249 F F F
157592024-02-12T01:41:20.556Z 2 040-059 142 142 142 249 249 249 F F F
157602024-02-12T01:41:20.556Z 3 060-079 142 142 142 248 248 248 F F F
157612024-02-12T01:41:20.556Z 4 080-099 138 138 138 241 241 241 F F F
157622024-02-12T01:41:20.556Z 5 100-119 137 137 137 240 240 240 F F F
157632024-02-12T01:41:20.556Z 6 120-139 142 142 142 248 248 248 F F F
157642024-02-12T01:41:20.556Z 7 140-159 137 137 137 240 240 240 F F F
157652024-02-12T01:41:20.556Z 8 160-179 136 136 136 239 239 239 F F F
157662024-02-12T01:41:20.556Z 9 180-199 142 142 142 248 248 248 F F F
157672024-02-12T01:41:20.556Z 10 200-219 142 142 142 248 248 248 F F F
157682024-02-12T01:41:20.556Z 11 220-239 142 142 142 249 249 249 F F F
157692024-02-12T01:41:20.562Z 12 240-259 140 140 140 244 244 244 F F F
157702024-02-12T01:41:20.562Z 13 260-279 141 141 141 247 247 247 F F F
157712024-02-12T01:41:20.563Z 14 280-299 141 141 141 247 247 247 F F F
157722024-02-12T01:41:20.563Z 15 300-319 138 138 138 242 242 242 F F F
157732024-02-12T01:41:20.563Z 16 320-339 139 139 139 243 243 243 F F F
157742024-02-12T01:41:20.563Z 17 340-359 142 142 142 248 248 248 F F F
157752024-02-12T01:41:20.563Z 18 360-379 142 142 142 249 249 249 F F F
157762024-02-12T01:41:20.563Z 19 380-399 142 142 142 249 249 249 F F F
157772024-02-12T01:41:20.563Z 20 400-419 139 139 139 243 243 243 F F F
157782024-02-12T01:41:20.563Z 21 420-439 143 143 143 250 250 250 F F F
157792024-02-12T01:41:20.563Z 22 440-459 142 142 142 249 249 249 F F F
157802024-02-12T01:41:20.563Z 23 460-479 142 142 142 249 249 249 F F F
157812024-02-12T01:41:20.563Z 24 480-499 141 141 141 247 247 247 F F F
157822024-02-12T01:41:20.563Z 25 500-519 138 138 138 242 242 242 F F F
157832024-02-12T01:41:20.563Z 26 520-539 143 143 143 250 250 250 F F F
157842024-02-12T01:41:20.563Z 27 540-559 138 138 138 242 242 242 F F F
157852024-02-12T01:41:20.563Z 28 560-579 140 140 140 246 246 246 F F F
157862024-02-12T01:41:20.563Z 29 580-599 140 140 140 246 246 246 F F F
157872024-02-12T01:41:20.563ZMax gen: 143, Max flush: 250
157882024-02-12T01:41:20.563Z
157892024-02-12T01:41:20.563ZBegin loop 72
157902024-02-12T01:41:20.563ZDownstairs to restart: 2
157912024-02-12T01:41:20.579Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45410792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
157922024-02-12T01:41:20.579Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45433281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
157932024-02-12T01:41:20.579Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454350269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
157942024-02-12T01:41:20.584Z{"msg":"Crucible 1a4305a9-1b3e-4a3c-a752-406117aba93d has session id: d6c28670-f2b7-4f6f-b693-bbe5cf01a682","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454387587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
157952024-02-12T01:41:20.584Z{"msg":"Upstairs opts: Upstairs UUID: 1a4305a9-1b3e-4a3c-a752-406117aba93d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454433275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
157962024-02-12T01:41:20.584Z{"msg":"Crucible stats registered with UUID: 1a4305a9-1b3e-4a3c-a752-406117aba93d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454466873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
157972024-02-12T01:41:20.584ZCrucible runtime is spawned
157982024-02-12T01:41:20.584Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454603977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
157992024-02-12T01:41:20.584Z{"msg":"1a4305a9-1b3e-4a3c-a752-406117aba93d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454701992Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158002024-02-12T01:41:20.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45473442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158012024-02-12T01:41:20.584Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45475109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158022024-02-12T01:41:20.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454787628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158032024-02-12T01:41:20.584Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454814527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158042024-02-12T01:41:20.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454850775Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158052024-02-12T01:41:20.584Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454878833Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158062024-02-12T01:41:20.584Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454913822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158072024-02-12T01:41:20.584Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45494774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158082024-02-12T01:41:20.584Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.454994068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158092024-02-12T01:41:20.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455053535Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158102024-02-12T01:41:20.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455095203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158112024-02-12T01:41:20.584Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455186709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158122024-02-12T01:41:20.584Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455865766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158132024-02-12T01:41:20.584Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455904655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158142024-02-12T01:41:20.584Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455932093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158152024-02-12T01:41:20.584Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.455969281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158162024-02-12T01:41:20.584Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.456495166Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158172024-02-12T01:41:20.585Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.456553564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158182024-02-12T01:41:20.585Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457082718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158192024-02-12T01:41:20.585Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457117207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158202024-02-12T01:41:20.585Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457143336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158212024-02-12T01:41:20.585Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457178114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158222024-02-12T01:41:20.585Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457227432Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158232024-02-12T01:41:20.585Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457267119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158242024-02-12T01:41:20.585Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457307558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158252024-02-12T01:41:20.585Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.457338136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158262024-02-12T01:41:20.585Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45767633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158272024-02-12T01:41:20.589Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45810901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158282024-02-12T01:41:20.589Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458145398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158292024-02-12T01:41:20.589Z{"msg":"[0]R flush_numbers[0..12]: [227, 249, 249, 248, 241, 240, 248, 240, 239, 248, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458161407Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158302024-02-12T01:41:20.589Z{"msg":"[0]R generation[0..12]: [130, 142, 142, 142, 138, 137, 142, 137, 136, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458195065Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158312024-02-12T01:41:20.589Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458229234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158322024-02-12T01:41:20.589Z{"msg":"[1]R flush_numbers[0..12]: [227, 249, 249, 248, 241, 240, 248, 240, 239, 248, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458264122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
158332024-02-12T01:41:20.589Zd6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158342024-02-12T01:41:20.589Z{"msg":"[1]R generation[0..12]: [130, 142, 142, 142, 138, 137, 142, 137, 136, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.4582988Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158352024-02-12T01:41:20.590Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458332359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158362024-02-12T01:41:20.590Z{"msg":"[2]R flush_numbers[0..12]: [227, 249, 249, 248, 241, 240, 248, 240, 239, 248, 248, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458365517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158372024-02-12T01:41:20.590Z{"msg":"[2]R generation[0..12]: [130, 142, 142, 142, 138, 137, 142, 137, 136, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458398176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158382024-02-12T01:41:20.590Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458431844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158392024-02-12T01:41:20.590Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458464853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158402024-02-12T01:41:20.590Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458496481Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158412024-02-12T01:41:20.590Z{"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458528399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158422024-02-12T01:41:20.590Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458559018Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158432024-02-12T01:41:20.590Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458591786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158442024-02-12T01:41:20.590Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458620555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158452024-02-12T01:41:20.590Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458653273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158462024-02-12T01:41:20.590Z{"msg":"Transition from WaitQuorum to Active","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158472024-02-12T01:41:20.590Zv":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458687612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158482024-02-12T01:41:20.590Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.45871971Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158492024-02-12T01:41:20.590Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458739379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158502024-02-12T01:41:20.590Z{"msg":"1a4305a9-1b3e-4a3c-a752-406117aba93d is now active with session: d6c28670-f2b7-4f6f-b693-bbe5cf01a682","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458766858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158512024-02-12T01:41:20.590Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458783077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711,"session_id":"d6c28670-f2b7-4f6f-b693-bbe5cf01a682"}
158522024-02-12T01:41:20.590Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:19.458809736Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
158532024-02-12T01:41:20.590ZLoading write count information from file "/tmp/test_repair_verify.data"
158542024-02-12T01:41:20.590ZRun Repair workload
158552024-02-12T01:41:20.590Z01/30 Write block 374 len 40960 data: 16 18 15 14 13 14 12 11 9 9
158562024-02-12T01:41:20.590Z02/30 Read block 94 len 36864
158572024-02-12T01:41:20.591Z03/30 Write block 410 len 24576 data: 14 13 13 15 15 16
158582024-02-12T01:41:20.591Z04/30 Write block 290 len 16384 data: 11 12 9 10
158592024-02-12T01:41:20.591Z05/30 Write block 349 len 36864 data: 7 9 11 14 12 8 9 7 6
158602024-02-12T01:41:20.594Z06/30 Read block 301 len 16384
158612024-02-12T01:41:20.594Z07/30 Write block 6 len 40960 data: 8 7 8 9 8 9 10 11 11 11
158622024-02-12T01:41:20.594Z08/30 Read block 222 len 12288
158632024-02-12T01:41:20.594Z09/30 Write block 412 len 32768 data: 14 16 16 17 14 13 10 9
158642024-02-12T01:41:20.594Z10/30 Read block 479 len 40960
158652024-02-12T01:41:20.597Z11/30 Read block 391 len 32768
158662024-02-12T01:41:20.597Z12/30 Write block 94 len 36864 data: 5 5 7 8 9 7 6 9 8
158672024-02-12T01:41:20.597Z13/30 Write block 201 len 4096 data: 10
158682024-02-12T01:41:20.598Z14/30 Read block 110 len 20480
158692024-02-12T01:41:20.601Z15/30 Flush
158702024-02-12T01:41:20.603Z16/30 Write block 118 len 12288 data: 16 14 15
158712024-02-12T01:41:20.603Z17/30 Read block 522 len 20480
158722024-02-12T01:41:20.607Z18/30 Write block 344 len 36864 data: 6 7 8 8 7 8 10 12 15
158732024-02-12T01:41:20.607Z19/30 Flush
158742024-02-12T01:41:20.610Z20/30 Write block 115 len 8192 data: 12 14
158752024-02-12T01:41:20.610Z21/30 Write block 46 len 4096 data: 12
158762024-02-12T01:41:20.610Z22/30 Write block 290 len 20480 data: 12 13 10 11 8
158772024-02-12T01:41:20.610Z23/30 Read block 397 len 40960
158782024-02-12T01:41:20.613Z24/30 Read block 98 len 40960
158792024-02-12T01:41:20.613Z25/30 Read block 277 len 16384
158802024-02-12T01:41:20.613Z26/30 Write block 268 len 32768 data: 13 12 13 14 15 16 16 11
158812024-02-12T01:41:20.616Z27/30 Read block 483 len 8192
158822024-02-12T01:41:20.616Z28/30 Read block 241 len 12288
158832024-02-12T01:41:20.616Z29/30 Read block 449 len 40960
158842024-02-12T01:41:20.616Z30/30 Write block 161 len 8192 data: 11 13
158852024-02-12T01:41:20.619Z----------------------------------------------------------------
158862024-02-12T01:41:20.619Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
158872024-02-12T01:41:20.620ZStates: Active Active Active
158882024-02-12T01:41:20.620ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
158892024-02-12T01:41:20.620Z 1 Acked 1000 Write 10 Done Done Done false
158902024-02-12T01:41:20.620Z 2 Acked 1001 Read 9 Done Done Done false
158912024-02-12T01:41:20.620Z 3 Acked 1002 Write 6 Done Done Done false
158922024-02-12T01:41:20.620Z 4 Acked 1003 Write 4 Done Done Done false
158932024-02-12T01:41:20.620Z 5 Acked 1004 Write 9 Done Done Done false
158942024-02-12T01:41:20.620Z 6 Acked 1005 Read 4 Done Done Done false
158952024-02-12T01:41:20.620Z 7 Acked 1006 Write 10 Done Done Done false
158962024-02-12T01:41:20.620Z 8 Acked 1007 Read 3 Done Done Done false
158972024-02-12T01:41:20.620Z 9 Acked 1008 Write 8 Done Done Done false
158982024-02-12T01:41:20.620Z 10 Acked 1009 Read 10 Done Done Done false
158992024-02-12T01:41:20.620Z 11 Acked 1010 Read 8 Done Done Sent false
159002024-02-12T01:41:20.620Z 12 Acked 1011 Write 9 Done Done Sent false
159012024-02-12T01:41:20.620Z 13 Acked 1012 Write 1 Done Done Sent false
159022024-02-12T01:41:20.620Z 14 Acked 1013 Read 5 Done Done Sent false
159032024-02-12T01:41:20.620Z 15 Acked 1014 Flush 0 Done Done Sent false
159042024-02-12T01:41:20.620Z 16 Acked 1015 Write 3 Done Done Sent false
159052024-02-12T01:41:20.620Z 17 Acked 1016 Read 5 Done Done Sent false
159062024-02-12T01:41:20.620Z 18 Acked 1017 Write 9 Done Done Sent false
159072024-02-12T01:41:20.620Z 19 Acked 1018 Flush 0 Done Done Sent false
159082024-02-12T01:41:20.620Z 20 Acked 1019 Write 2 Done Done Sent false
159092024-02-12T01:41:20.620Z 21 Acked 1020 Write 1 Done Done Sent false
159102024-02-12T01:41:20.620Z 22 Acked 1021 Write 5 Done Done Sent false
159112024-02-12T01:41:20.621Z 23 Acked 1022 Read 10 Done Done Sent false
159122024-02-12T01:41:20.621Z 24 Acked 1023 Read 10 Done Done Sent false
159132024-02-12T01:41:20.621Z 25 Acked 1024 Read 4 Done Done Sent false
159142024-02-12T01:41:20.621Z 26 Acked 1025 Write 8 Done Done Sent false
159152024-02-12T01:41:20.621Z 27 Acked 1026 Read 2 Done Done Sent false
159162024-02-12T01:41:20.621Z 28 Acked 1027 Read 3 Done Done Sent false
159172024-02-12T01:41:20.621Z 29 Acked 1028 Read 10 Done Done Sent false
159182024-02-12T01:41:20.621Z 30 Acked 1029 Write 2 Sent Sent Sent false
159192024-02-12T01:41:20.621Z STATES DS:0 DS:1 DS:2 TOTAL
159202024-02-12T01:41:20.621Z New 0 0 0 0
159212024-02-12T01:41:20.621Z Sent 1 1 20 22
159222024-02-12T01:41:20.621Z Done 29 29 10 68
159232024-02-12T01:41:20.621Z Skipped 0 0 0 0
159242024-02-12T01:41:20.621Z Error 0 0 0 0
159252024-02-12T01:41:20.621ZLast Flush: 1018 1018 0
159262024-02-12T01:41:20.621ZDownstairs last five completed:
159272024-02-12T01:41:20.621ZUpstairs last five completed: 30 29 28 27 26
159282024-02-12T01:41:20.621Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:19.492547686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1711}
159292024-02-12T01:41:20.621ZWrote out file "/tmp/test_repair_verify.data"
159302024-02-12T01:41:20.625Z
159312024-02-12T01:41:20.625ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159322024-02-12T01:41:20.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159332024-02-12T01:41:20.644Z 0 000-019 144 144 130 251 251 227 F F T <---
159342024-02-12T01:41:20.644Z 1 020-039 142 142 142 249 249 249 F F F
159352024-02-12T01:41:20.644Z 2 040-059 142 142 142 249 249 249 T T F <---
159362024-02-12T01:41:20.644Z 3 060-079 142 142 142 248 248 248 F F F
159372024-02-12T01:41:20.644Z 4 080-099 144 144 138 251 251 241 F F F <---
159382024-02-12T01:41:20.644Z 5 100-119 144 144 137 252 252 240 T T F <---
159392024-02-12T01:41:20.644Z 6 120-139 144 144 142 252 252 248 F F F <---
159402024-02-12T01:41:20.644Z 7 140-159 137 137 137 240 240 240 F F F
159412024-02-12T01:41:20.644Z 8 160-179 136 136 136 239 239 239 T T F <---
159422024-02-12T01:41:20.644Z 9 180-199 142 142 142 248 248 248 F F F
159432024-02-12T01:41:20.644Z 10 200-219 144 144 142 251 251 248 F F F <---
159442024-02-12T01:41:20.648Z 11 220-239 142 142 142 249 249 249 F F F
159452024-02-12T01:41:20.648Z 12 240-259 140 140 140 244 244 244 F F F
159462024-02-12T01:41:20.648Z 13 260-279 141 141 141 247 247 247 T T F <---
159472024-02-12T01:41:20.648Z 14 280-299 144 144 141 251 251 247 T T T <---
159482024-02-12T01:41:20.648Z 15 300-319 138 138 138 242 242 242 F F F
159492024-02-12T01:41:20.648Z 16 320-339 139 139 139 243 243 243 F F F
159502024-02-12T01:41:20.648Z 17 340-359 144 144 142 252 252 248 F F T <---
159512024-02-12T01:41:20.648Z 18 360-379 144 144 142 251 251 249 F F T <---
159522024-02-12T01:41:20.648Z 19 380-399 144 144 142 251 251 249 F F T <---
159532024-02-12T01:41:20.648Z 20 400-419 144 144 139 251 251 243 F F T <---
159542024-02-12T01:41:20.648Z 21 420-439 143 143 143 250 250 250 F F F
159552024-02-12T01:41:20.648Z 22 440-459 142 142 142 249 249 249 F F F
159562024-02-12T01:41:20.648Z 23 460-479 142 142 142 249 249 249 F F F
159572024-02-12T01:41:20.648Z 24 480-499 141 141 141 247 247 247 F F F
159582024-02-12T01:41:20.648Z 25 500-519 138 138 138 242 242 242 F F F
159592024-02-12T01:41:20.648Z 26 520-539 143 143 143 250 250 250 F F F
159602024-02-12T01:41:20.648Z 27 540-559 138 138 138 242 242 242 F F F
159612024-02-12T01:41:20.648Z 28 560-579 140 140 140 246 246 246 F F F
159622024-02-12T01:41:20.649Z 29 580-599 140 140 140 246 246 246 F F F
159632024-02-12T01:41:20.649ZMax gen: 144, Max flush: 252
159642024-02-12T01:41:20.649ZError: Difference in extent metadata found!
159652024-02-12T01:41:20.649ZA Difference in extent metadata is expected here
159662024-02-12T01:41:20.649ZOn loop 72
159672024-02-12T01:41:20.649Z
159682024-02-12T01:41:20.652ZVerifying data now
159692024-02-12T01:41:26.303ZNo change after verify
159702024-02-12T01:41:26.306ZLoop: 72 Downstairs dump after verify (and repair):
159712024-02-12T01:41:26.315ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159722024-02-12T01:41:26.315Z 0 000-019 144 144 144 251 251 251 F F F
159732024-02-12T01:41:26.315Z 1 020-039 142 142 142 249 249 249 F F F
159742024-02-12T01:41:26.315Z 2 040-059 145 145 145 253 253 253 F F F
159752024-02-12T01:41:26.315Z 3 060-079 142 142 142 248 248 248 F F F
159762024-02-12T01:41:26.315Z 4 080-099 144 144 144 251 251 251 F F F
159772024-02-12T01:41:26.315Z 5 100-119 145 145 145 253 253 253 F F F
159782024-02-12T01:41:26.315Z 6 120-139 144 144 144 252 252 252 F F F
159792024-02-12T01:41:26.315Z 7 140-159 137 137 137 240 240 240 F F F
159802024-02-12T01:41:26.315Z 8 160-179 145 145 145 253 253 253 F F F
159812024-02-12T01:41:26.315Z 9 180-199 142 142 142 248 248 248 F F F
159822024-02-12T01:41:26.315Z 10 200-219 144 144 144 251 251 251 F F F
159832024-02-12T01:41:26.315Z 11 220-239 142 142 142 249 249 249 F F F
159842024-02-12T01:41:26.315Z 12 240-259 140 140 140 244 244 244 F F F
159852024-02-12T01:41:26.315Z 13 260-279 145 145 145 253 253 253 F F F
159862024-02-12T01:41:26.318Z 14 280-299 145 145 145 253 253 253 F F F
159872024-02-12T01:41:26.318Z 15 300-319 138 138 138 242 242 242 F F F
159882024-02-12T01:41:26.318Z 16 320-339 139 139 139 243 243 243 F F F
159892024-02-12T01:41:26.318Z 17 340-359 144 144 144 252 252 252 F F F
159902024-02-12T01:41:26.319Z 18 360-379 144 144 144 251 251 251 F F F
159912024-02-12T01:41:26.319Z 19 380-399 144 144 144 251 251 251 F F F
159922024-02-12T01:41:26.319Z 20 400-419 144 144 144 251 251 251 F F F
159932024-02-12T01:41:26.319Z 21 420-439 143 143 143 250 250 250 F F F
159942024-02-12T01:41:26.319Z 22 440-459 142 142 142 249 249 249 F F F
159952024-02-12T01:41:26.319Z 23 460-479 142 142 142 249 249 249 F F F
159962024-02-12T01:41:26.319Z 24 480-499 141 141 141 247 247 247 F F F
159972024-02-12T01:41:26.319Z 25 500-519 138 138 138 242 242 242 F F F
159982024-02-12T01:41:26.319Z 26 520-539 143 143 143 250 250 250 F F F
159992024-02-12T01:41:26.319Z 27 540-559 138 138 138 242 242 242 F F F
160002024-02-12T01:41:26.319Z 28 560-579 140 140 140 246 246 246 F F F
160012024-02-12T01:41:26.319Z 29 580-599 140 140 140 246 246 246 F F F
160022024-02-12T01:41:26.319ZMax gen: 145, Max flush: 253
160032024-02-12T01:41:26.319Z
160042024-02-12T01:41:26.319ZBegin loop 73
160052024-02-12T01:41:26.319ZDownstairs to restart: 1
160062024-02-12T01:41:26.336Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.210989514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
160072024-02-12T01:41:26.336Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211207354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
160082024-02-12T01:41:26.336Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211224053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
160092024-02-12T01:41:26.336Z{"msg":"Crucible 418b607f-d6d7-4bb0-9fcd-042cac98471b has session id: 505d74a4-3cdf-4c6c-b6cd-0756947540f6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211238602Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160102024-02-12T01:41:26.340Z{"msg":"Upstairs opts: Upstairs UUID: 418b607f-d6d7-4bb0-9fcd-042cac98471b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.21127497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160112024-02-12T01:41:26.340Z{"msg":"Crucible runtime is spawned
160122024-02-12T01:41:26.340ZCrucible stats registered with UUID: 418b607f-d6d7-4bb0-9fcd-042cac98471b","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211309179Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160132024-02-12T01:41:26.341Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211396105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
160142024-02-12T01:41:26.341Z{"msg":"418b607f-d6d7-4bb0-9fcd-042cac98471b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211506719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160152024-02-12T01:41:26.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211535598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160162024-02-12T01:41:26.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211551847Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160172024-02-12T01:41:26.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211589465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160182024-02-12T01:41:26.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211611414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160192024-02-12T01:41:26.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211632313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160202024-02-12T01:41:26.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211653592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160212024-02-12T01:41:26.341Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211686801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160222024-02-12T01:41:26.341Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211712719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160232024-02-12T01:41:26.341Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211748048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160242024-02-12T01:41:26.341Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:25.211813815Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160252024-02-12T01:41:26.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211848143Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160262024-02-12T01:41:26.341Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.211873682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160272024-02-12T01:41:26.341Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:25.21191572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160282024-02-12T01:41:26.341Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:25.212135879Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160292024-02-12T01:41:26.341Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212181437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160302024-02-12T01:41:26.341Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212221895Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160312024-02-12T01:41:26.341Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212251614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160322024-02-12T01:41:26.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212588728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160332024-02-12T01:41:26.341Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212622156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160342024-02-12T01:41:26.341Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212637145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160352024-02-12T01:41:26.342Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.212650525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160362024-02-12T01:41:26.342Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213136722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160372024-02-12T01:41:26.342Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213252686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160382024-02-12T01:41:26.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213602919Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160392024-02-12T01:41:26.342Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213653067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160402024-02-12T01:41:26.342Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213698905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160412024-02-12T01:41:26.342Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213716984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160422024-02-12T01:41:26.342Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213739813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160432024-02-12T01:41:26.342Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:25.213753932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160442024-02-12T01:41:27.338Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.213467786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160452024-02-12T01:41:27.343Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.213705125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160462024-02-12T01:41:27.343Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.214492527Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160472024-02-12T01:41:27.343Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.214530355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160482024-02-12T01:41:27.343Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215023552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160492024-02-12T01:41:27.343Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215532208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160502024-02-12T01:41:27.343Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215565076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160512024-02-12T01:41:27.343Z{"msg":"[0]R flush_numbers[0..12]: [251, 249, 253, 248, 251, 253, 252, 240, 253, 248, 251, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215588675Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160522024-02-12T01:41:27.343Z{"msg":"[0]R generation[0..12]: [144, 142, 145, 142, 144, 145, 144, 137, 145, 142, 144, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215623663Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160532024-02-12T01:41:27.343Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215645102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160542024-02-12T01:41:27.343Z{"msg":"[1]R flush_numbers[0..12]: [251, 249, 253, 248, 251, 253, 252, 240, 253, 248, 251, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215675511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160552024-02-12T01:41:27.343Z{"msg":"[1]R generation[0..12]: [144, 142, 145, 142, 144, 145, 144, 137, 145, 142, 144, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.2156958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160562024-02-12T01:41:27.343Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215731938Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160572024-02-12T01:41:27.343Z{"msg":"[2]R flush_numbers[0..12]: [251, 249, 253, 248, 251, 253, 252, 240, 253, 248, 251, 249]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215748417Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160582024-02-12T01:41:27.344Z{"msg":"[2]R generation[0..12]: [144, 142, 145, 142, 144, 145, 144, 137, 145, 142, 144, 142]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
160592024-02-12T01:41:27.344Z,"time":"2024-02-12T01:41:26.215769447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160602024-02-12T01:41:27.344Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215799465Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160612024-02-12T01:41:27.344Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215814224Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160622024-02-12T01:41:27.344Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215851883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160632024-02-12T01:41:27.344Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215884641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160642024-02-12T01:41:27.344Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.21590212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160652024-02-12T01:41:27.344Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215934099Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160662024-02-12T01:41:27.344Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215952798Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160672024-02-12T01:41:27.344Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.215972437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160682024-02-12T01:41:27.344Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.216008515Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160692024-02-12T01:41:27.344Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.216045843Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160702024-02-12T01:41:27.344Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.216077832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160712024-02-12T01:41:27.344Z{"msg":"418b607f-d6d7-4bb0-9fcd-042cac98471b is now active with session: 505d74a4-3cdf-4c6c-b6cd-0756947540f6","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.216093131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160722024-02-12T01:41:27.344Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.21611662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719,"session_id":"505d74a4-3cdf-4c6c-b6cd-0756947540f6"}
160732024-02-12T01:41:27.344Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:26.216150488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
160742024-02-12T01:41:27.344ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
160752024-02-12T01:41:27.344ZLoading write count information from file "/tmp/test_repair_verify.data"
160762024-02-12T01:41:27.344ZRun Repair workload
160772024-02-12T01:41:27.348Z01/30 Write block 77 len 32768 data: 10 10 13 15 15 17 14 15
160782024-02-12T01:41:27.348Z02/30 Write block 105 len 36864 data: 13 11 12 12 11 10 10 7 10
160792024-02-12T01:41:27.348Z03/30 Write block 551 len 4096 data: 15
160802024-02-12T01:41:27.348Z04/30 Write block 404 len 12288 data: 10 11 12
160812024-02-12T01:41:27.348Z05/30 Write block 144 len 8192 data: 17 17
160822024-02-12T01:41:27.348Z06/30 Write block 59 len 40960 data: 13 10 9 12 11 11 13 12 14 13
160832024-02-12T01:41:27.348Z07/30 Read block 590 len 4096
160842024-02-12T01:41:27.348Z08/30 Flush
160852024-02-12T01:41:27.351Z09/30 Write block 360 len 20480 data: 9 8 7 8 12
160862024-02-12T01:41:27.351Z10/30 Read block 61 len 20480
160872024-02-12T01:41:27.354Z11/30 Read block 438 len 36864
160882024-02-12T01:41:27.354Z12/30 Flush
160892024-02-12T01:41:27.357Z13/30 Write block 128 len 28672 data: 12 11 10 13 13 11 11
160902024-02-12T01:41:27.357Z14/30 Write block 63 len 24576 data: 12 12 14 13 15 14
160912024-02-12T01:41:27.357Z15/30 Read block 394 len 20480
160922024-02-12T01:41:27.360Z16/30 Read block 533 len 40960
160932024-02-12T01:41:27.360Z17/30 Read block 383 len 36864
160942024-02-12T01:41:27.360Z18/30 Flush
160952024-02-12T01:41:27.363Z19/30 Write block 337 len 28672 data: 7 6 7 7 5 5 8
160962024-02-12T01:41:27.366Z20/30 Read block 475 len 4096
160972024-02-12T01:41:27.366Z21/30 Flush
160982024-02-12T01:41:27.369Z22/30 Write block 215 len 8192 data: 10 14
160992024-02-12T01:41:27.369Z23/30 Write block 217 len 24576 data: 10 13 13 11 11 11
161002024-02-12T01:41:27.369Z24/30 Flush
161012024-02-12T01:41:27.372Z25/30 Write block 278 len 12288 data: 14 15 14
161022024-02-12T01:41:27.372Z26/30 Read block 590 len 16384
161032024-02-12T01:41:27.373Z27/30 Write block 28 len 4096 data: 14
161042024-02-12T01:41:27.376Z28/30 Write block 400 len 28672 data: 10 9 11 9 11 12 13
161052024-02-12T01:41:27.376Z29/30 Read block 148 len 32768
161062024-02-12T01:41:27.376Z30/30 Write block 400 len 12288 data: 11 10 12
161072024-02-12T01:41:27.376Z----------------------------------------------------------------
161082024-02-12T01:41:27.376Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
161092024-02-12T01:41:27.376ZStates: Active Active Active
161102024-02-12T01:41:27.376ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161112024-02-12T01:41:27.376Z 1 Acked 1000 Write 8 Done Done Done false
161122024-02-12T01:41:27.376Z 2 Acked 1001 Write 9 Done Done Done false
161132024-02-12T01:41:27.376Z 3 Acked 1002 Write 1 Done Done Done false
161142024-02-12T01:41:27.376Z 4 Acked 1003 Write 3 Done Sent Done false
161152024-02-12T01:41:27.376Z 5 Acked 1004 Write 2 Done Sent Done false
161162024-02-12T01:41:27.376Z 6 Acked 1005 Write 10 Done Sent Done false
161172024-02-12T01:41:27.376Z 7 Acked 1006 Read 1 Done Sent Done false
161182024-02-12T01:41:27.376Z 8 Acked 1007 Flush 0 Done Sent Done false
161192024-02-12T01:41:27.376Z 9 Acked 1008 Write 5 Done Sent Done false
161202024-02-12T01:41:27.377Z 10 Acked 1009 Read 5 Done Sent Done false
161212024-02-12T01:41:27.377Z 11 Acked 1010 Read 9 Done Sent Done false
161222024-02-12T01:41:27.377Z 12 Acked 1011 Flush 0 Done Sent Done false
161232024-02-12T01:41:27.377Z 13 Acked 1012 Write 7 Done Sent Done false
161242024-02-12T01:41:27.377Z 14 Acked 1013 Write 6 Done Sent Done false
161252024-02-12T01:41:27.377Z 15 Acked 1014 Read 5 Done Sent Done false
161262024-02-12T01:41:27.377Z 16 Acked 1015 Read 10 Done Sent Done false
161272024-02-12T01:41:27.377Z 17 Acked 1016 Read 9 Done Sent Done false
161282024-02-12T01:41:27.377Z 18 Acked 1017 Flush 0 Done Sent Done false
161292024-02-12T01:41:27.377Z 19 Acked 1018 Write 7 Done Sent Done false
161302024-02-12T01:41:27.377Z 20 Acked 1019 Read 1 Done Sent Done false
161312024-02-12T01:41:27.377Z 21 Acked 1020 Flush 0 Done Sent Done false
161322024-02-12T01:41:27.377Z 22 Acked 1021 Write 2 Done Sent Done false
161332024-02-12T01:41:27.377Z 23 Acked 1022 Write 6 Done Sent Done false
161342024-02-12T01:41:27.377Z 24 Acked 1023 Flush 0 Done Sent Done false
161352024-02-12T01:41:27.377Z 25 Acked 1024 Write 3 Done Sent Done false
161362024-02-12T01:41:27.377Z 26 Acked 1025 Read 4 Done Sent Done false
161372024-02-12T01:41:27.377Z 27 Acked 1026 Write 1 Done Sent Done false
161382024-02-12T01:41:27.377Z 28 Acked 1027 Write 7 Done Sent Done false
161392024-02-12T01:41:27.377Z 29 Acked 1028 Read 8 Done Sent Done false
161402024-02-12T01:41:27.377Z 30 Acked 1029 Write 3 Sent Sent Sent false
161412024-02-12T01:41:27.377Z STATES DS:0 DS:1 DS:2 TOTAL
161422024-02-12T01:41:27.377Z New 0 0 0 0
161432024-02-12T01:41:27.377Z Sent 1 27 1 29
161442024-02-12T01:41:27.378Z Done 29 3 29 61
161452024-02-12T01:41:27.378Z Skipped 0 0 0 0
161462024-02-12T01:41:27.378Z Error 0 0 0 0
161472024-02-12T01:41:27.378ZLast Flush: 1023 0 1023
161482024-02-12T01:41:27.378ZDownstairs last five completed:
161492024-02-12T01:41:27.378ZUpstairs last five completed: 30 29 28 27 26
161502024-02-12T01:41:27.378Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:26.250897951Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1719}
161512024-02-12T01:41:27.381ZWrote out file "/tmp/test_repair_verify.data"
161522024-02-12T01:41:27.381Z
161532024-02-12T01:41:27.384ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161542024-02-12T01:41:27.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161552024-02-12T01:41:27.401Z 0 000-019 144 144 144 251 251 251 F F F
161562024-02-12T01:41:27.401Z 1 020-039 142 142 142 249 249 249 T F T <---
161572024-02-12T01:41:27.401Z 2 040-059 146 145 146 254 253 254 F F F <---
161582024-02-12T01:41:27.401Z 3 060-079 146 142 146 256 248 256 F T F <---
161592024-02-12T01:41:27.401Z 4 080-099 146 144 146 254 251 254 F T F <---
161602024-02-12T01:41:27.401Z 5 100-119 146 145 146 254 253 254 F T F <---
161612024-02-12T01:41:27.401Z 6 120-139 146 144 146 256 252 256 F F F <---
161622024-02-12T01:41:27.401Z 7 140-159 146 137 146 254 240 254 F F F <---
161632024-02-12T01:41:27.401Z 8 160-179 145 145 145 253 253 253 F F F
161642024-02-12T01:41:27.401Z 9 180-199 142 142 142 248 248 248 F F F
161652024-02-12T01:41:27.401Z 10 200-219 146 144 146 258 251 258 F F F <---
161662024-02-12T01:41:27.404Z 11 220-239 146 142 146 258 249 258 F F F <---
161672024-02-12T01:41:27.405Z 12 240-259 140 140 140 244 244 244 F F F
161682024-02-12T01:41:27.405Z 13 260-279 145 145 145 253 253 253 T F T <---
161692024-02-12T01:41:27.405Z 14 280-299 145 145 145 253 253 253 T F T <---
161702024-02-12T01:41:27.405Z 15 300-319 138 138 138 242 242 242 F F F
161712024-02-12T01:41:27.405Z 16 320-339 146 139 146 257 243 257 F F F <---
161722024-02-12T01:41:27.405Z 17 340-359 146 144 146 257 252 257 F F F <---
161732024-02-12T01:41:27.405Z 18 360-379 146 144 146 255 251 255 F F F <---
161742024-02-12T01:41:27.405Z 19 380-399 144 144 144 251 251 251 F F F
161752024-02-12T01:41:27.405Z 20 400-419 146 144 146 254 251 254 T F T <---
161762024-02-12T01:41:27.405Z 21 420-439 143 143 143 250 250 250 F F F
161772024-02-12T01:41:27.405Z 22 440-459 142 142 142 249 249 249 F F F
161782024-02-12T01:41:27.405Z 23 460-479 142 142 142 249 249 249 F F F
161792024-02-12T01:41:27.405Z 24 480-499 141 141 141 247 247 247 F F F
161802024-02-12T01:41:27.405Z 25 500-519 138 138 138 242 242 242 F F F
161812024-02-12T01:41:27.405Z 26 520-539 143 143 143 250 250 250 F F F
161822024-02-12T01:41:27.405Z 27 540-559 146 138 146 254 242 254 F T F <---
161832024-02-12T01:41:27.405Z 28 560-579 140 140 140 246 246 246 F F F
161842024-02-12T01:41:27.405Z 29 580-599 140 140 140 246 246 246 F F F
161852024-02-12T01:41:27.405ZMax gen: 146, Max flush: 258
161862024-02-12T01:41:27.405ZError: Difference in extent metadata found!
161872024-02-12T01:41:27.405ZA Difference in extent metadata is expected here
161882024-02-12T01:41:27.405ZOn loop 73
161892024-02-12T01:41:27.405Z
161902024-02-12T01:41:27.409ZVerifying data now
161912024-02-12T01:41:33.172ZNo change after verify
161922024-02-12T01:41:33.172ZLoop: 73 Downstairs dump after verify (and repair):
161932024-02-12T01:41:33.187ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
161942024-02-12T01:41:33.187Z 0 000-019 144 144 144 251 251 251 F F F
161952024-02-12T01:41:33.187Z 1 020-039 147 147 147 259 259 259 F F F
161962024-02-12T01:41:33.187Z 2 040-059 146 146 146 254 254 254 F F F
161972024-02-12T01:41:33.187Z 3 060-079 146 146 146 256 256 256 F F F
161982024-02-12T01:41:33.187Z 4 080-099 146 146 146 254 254 254 F F F
161992024-02-12T01:41:33.187Z 5 100-119 146 146 146 254 254 254 F F F
162002024-02-12T01:41:33.187Z 6 120-139 146 146 146 256 256 256 F F F
162012024-02-12T01:41:33.188Z 7 140-159 146 146 146 254 254 254 F F F
162022024-02-12T01:41:33.188Z 8 160-179 145 145 145 253 253 253 F F F
162032024-02-12T01:41:33.188Z 9 180-199 142 142 142 248 248 248 F F F
162042024-02-12T01:41:33.188Z 10 200-219 146 146 146 258 258 258 F F F
162052024-02-12T01:41:33.188Z 11 220-239 146 146 146 258 258 258 F F F
162062024-02-12T01:41:33.191Z 12 240-259 140 140 140 244 244 244 F F F
162072024-02-12T01:41:33.191Z 13 260-279 147 147 147 259 259 259 F F F
162082024-02-12T01:41:33.191Z 14 280-299 147 147 147 259 259 259 F F F
162092024-02-12T01:41:33.191Z 15 300-319 138 138 138 242 242 242 F F F
162102024-02-12T01:41:33.191Z 16 320-339 146 146 146 257 257 257 F F F
162112024-02-12T01:41:33.191Z 17 340-359 146 146 146 257 257 257 F F F
162122024-02-12T01:41:33.191Z 18 360-379 146 146 146 255 255 255 F F F
162132024-02-12T01:41:33.191Z 19 380-399 144 144 144 251 251 251 F F F
162142024-02-12T01:41:33.191Z 20 400-419 147 147 147 259 259 259 F F F
162152024-02-12T01:41:33.191Z 21 420-439 143 143 143 250 250 250 F F F
162162024-02-12T01:41:33.191Z 22 440-459 142 142 142 249 249 249 F F F
162172024-02-12T01:41:33.192Z 23 460-479 142 142 142 249 249 249 F F F
162182024-02-12T01:41:33.192Z 24 480-499 141 141 141 247 247 247 F F F
162192024-02-12T01:41:33.192Z 25 500-519 138 138 138 242 242 242 F F F
162202024-02-12T01:41:33.192Z 26 520-539 143 143 143 250 250 250 F F F
162212024-02-12T01:41:33.192Z 27 540-559 146 146 146 254 254 254 F F F
162222024-02-12T01:41:33.192Z 28 560-579 140 140 140 246 246 246 F F F
162232024-02-12T01:41:33.192Z 29 580-599 140 140 140 246 246 246 F F F
162242024-02-12T01:41:33.192ZMax gen: 147, Max flush: 259
162252024-02-12T01:41:33.192Z
162262024-02-12T01:41:33.192ZBegin loop 74
162272024-02-12T01:41:33.192ZDownstairs to restart: 2
162282024-02-12T01:41:33.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.082868722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
162292024-02-12T01:41:33.208Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083131899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
162302024-02-12T01:41:33.209ZCrucible runtime is spawned
162312024-02-12T01:41:33.213Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083184017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
162322024-02-12T01:41:33.213Z{"msg":"Crucible 08245980-72c0-40bf-ad9e-fdb8141e545f has session id: 82157817-90bd-4194-8b78-ff50454f5e01","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083226425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162332024-02-12T01:41:33.213Z{"msg":"Upstairs opts: Upstairs UUID: 08245980-72c0-40bf-ad9e-fdb8141e545f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083270073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162342024-02-12T01:41:33.213Z{"msg":"Crucible stats registered with UUID: 08245980-72c0-40bf-ad9e-fdb8141e545f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083307371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162352024-02-12T01:41:33.213Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.08333484Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
162362024-02-12T01:41:33.213Z{"msg":"08245980-72c0-40bf-ad9e-fdb8141e545f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083360269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162372024-02-12T01:41:33.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083377918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162382024-02-12T01:41:33.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083409026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162392024-02-12T01:41:33.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083439045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162402024-02-12T01:41:33.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083475963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162412024-02-12T01:41:33.214Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083493342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162422024-02-12T01:41:33.214Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083510711Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162432024-02-12T01:41:33.214Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083563349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162442024-02-12T01:41:33.214Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083585438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162452024-02-12T01:41:33.214Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083653855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162462024-02-12T01:41:33.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083717001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162472024-02-12T01:41:33.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083943611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162482024-02-12T01:41:33.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.083996418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162492024-02-12T01:41:33.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.084366801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162502024-02-12T01:41:33.214Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.084407319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162512024-02-12T01:41:33.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.084500334Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162522024-02-12T01:41:33.214Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.084531133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162532024-02-12T01:41:33.214Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.084913604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162542024-02-12T01:41:33.214Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085054148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162552024-02-12T01:41:33.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.0854241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162562024-02-12T01:41:33.214Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085464738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162572024-02-12T01:41:33.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085526265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162582024-02-12T01:41:33.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085558553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162592024-02-12T01:41:33.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085598472Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162602024-02-12T01:41:33.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.08562513Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162612024-02-12T01:41:33.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.08563896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162622024-02-12T01:41:33.214Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.085659199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162632024-02-12T01:41:33.214Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086155995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162642024-02-12T01:41:33.218Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086584635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162652024-02-12T01:41:33.218Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086615683Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162662024-02-12T01:41:33.218Z{"msg":"[0]R flush_numbers[0..12]: [251, 259, 254, 256, 254, 254, 256, 254, 253, 248, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086631142Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162672024-02-12T01:41:33.218Z{"msg":"[0]R generation[0..12]: [144, 147, 146, 146, 146, 146, 146, 146, 145, 142, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086645512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162682024-02-12T01:41:33.218Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086716368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162692024-02-12T01:41:33.219Z{"msg":"[1]R flush_numbers[0..12]: [251, 259, 254, 256, 254, 254, 256, 254, 253, 248, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086744007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162702024-02-12T01:41:33.219Z{"msg":"[1]R generation[0..12]: [144, 147, 146, 146, 146, 146, 146, 146, 145, 142, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086758676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162712024-02-12T01:41:33.219Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086779275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162722024-02-12T01:41:33.219Z{"msg":"[2]R flush_numbers[0..12]: [251, 259, 254, 256, 254, 254, 256, 254, 253, 248, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086793435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162732024-02-12T01:41:33.219Z{"msg":"[2]R generation[0..12]: [144, 147, 146, 146, 146, 146, 146, 146, 145, 142, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086813744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162742024-02-12T01:41:33.219Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086828973Z","Wait for a query_work_queue command to finish before sending IO
162752024-02-12T01:41:33.219Zhostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162762024-02-12T01:41:33.219Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086855112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162772024-02-12T01:41:33.219Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086869341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162782024-02-12T01:41:33.219Z{"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.08688885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162792024-02-12T01:41:33.219Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086911619Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162802024-02-12T01:41:33.219Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086929228Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162812024-02-12T01:41:33.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086954087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162822024-02-12T01:41:33.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086968236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162832024-02-12T01:41:33.219Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.086996415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162842024-02-12T01:41:33.219Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.087020684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162852024-02-12T01:41:33.219Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.087051242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162862024-02-12T01:41:33.219Z{"msg":"08245980-72c0-40bf-ad9e-fdb8141e545f is now active with session: 82157817-90bd-4194-8b78-ff50454f5e01","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.087083141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162872024-02-12T01:41:33.219Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.087113319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727,"session_id":"82157817-90bd-4194-8b78-ff50454f5e01"}
162882024-02-12T01:41:33.219Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:32.087132169Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
162892024-02-12T01:41:33.219ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162902024-02-12T01:41:33.219ZLoading write count information from file "/tmp/test_repair_verify.data"
162912024-02-12T01:41:33.219ZRun Repair workload
162922024-02-12T01:41:33.219Z01/30 Flush
162932024-02-12T01:41:33.219Z02/30 Flush
162942024-02-12T01:41:33.220Z03/30 Flush
162952024-02-12T01:41:33.220Z04/30 Write block 361 len 40960 data: 9 8 9 13 12 11 16 15 15 15
162962024-02-12T01:41:33.220Z05/30 Read block 416 len 20480
162972024-02-12T01:41:33.223Z06/30 Write block 361 len 36864 data: 10 9 10 14 13 12 17 16 16
162982024-02-12T01:41:33.223Z07/30 Write block 103 len 32768 data: 8 11 14 12 13 13 12 11
162992024-02-12T01:41:33.223Z08/30 Write block 268 len 16384 data: 14 13 14 15
163002024-02-12T01:41:33.223Z09/30 Write block 349 len 16384 data: 9 11 13 16
163012024-02-12T01:41:33.223Z10/30 Write block 288 len 28672 data: 12 11 13 14 11 12 9
163022024-02-12T01:41:33.223Z11/30 Read block 126 len 4096
163032024-02-12T01:41:33.226Z12/30 Write block 473 len 16384 data: 14 14 12 9
163042024-02-12T01:41:33.226Z13/30 Write block 429 len 28672 data: 6 7 6 7 7 8 7
163052024-02-12T01:41:33.226Z14/30 Write block 97 len 4096 data: 9
163062024-02-12T01:41:33.226Z15/30 Read block 506 len 16384
163072024-02-12T01:41:33.232Z16/30 Write block 62 len 40960 data: 13 13 13 15 14 16 15 15 14 14
163082024-02-12T01:41:33.232Z17/30 Read block 86 len 4096
163092024-02-12T01:41:33.232Z18/30 Read block 439 len 24576
163102024-02-12T01:41:33.232Z19/30 Write block 471 len 4096 data: 10
163112024-02-12T01:41:33.232Z20/30 Write block 260 len 12288 data: 14 10 7
163122024-02-12T01:41:33.235Z21/30 Write block 437 len 20480 data: 9 9 12 11 12
163132024-02-12T01:41:33.236Z22/30 Read block 308 len 40960
163142024-02-12T01:41:33.236Z23/30 Read block 193 len 8192
163152024-02-12T01:41:33.236Z24/30 Write block 186 len 4096 data: 10
163162024-02-12T01:41:33.236Z25/30 Write block 192 len 12288 data: 11 10 9
163172024-02-12T01:41:33.236Z26/30 Read block 482 len 4096
163182024-02-12T01:41:33.236Z27/30 Write block 149 len 8192 data: 15 16
163192024-02-12T01:41:33.236Z28/30 Write block 155 len 28672 data: 7 6 7 8 9 12 12
163202024-02-12T01:41:33.236Z29/30 Read block 13 len 8192
163212024-02-12T01:41:33.239Z30/30 Write block 42 len 32768 data: 7 11 11 11 13 14 15 16
163222024-02-12T01:41:33.239Z----------------------------------------------------------------
163232024-02-12T01:41:33.239Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:28
163242024-02-12T01:41:33.239ZStates: Active Active Active
163252024-02-12T01:41:33.239ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
163262024-02-12T01:41:33.239Z 3 Acked 1002 Flush 0 Done Done Sent false
163272024-02-12T01:41:33.239Z 4 Acked 1003 Write 10 Done Done Sent false
163282024-02-12T01:41:33.239Z 5 Acked 1004 Read 5 Done Done Sent false
163292024-02-12T01:41:33.240Z 6 Acked 1005 Write 9 Done Done Sent false
163302024-02-12T01:41:33.240Z 7 Acked 1006 Write 8 Done Done Sent false
163312024-02-12T01:41:33.240Z 8 Acked 1007 Write 4 Done Done Sent false
163322024-02-12T01:41:33.240Z 9 Acked 1008 Write 4 Done Done Sent false
163332024-02-12T01:41:33.240Z 10 Acked 1009 Write 7 Done Done Sent false
163342024-02-12T01:41:33.240Z 11 Acked 1010 Read 1 Done Done Sent false
163352024-02-12T01:41:33.240Z 12 Acked 1011 Write 4 Done Done Sent false
163362024-02-12T01:41:33.240Z 13 Acked 1012 Write 7 Done Done Sent false
163372024-02-12T01:41:33.240Z 14 Acked 1013 Write 1 Done Done Sent false
163382024-02-12T01:41:33.240Z 15 Acked 1014 Read 4 Done Done Sent false
163392024-02-12T01:41:33.240Z 16 Acked 1015 Write 10 Done Done Sent false
163402024-02-12T01:41:33.240Z 17 Acked 1016 Read 1 Done Done Sent false
163412024-02-12T01:41:33.240Z 18 Acked 1017 Read 6 Done Done Sent false
163422024-02-12T01:41:33.240Z 19 Acked 1018 Write 1 Done Done Sent false
163432024-02-12T01:41:33.240Z 20 Acked 1019 Write 3 Done Done Sent false
163442024-02-12T01:41:33.240Z 21 Acked 1020 Write 5 Done Done Sent false
163452024-02-12T01:41:33.240Z 22 Acked 1021 Read 10 Done Done Sent false
163462024-02-12T01:41:33.240Z 23 Acked 1022 Read 2 Done Done Sent false
163472024-02-12T01:41:33.240Z 24 Acked 1023 Write 1 Done Done Sent false
163482024-02-12T01:41:33.240Z 25 Acked 1024 Write 3 Done Done Sent false
163492024-02-12T01:41:33.240Z 26 Acked 1025 Read 1 Done Done Sent false
163502024-02-12T01:41:33.240Z 27 Acked 1026 Write 2 Done Done Sent false
163512024-02-12T01:41:33.240Z 28 Acked 1027 Write 7 Done Done Sent false
163522024-02-12T01:41:33.240Z 29 Acked 1028 Read 2 Done Done Sent false
163532024-02-12T01:41:33.240Z 30 Acked 1029 Write 8 Sent Sent Sent false
163542024-02-12T01:41:33.241Z STATES DS:0 DS:1 DS:2 TOTAL
163552024-02-12T01:41:33.241Z New 0 0 0 0
163562024-02-12T01:41:33.241Z Sent 1 1 28 30
163572024-02-12T01:41:33.241Z Done 27 27 0 54
163582024-02-12T01:41:33.241Z Skipped 0 0 0 0
163592024-02-12T01:41:33.241Z Error 0 0 0 0
163602024-02-12T01:41:33.241ZLast Flush: 1002 1002 1001
163612024-02-12T01:41:33.241ZDownstairs last five completed: 1001 1000
163622024-02-12T01:41:33.241ZUpstairs last five completed: 30 29 28 27 26
163632024-02-12T01:41:33.241Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:32.111737205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1727}
163642024-02-12T01:41:33.241ZWrote out file "/tmp/test_repair_verify.data"
163652024-02-12T01:41:33.244Z
163662024-02-12T01:41:33.247ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163672024-02-12T01:41:33.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163682024-02-12T01:41:33.265Z 0 000-019 144 144 144 251 251 251 F F F
163692024-02-12T01:41:33.265Z 1 020-039 147 147 147 259 259 259 F F F
163702024-02-12T01:41:33.265Z 2 040-059 146 146 146 254 254 254 T T F <---
163712024-02-12T01:41:33.265Z 3 060-079 146 146 146 256 256 256 T T F <---
163722024-02-12T01:41:33.265Z 4 080-099 146 146 146 254 254 254 T T F <---
163732024-02-12T01:41:33.265Z 5 100-119 146 146 146 254 254 254 T T F <---
163742024-02-12T01:41:33.265Z 6 120-139 146 146 146 256 256 256 F F F
163752024-02-12T01:41:33.265Z 7 140-159 146 146 146 254 254 254 T T F <---
163762024-02-12T01:41:33.265Z 8 160-179 145 145 145 253 253 253 T T F <---
163772024-02-12T01:41:33.265Z 9 180-199 142 142 142 248 248 248 T T F <---
163782024-02-12T01:41:33.265Z 10 200-219 146 146 146 258 258 258 F F F
163792024-02-12T01:41:33.265Z 11 220-239 146 146 146 258 258 258 F F F
163802024-02-12T01:41:33.265Z 12 240-259 140 140 140 244 244 244 F F F
163812024-02-12T01:41:33.265Z 13 260-279 147 147 147 259 259 259 T T F <---
163822024-02-12T01:41:33.269Z 14 280-299 147 147 147 259 259 259 T T F <---
163832024-02-12T01:41:33.269Z 15 300-319 138 138 138 242 242 242 F F F
163842024-02-12T01:41:33.269Z 16 320-339 146 146 146 257 257 257 F F F
163852024-02-12T01:41:33.269Z 17 340-359 146 146 146 257 257 257 T T F <---
163862024-02-12T01:41:33.269Z 18 360-379 146 146 146 255 255 255 T T F <---
163872024-02-12T01:41:33.269Z 19 380-399 144 144 144 251 251 251 F F F
163882024-02-12T01:41:33.269Z 20 400-419 147 147 147 259 259 259 F F F
163892024-02-12T01:41:33.269Z 21 420-439 143 143 143 250 250 250 T T F <---
163902024-02-12T01:41:33.269Z 22 440-459 142 142 142 249 249 249 T T F <---
163912024-02-12T01:41:33.269Z 23 460-479 142 142 142 249 249 249 T T F <---
163922024-02-12T01:41:33.269Z 24 480-499 141 141 141 247 247 247 F F F
163932024-02-12T01:41:33.269Z 25 500-519 138 138 138 242 242 242 F F F
163942024-02-12T01:41:33.269Z 26 520-539 143 143 143 250 250 250 F F F
163952024-02-12T01:41:33.269Z 27 540-559 146 146 146 254 254 254 F F F
163962024-02-12T01:41:33.269Z 28 560-579 140 140 140 246 246 246 F F F
163972024-02-12T01:41:33.269Z 29 580-599 140 140 140 246 246 246 F F F
163982024-02-12T01:41:33.270ZMax gen: 147, Max flush: 259
163992024-02-12T01:41:33.270ZError: Difference in extent metadata found!
164002024-02-12T01:41:33.270ZA Difference in extent metadata is expected here
164012024-02-12T01:41:33.270ZOn loop 74
164022024-02-12T01:41:33.270Z
164032024-02-12T01:41:33.273ZVerifying data now
164042024-02-12T01:41:37.977ZNo change after verify
164052024-02-12T01:41:37.977ZLoop: 74 Downstairs dump after verify (and repair):
164062024-02-12T01:41:37.992ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164072024-02-12T01:41:37.992Z 0 000-019 144 144 144 251 251 251 F F F
164082024-02-12T01:41:37.992Z 1 020-039 147 147 147 259 259 259 F F F
164092024-02-12T01:41:37.992Z 2 040-059 148 148 148 260 260 260 F F F
164102024-02-12T01:41:37.992Z 3 060-079 148 148 148 260 260 260 F F F
164112024-02-12T01:41:37.992Z 4 080-099 148 148 148 260 260 260 F F F
164122024-02-12T01:41:37.992Z 5 100-119 148 148 148 260 260 260 F F F
164132024-02-12T01:41:37.992Z 6 120-139 146 146 146 256 256 256 F F F
164142024-02-12T01:41:37.996Z 7 140-159 148 148 148 260 260 260 F F F
164152024-02-12T01:41:37.996Z 8 160-179 148 148 148 260 260 260 F F F
164162024-02-12T01:41:37.996Z 9 180-199 148 148 148 260 260 260 F F F
164172024-02-12T01:41:37.996Z 10 200-219 146 146 146 258 258 258 F F F
164182024-02-12T01:41:37.996Z 11 220-239 146 146 146 258 258 258 F F F
164192024-02-12T01:41:37.996Z 12 240-259 140 140 140 244 244 244 F F F
164202024-02-12T01:41:37.996Z 13 260-279 148 148 148 260 260 260 F F F
164212024-02-12T01:41:37.996Z 14 280-299 148 148 148 260 260 260 F F F
164222024-02-12T01:41:37.996Z 15 300-319 138 138 138 242 242 242 F F F
164232024-02-12T01:41:37.996Z 16 320-339 146 146 146 257 257 257 F F F
164242024-02-12T01:41:37.997Z 17 340-359 148 148 148 260 260 260 F F F
164252024-02-12T01:41:37.997Z 18 360-379 148 148 148 260 260 260 F F F
164262024-02-12T01:41:37.997Z 19 380-399 144 144 144 251 251 251 F F F
164272024-02-12T01:41:37.997Z 20 400-419 147 147 147 259 259 259 F F F
164282024-02-12T01:41:37.997Z 21 420-439 148 148 148 260 260 260 F F F
164292024-02-12T01:41:37.997Z 22 440-459 148 148 148 260 260 260 F F F
164302024-02-12T01:41:37.997Z 23 460-479 148 148 148 260 260 260 F F F
164312024-02-12T01:41:37.997Z 24 480-499 141 141 141 247 247 247 F F F
164322024-02-12T01:41:37.997Z 25 500-519 138 138 138 242 242 242 F F F
164332024-02-12T01:41:37.997Z 26 520-539 143 143 143 250 250 250 F F F
164342024-02-12T01:41:37.997Z 27 540-559 146 146 146 254 254 254 F F F
164352024-02-12T01:41:37.997Z 28 560-579 140 140 140 246 246 246 F F F
164362024-02-12T01:41:37.997Z 29 580-599 140 140 140 246 246 246 F F F
164372024-02-12T01:41:37.998ZMax gen: 148, Max flush: 260
164382024-02-12T01:41:37.998Z
164392024-02-12T01:41:37.998ZBegin loop 75
164402024-02-12T01:41:37.998ZDownstairs to restart: 2
164412024-02-12T01:41:38.019Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892523487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
164422024-02-12T01:41:38.019Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892750026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
164432024-02-12T01:41:38.019Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892766825Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
164442024-02-12T01:41:38.019Z{"msg":"Crucible 75a875f6-47b6-472d-8862-93e46c296c7a has session id: a2b1910d-01d5-4257-ab19-f1ceddfbd596","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892798503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164452024-02-12T01:41:38.026Z{"msg":"Upstairs opts: Upstairs UUID: 75a875f6-47b6-472d-8862-93e46c296c7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892837501Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735Crucible runtime is spawned
164462024-02-12T01:41:38.026Z,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164472024-02-12T01:41:38.026Z{"msg":"Crucible stats registered with UUID: 75a875f6-47b6-472d-8862-93e46c296c7a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.89287635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164482024-02-12T01:41:38.026Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.892946096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
164492024-02-12T01:41:38.026Z{"msg":"75a875f6-47b6-472d-8862-93e46c296c7a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893013103Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164502024-02-12T01:41:38.026Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893037902Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164512024-02-12T01:41:38.026Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893053891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164522024-02-12T01:41:38.027Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.89307795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164532024-02-12T01:41:38.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893118078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164542024-02-12T01:41:38.027Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893152186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164552024-02-12T01:41:38.027Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893173036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164562024-02-12T01:41:38.027Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893198534Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164572024-02-12T01:41:38.027Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893237373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164582024-02-12T01:41:38.027Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893277651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164592024-02-12T01:41:38.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893398085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164602024-02-12T01:41:38.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893426684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164612024-02-12T01:41:38.027Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.893442063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164622024-02-12T01:41:38.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894141079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164632024-02-12T01:41:38.027Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894177818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164642024-02-12T01:41:38.027Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894226845Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164652024-02-12T01:41:38.028Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894264664Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164662024-02-12T01:41:38.028Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894350169Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164672024-02-12T01:41:38.028Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894411357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164682024-02-12T01:41:38.028Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894866875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164692024-02-12T01:41:38.028Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894904193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164702024-02-12T01:41:38.028Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.894924422Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164712024-02-12T01:41:38.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895419208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164722024-02-12T01:41:38.028Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895471936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164732024-02-12T01:41:38.028Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895516354Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164742024-02-12T01:41:38.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895557172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164752024-02-12T01:41:38.028Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.89559429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164762024-02-12T01:41:38.028Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895628239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164772024-02-12T01:41:38.028Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895659627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164782024-02-12T01:41:38.028Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895694655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164792024-02-12T01:41:38.028Z{"msg":"[0]R flush_numbers[0..12]: [251, 259, 260, 260, 260, 260, 256, 260, 260, 260, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895729184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164802024-02-12T01:41:38.029Z{"msg":"[0]R generation[0..12]: [144, 147, 148, 148, 148, 148, 146, 148, 148, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895765822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164812024-02-12T01:41:38.029ZWait for a query_work_queue command to finish before sending IO
164822024-02-12T01:41:38.029Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.89580813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164832024-02-12T01:41:38.029Z{"msg":"[1]R flush_numbers[0..12]: [251, 259, 260, 260, 260, 260, 256, 260, 260, 260, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895849198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164842024-02-12T01:41:38.029Z{"msg":"[1]R generation[0..12]: [144, 147, 148, 148, 148, 148, 146, 148, 148, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895886336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164852024-02-12T01:41:38.029Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895924634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164862024-02-12T01:41:38.029Z{"msg":"[2]R flush_numbers[0..12]: [251, 259, 260, 260, 260, 260, 256, 260, 260, 260, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.895963632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164872024-02-12T01:41:38.029Z{"msg":"[2]R generation[0..12]: [144, 147, 148, 148, 148, 148, 146, 148, 148, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896000001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164882024-02-12T01:41:38.029Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896038889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164892024-02-12T01:41:38.029Z{"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896072797Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164902024-02-12T01:41:38.029Z{"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896104436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164912024-02-12T01:41:38.029Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896139634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164922024-02-12T01:41:38.030Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896175502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164932024-02-12T01:41:38.030ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164942024-02-12T01:41:38.030Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896214191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164952024-02-12T01:41:38.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896240309Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164962024-02-12T01:41:38.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896255038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164972024-02-12T01:41:38.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896275937Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164982024-02-12T01:41:38.030Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896289667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
164992024-02-12T01:41:38.030Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896308916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
165002024-02-12T01:41:38.030Z{"msg":"75a875f6-47b6-472d-8862-93e46c296c7a is now active with session: a2b1910d-01d5-4257-ab19-f1ceddfbd596","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896322005Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
165012024-02-12T01:41:38.030Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896340805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735,"session_id":"a2b1910d-01d5-4257-ab19-f1ceddfbd596"}
165022024-02-12T01:41:38.030Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:36.896353404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
165032024-02-12T01:41:38.030ZLoading write count information from file "/tmp/test_repair_verify.data"
165042024-02-12T01:41:38.030ZRun Repair workload
165052024-02-12T01:41:38.035Z01/30 Write block 82 len 24576 data: 18 15 16 16 13 16
165062024-02-12T01:41:38.035Z02/30 Read block 570 len 36864
165072024-02-12T01:41:38.035Z03/30 Write block 403 len 4096 data: 10
165082024-02-12T01:41:38.035Z04/30 Write block 570 len 20480 data: 15 13 15 18 17
165092024-02-12T01:41:38.035Z05/30 Write block 339 len 32768 data: 8 8 6 6 9 7 8 9
165102024-02-12T01:41:38.035Z06/30 Read block 69 len 16384
165112024-02-12T01:41:38.035Z07/30 Read block 177 len 8192
165122024-02-12T01:41:38.035Z08/30 Write block 393 len 40960 data: 17 16 12 11 11 11 9 12 11 13
165132024-02-12T01:41:38.035Z09/30 Write block 183 len 16384 data: 13 14 11 11
165142024-02-12T01:41:38.035Z10/30 Flush
165152024-02-12T01:41:38.050Z11/30 Flush
165162024-02-12T01:41:38.050Z12/30 Write block 350 len 8192 data: 12 14
165172024-02-12T01:41:38.051Z13/30 Read block 124 len 20480
165182024-02-12T01:41:38.051Z14/30 Flush
165192024-02-12T01:41:38.054Z15/30 Write block 340 len 4096 data: 9
165202024-02-12T01:41:38.054Z16/30 Write block 196 len 8192 data: 10 11
165212024-02-12T01:41:38.054Z17/30 Flush
165222024-02-12T01:41:38.054Z18/30 Write block 455 len 40960 data: 8 8 9 13 10 9 10 10 10 9
165232024-02-12T01:41:38.054Z19/30 Flush
165242024-02-12T01:41:38.055Z20/30 Write block 520 len 24576 data: 11 11 12 12 11 7
165252024-02-12T01:41:38.055Z21/30 Read block 390 len 28672
165262024-02-12T01:41:38.055Z22/30 Read block 83 len 8192
165272024-02-12T01:41:38.055Z23/30 Write block 423 len 36864 data: 8 11 8 8 6 5 7 8 7
165282024-02-12T01:41:38.055Z24/30 Write block 62 len 24576 data: 14 14 14 16 15 17
165292024-02-12T01:41:38.055Z25/30 Read block 592 len 28672
165302024-02-12T01:41:38.055Z26/30 Flush
165312024-02-12T01:41:38.059Z27/30 Write block 124 len 32768 data: 15 14 16 15 13 12 11 14
165322024-02-12T01:41:38.059Z28/30 Write block 87 len 40960 data: 17 14 12 10 11 9 7 6 6 8
165332024-02-12T01:41:38.059Z29/30 Write block 60 len 4096 data: 11
165342024-02-12T01:41:38.059Z30/30 Read block 373 len 36864
165352024-02-12T01:41:38.059Z----------------------------------------------------------------
165362024-02-12T01:41:38.059Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
165372024-02-12T01:41:38.059ZStates: Active Active Active
165382024-02-12T01:41:38.059ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165392024-02-12T01:41:38.059Z 1 Acked 1000 Write 6 Done Done Done false
165402024-02-12T01:41:38.059Z 2 Acked 1001 Read 9 Done Done Done false
165412024-02-12T01:41:38.060Z 3 Acked 1002 Write 1 Done Done Done false
165422024-02-12T01:41:38.060Z 4 Acked 1003 Write 5 Done Done Done false
165432024-02-12T01:41:38.060Z 5 Acked 1004 Write 8 Done Done Sent false
165442024-02-12T01:41:38.060Z 6 Acked 1005 Read 4 Done Done Sent false
165452024-02-12T01:41:38.060Z 7 Acked 1006 Read 2 Done Done Sent false
165462024-02-12T01:41:38.060Z 8 Acked 1007 Write 10 Done Done Sent false
165472024-02-12T01:41:38.060Z 9 Acked 1008 Write 4 Done Done Sent false
165482024-02-12T01:41:38.060Z 10 Acked 1009 Flush 0 Done Done Sent false
165492024-02-12T01:41:38.060Z 11 Acked 1010 Flush 0 Done Done Sent false
165502024-02-12T01:41:38.060Z 12 Acked 1011 Write 2 Done Done Sent false
165512024-02-12T01:41:38.060Z 13 Acked 1012 Read 5 Done Done Sent false
165522024-02-12T01:41:38.060Z 14 Acked 1013 Flush 0 Done Done Sent false
165532024-02-12T01:41:38.060Z 15 Acked 1014 Write 1 Done Done Sent false
165542024-02-12T01:41:38.060Z 16 Acked 1015 Write 2 Done Done Sent false
165552024-02-12T01:41:38.060Z 17 Acked 1016 Flush 0 Done Done Sent false
165562024-02-12T01:41:38.061Z 18 Acked 1017 Write 10 Done Done Sent false
165572024-02-12T01:41:38.061Z 19 Acked 1018 Flush 0 Done Done Sent false
165582024-02-12T01:41:38.061Z 20 Acked 1019 Write 6 Done Done Sent false
165592024-02-12T01:41:38.061Z 21 Acked 1020 Read 7 Done Done Sent false
165602024-02-12T01:41:38.061Z 22 Acked 1021 Read 2 Done Done Sent false
165612024-02-12T01:41:38.061Z 23 Acked 1022 Write 9 Done Done Sent false
165622024-02-12T01:41:38.061Z 24 Acked 1023 Write 6 Done Done Sent false
165632024-02-12T01:41:38.061Z 25 Acked 1024 Read 7 Done Done Sent false
165642024-02-12T01:41:38.061Z 26 Acked 1025 Flush 0 Done Done Sent false
165652024-02-12T01:41:38.061Z 27 Acked 1026 Write 8 Done Done Sent false
165662024-02-12T01:41:38.061Z 28 Acked 1027 Write 10 Done Done Sent false
165672024-02-12T01:41:38.061Z 29 Acked 1028 Write 1 Done Done Sent false
165682024-02-12T01:41:38.061Z 30 Acked 1029 Read 9 Done Done Sent false
165692024-02-12T01:41:38.061Z STATES DS:0 DS:1 DS:2 TOTAL
165702024-02-12T01:41:38.062Z New 0 0 0 0
165712024-02-12T01:41:38.062Z Sent 0 0 26 26
165722024-02-12T01:41:38.062Z Done 30 30 4 64
165732024-02-12T01:41:38.062Z Skipped 0 0 0 0
165742024-02-12T01:41:38.062Z Error 0 0 0 0
165752024-02-12T01:41:38.062ZLast Flush: 1025 1025 0
165762024-02-12T01:41:38.062ZDownstairs last five completed:
165772024-02-12T01:41:38.062ZUpstairs last five completed: 30 29 28 27 26
165782024-02-12T01:41:38.062Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:36.932503799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1735}
165792024-02-12T01:41:38.066ZWrote out file "/tmp/test_repair_verify.data"
165802024-02-12T01:41:38.066Z
165812024-02-12T01:41:38.066ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
165822024-02-12T01:41:38.083ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165832024-02-12T01:41:38.083Z 0 000-019 144 144 144 251 251 251 F F F
165842024-02-12T01:41:38.083Z 1 020-039 147 147 147 259 259 259 F F F
165852024-02-12T01:41:38.083Z 2 040-059 148 148 148 260 260 260 F F F
165862024-02-12T01:41:38.083Z 3 060-079 150 150 148 266 266 260 T T F <---
165872024-02-12T01:41:38.083Z 4 080-099 150 150 148 261 261 260 T T T <---
165882024-02-12T01:41:38.083Z 5 100-119 148 148 148 260 260 260 F F F
165892024-02-12T01:41:38.083Z 6 120-139 146 146 146 256 256 256 T T F <---
165902024-02-12T01:41:38.083Z 7 140-159 148 148 148 260 260 260 F F F
165912024-02-12T01:41:38.084Z 8 160-179 148 148 148 260 260 260 F F F
165922024-02-12T01:41:38.084Z 9 180-199 150 150 148 264 264 260 F F F <---
165932024-02-12T01:41:38.084Z 10 200-219 146 146 146 258 258 258 F F F
165942024-02-12T01:41:38.087Z 11 220-239 146 146 146 258 258 258 F F F
165952024-02-12T01:41:38.087Z 12 240-259 140 140 140 244 244 244 F F F
165962024-02-12T01:41:38.088Z 13 260-279 148 148 148 260 260 260 F F F
165972024-02-12T01:41:38.088Z 14 280-299 148 148 148 260 260 260 F F F
165982024-02-12T01:41:38.088Z 15 300-319 138 138 138 242 242 242 F F F
165992024-02-12T01:41:38.088Z 16 320-339 150 150 146 261 261 257 F F F <---
166002024-02-12T01:41:38.088Z 17 340-359 150 150 148 264 264 260 F F F <---
166012024-02-12T01:41:38.088Z 18 360-379 148 148 148 260 260 260 F F F
166022024-02-12T01:41:38.088Z 19 380-399 150 150 144 261 261 251 F F F <---
166032024-02-12T01:41:38.088Z 20 400-419 150 150 147 261 261 259 F F T <---
166042024-02-12T01:41:38.088Z 21 420-439 150 150 148 266 266 260 F F F <---
166052024-02-12T01:41:38.088Z 22 440-459 150 150 148 265 265 260 F F F <---
166062024-02-12T01:41:38.088Z 23 460-479 150 150 148 265 265 260 F F F <---
166072024-02-12T01:41:38.088Z 24 480-499 141 141 141 247 247 247 F F F
166082024-02-12T01:41:38.088Z 25 500-519 138 138 138 242 242 242 F F F
166092024-02-12T01:41:38.088Z 26 520-539 150 150 143 266 266 250 F F F <---
166102024-02-12T01:41:38.089Z 27 540-559 146 146 146 254 254 254 F F F
166112024-02-12T01:41:38.089Z 28 560-579 150 150 140 261 261 246 F F T <---
166122024-02-12T01:41:38.089Z 29 580-599 140 140 140 246 246 246 F F F
166132024-02-12T01:41:38.089ZMax gen: 150, Max flush: 266
166142024-02-12T01:41:38.089ZError: Difference in extent metadata found!
166152024-02-12T01:41:38.089ZA Difference in extent metadata is expected here
166162024-02-12T01:41:38.089ZOn loop 75
166172024-02-12T01:41:38.089Z
166182024-02-12T01:41:38.093ZVerifying data now
166192024-02-12T01:41:43.722ZNo change after verify
166202024-02-12T01:41:43.722ZLoop: 75 Downstairs dump after verify (and repair):
166212024-02-12T01:41:43.738ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166222024-02-12T01:41:43.738Z 0 000-019 144 144 144 251 251 251 F F F
166232024-02-12T01:41:43.738Z 1 020-039 147 147 147 259 259 259 F F F
166242024-02-12T01:41:43.738Z 2 040-059 148 148 148 260 260 260 F F F
166252024-02-12T01:41:43.738Z 3 060-079 151 151 151 267 267 267 F F F
166262024-02-12T01:41:43.738Z 4 080-099 151 151 151 267 267 267 F F F
166272024-02-12T01:41:43.739Z 5 100-119 148 148 148 260 260 260 F F F
166282024-02-12T01:41:43.739Z 6 120-139 151 151 151 267 267 267 F F F
166292024-02-12T01:41:43.739Z 7 140-159 148 148 148 260 260 260 F F F
166302024-02-12T01:41:43.739Z 8 160-179 148 148 148 260 260 260 F F F
166312024-02-12T01:41:43.742Z 9 180-199 150 150 150 264 264 264 F F F
166322024-02-12T01:41:43.742Z 10 200-219 146 146 146 258 258 258 F F F
166332024-02-12T01:41:43.742Z 11 220-239 146 146 146 258 258 258 F F F
166342024-02-12T01:41:43.742Z 12 240-259 140 140 140 244 244 244 F F F
166352024-02-12T01:41:43.742Z 13 260-279 148 148 148 260 260 260 F F F
166362024-02-12T01:41:43.742Z 14 280-299 148 148 148 260 260 260 F F F
166372024-02-12T01:41:43.742Z 15 300-319 138 138 138 242 242 242 F F F
166382024-02-12T01:41:43.742Z 16 320-339 150 150 150 261 261 261 F F F
166392024-02-12T01:41:43.742Z 17 340-359 150 150 150 264 264 264 F F F
166402024-02-12T01:41:43.742Z 18 360-379 148 148 148 260 260 260 F F F
166412024-02-12T01:41:43.742Z 19 380-399 150 150 150 261 261 261 F F F
166422024-02-12T01:41:43.743Z 20 400-419 150 150 150 261 261 261 F F F
166432024-02-12T01:41:43.743Z 21 420-439 150 150 150 266 266 266 F F F
166442024-02-12T01:41:43.743Z 22 440-459 150 150 150 265 265 265 F F F
166452024-02-12T01:41:43.743Z 23 460-479 150 150 150 265 265 265 F F F
166462024-02-12T01:41:43.743Z 24 480-499 141 141 141 247 247 247 F F F
166472024-02-12T01:41:43.743Z 25 500-519 138 138 138 242 242 242 F F F
166482024-02-12T01:41:43.743Z 26 520-539 150 150 150 266 266 266 F F F
166492024-02-12T01:41:43.743Z 27 540-559 146 146 146 254 254 254 F F F
166502024-02-12T01:41:43.743Z 28 560-579 150 150 150 261 261 261 F F F
166512024-02-12T01:41:43.743Z 29 580-599 140 140 140 246 246 246 F F F
166522024-02-12T01:41:43.743ZMax gen: 151, Max flush: 267
166532024-02-12T01:41:43.746Z
166542024-02-12T01:41:43.746ZBegin loop 76
166552024-02-12T01:41:43.746ZDownstairs to restart: 2
166562024-02-12T01:41:43.763Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636186372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
166572024-02-12T01:41:43.763Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
166582024-02-12T01:41:43.763Z,"time":"2024-02-12T01:41:42.636471218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
166592024-02-12T01:41:43.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636511356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
166602024-02-12T01:41:43.770Z{"msg":"Crucible d2b227ce-68c7-40a8-9d73-a9885b7390dd has session id: d21bef45-78fd-4cd3-bacc-f3a71c7c1e76","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636541345Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166612024-02-12T01:41:43.770Z{"msg":"Upstairs opts: Upstairs UUID: d2b227ce-68c7-40a8-9d73-a9885b7390dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636591172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166622024-02-12T01:41:43.770Z{"msg":"Crucible stats registered with UUID: d2b227ce-68c7-40a8-9d73-a9885b7390dd","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636612981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166632024-02-12T01:41:43.770Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636650429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
166642024-02-12T01:41:43.770Z{"msg":"d2b227ce-68c7-40a8-9d73-a9885b7390dd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636666849Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166652024-02-12T01:41:43.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636691907Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166662024-02-12T01:41:43.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636722496Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166672024-02-12T01:41:43.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636766604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166682024-02-12T01:41:43.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636802162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166692024-02-12T01:41:43.770Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636859839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166702024-02-12T01:41:43.770Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636908517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166712024-02-12T01:41:43.770Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636945825Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166722024-02-12T01:41:43.770Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.636986724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166732024-02-12T01:41:43.770Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637009082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166742024-02-12T01:41:43.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637034741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166752024-02-12T01:41:43.770Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637179034Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166762024-02-12T01:41:43.770Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:42.637313248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166772024-02-12T01:41:43.770Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:42.637357436Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166782024-02-12T01:41:43.771Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:42.637416093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166792024-02-12T01:41:43.771Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637448901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166802024-02-12T01:41:43.771Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.63747525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166812024-02-12T01:41:43.771Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637499139Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166822024-02-12T01:41:43.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637713579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166832024-02-12T01:41:43.771Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637772386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166842024-02-12T01:41:43.771Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637798835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166852024-02-12T01:41:43.771Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.637830553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166862024-02-12T01:41:43.771Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638363908Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166872024-02-12T01:41:43.771Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638404236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166882024-02-12T01:41:43.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638908072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166892024-02-12T01:41:43.771Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.63894134Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166902024-02-12T01:41:43.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638956219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166912024-02-12T01:41:43.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638969259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166922024-02-12T01:41:43.771Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.638993368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166932024-02-12T01:41:43.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:42.639008067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166942024-02-12T01:41:44.765Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.638504512Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166952024-02-12T01:41:44.765Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.638593738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166962024-02-12T01:41:44.770Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.639436148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166972024-02-12T01:41:44.770Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.639516464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166982024-02-12T01:41:44.770Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.639984692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
166992024-02-12T01:41:44.770Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640501997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167002024-02-12T01:41:44.770Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640545585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167012024-02-12T01:41:44.770Z{"msg":"[0]R flush_numbers[0..12]: [251, 259, 260, 267, 267, 260, 267, 260, 260, 264, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640573533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167022024-02-12T01:41:44.770Z{"msg":"[0]R generation[0..12]: [144, 147, 148, 151, 151, 148, 151, 148, 148, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640607982Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167032024-02-12T01:41:44.770Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167042024-02-12T01:41:44.770Z{"msg":"[1]R flush_numbers[0..12]: [251, 259, 260, 267, 267, 260, 267, 260, 260, 264, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640691728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167052024-02-12T01:41:44.770Z{"msg":"[1]R generation[0..12]: [144, 147, 148, 151, 151, 148, 151, 148, 148, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640726446Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167062024-02-12T01:41:44.770Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640763254Z","Wait for a query_work_queue command to finish before sending IO
167072024-02-12T01:41:44.770Zhostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167082024-02-12T01:41:44.770Z{"msg":"[2]R flush_numbers[0..12]: [251, 259, 260, 267, 267, 260, 267, 260, 260, 264, 258, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640805362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167092024-02-12T01:41:44.770Z{"msg":"[2]R generation[0..12]: [144, 147, 148, 151, 151, 148, 151, 148, 148, 150, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.64084362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167102024-02-12T01:41:44.770Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640882638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167112024-02-12T01:41:44.771Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640918187Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167122024-02-12T01:41:44.771Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640953685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167132024-02-12T01:41:44.771Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.640988173Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167142024-02-12T01:41:44.771Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641023492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167152024-02-12T01:41:44.771Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641040511Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167162024-02-12T01:41:44.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.64106914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167172024-02-12T01:41:44.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641104928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167182024-02-12T01:41:44.771Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641141206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167192024-02-12T01:41:44.771Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641177544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167202024-02-12T01:41:44.771Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641216373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167212024-02-12T01:41:44.771Z{"msg":"d2b227ce-68c7-40a8-9d73-a9885b7390dd is now active with session: d21bef45-78fd-4cd3-bacc-f3a71c7c1e76","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641250941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167222024-02-12T01:41:44.771ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167232024-02-12T01:41:44.771Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641286939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743,"session_id":"d21bef45-78fd-4cd3-bacc-f3a71c7c1e76"}
167242024-02-12T01:41:44.771Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:43.641318698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
167252024-02-12T01:41:44.771ZLoading write count information from file "/tmp/test_repair_verify.data"
167262024-02-12T01:41:44.771ZRun Repair workload
167272024-02-12T01:41:44.774Z01/30 Write block 301 len 32768 data: 6 5 6 8 10 11 13 14
167282024-02-12T01:41:44.774Z02/30 Write block 25 len 4096 data: 11
167292024-02-12T01:41:44.775Z03/30 Read block 88 len 36864
167302024-02-12T01:41:44.775Z04/30 Read block 231 len 24576
167312024-02-12T01:41:44.775Z05/30 Write block 292 len 24576 data: 12 13 10 11 9 9
167322024-02-12T01:41:44.775Z06/30 Read block 61 len 36864
167332024-02-12T01:41:44.780Z07/30 Read block 187 len 32768
167342024-02-12T01:41:44.780Z08/30 Read block 371 len 36864
167352024-02-12T01:41:44.781Z09/30 Read block 526 len 24576
167362024-02-12T01:41:44.783Z10/30 Read block 70 len 8192
167372024-02-12T01:41:44.783Z11/30 Write block 320 len 40960 data: 9 9 11 9 10 9 11 10 10 9
167382024-02-12T01:41:44.783Z12/30 Read block 546 len 24576
167392024-02-12T01:41:44.783Z13/30 Write block 231 len 4096 data: 8
167402024-02-12T01:41:44.784Z14/30 Flush
167412024-02-12T01:41:44.787Z15/30 Flush
167422024-02-12T01:41:44.787Z16/30 Write block 258 len 28672 data: 13 11 15 11 8 8 8
167432024-02-12T01:41:44.787Z17/30 Write block 270 len 12288 data: 15 16 16
167442024-02-12T01:41:44.787Z18/30 Write block 311 len 20480 data: 12 10 12 10 11
167452024-02-12T01:41:44.790Z19/30 Write block 222 len 16384 data: 12 12 12 11
167462024-02-12T01:41:44.790Z20/30 Write block 389 len 16384 data: 13 17 16 16
167472024-02-12T01:41:44.790Z21/30 Read block 384 len 8192
167482024-02-12T01:41:44.790Z22/30 Read block 560 len 24576
167492024-02-12T01:41:44.790Z23/30 Read block 179 len 16384
167502024-02-12T01:41:44.793Z24/30 Read block 57 len 12288
167512024-02-12T01:41:44.793Z25/30 Flush
167522024-02-12T01:41:44.796Z26/30 Write block 398 len 24576 data: 12 10 13 12 14 11
167532024-02-12T01:41:44.796Z27/30 Read block 313 len 40960
167542024-02-12T01:41:44.799Z28/30 Read block 589 len 28672
167552024-02-12T01:41:44.799Z29/30 Read block 358 len 16384
167562024-02-12T01:41:44.799Z30/30 Read block 0 len 32768
167572024-02-12T01:41:44.802Z----------------------------------------------------------------
167582024-02-12T01:41:44.802Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
167592024-02-12T01:41:44.802ZStates: Active Active Active
167602024-02-12T01:41:44.802ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167612024-02-12T01:41:44.802Z 1 Acked 1000 Write 8 Done Done Done false
167622024-02-12T01:41:44.803Z 2 Acked 1001 Write 1 Done Done Done false
167632024-02-12T01:41:44.803Z 3 Acked 1002 Read 9 Done Done Done false
167642024-02-12T01:41:44.803Z 4 Acked 1003 Read 6 Done Done Sent false
167652024-02-12T01:41:44.803Z 5 Acked 1004 Write 6 Done Done Sent false
167662024-02-12T01:41:44.803Z 6 Acked 1005 Read 9 Done Done Sent false
167672024-02-12T01:41:44.803Z 7 Acked 1006 Read 8 Done Done Sent false
167682024-02-12T01:41:44.803Z 8 Acked 1007 Read 9 Done Done Sent false
167692024-02-12T01:41:44.803Z 9 Acked 1008 Read 6 Done Done Sent false
167702024-02-12T01:41:44.803Z 10 Acked 1009 Read 2 Done Done Sent false
167712024-02-12T01:41:44.803Z 11 Acked 1010 Write 10 Done Done Sent false
167722024-02-12T01:41:44.803Z 12 Acked 1011 Read 6 Done Done Sent false
167732024-02-12T01:41:44.803Z 13 Acked 1012 Write 1 Done Done Sent false
167742024-02-12T01:41:44.803Z 14 Acked 1013 Flush 0 Done Done Sent false
167752024-02-12T01:41:44.803Z 15 Acked 1014 Flush 0 Done Done Sent false
167762024-02-12T01:41:44.803Z 16 Acked 1015 Write 7 Done Done Sent false
167772024-02-12T01:41:44.803Z 17 Acked 1016 Write 3 Done Done Sent false
167782024-02-12T01:41:44.803Z 18 Acked 1017 Write 5 Done Done Sent false
167792024-02-12T01:41:44.803Z 19 Acked 1018 Write 4 Done Done Sent false
167802024-02-12T01:41:44.803Z 20 Acked 1019 Write 4 Done Done Sent false
167812024-02-12T01:41:44.803Z 21 Acked 1020 Read 2 Done Done Sent false
167822024-02-12T01:41:44.803Z 22 Acked 1021 Read 6 Done Done Sent false
167832024-02-12T01:41:44.803Z 23 Acked 1022 Read 4 Done Done Sent false
167842024-02-12T01:41:44.803Z 24 Acked 1023 Read 3 Done Done Sent false
167852024-02-12T01:41:44.804Z 25 Acked 1024 Flush 0 Done Done Sent false
167862024-02-12T01:41:44.804Z 26 Acked 1025 Write 6 Done Done Sent false
167872024-02-12T01:41:44.804Z 27 Acked 1026 Read 10 Done Done Sent false
167882024-02-12T01:41:44.804Z 28 Acked 1027 Read 7 Done Done Sent false
167892024-02-12T01:41:44.804Z 29 Acked 1028 Read 4 Done Done Sent false
167902024-02-12T01:41:44.804Z 30 Acked 1029 Read 8 Sent Done Sent false
167912024-02-12T01:41:44.804Z STATES DS:0 DS:1 DS:2 TOTAL
167922024-02-12T01:41:44.804Z New 0 0 0 0
167932024-02-12T01:41:44.804Z Sent 1 0 27 28
167942024-02-12T01:41:44.804Z Done 29 30 3 62
167952024-02-12T01:41:44.804Z Skipped 0 0 0 0
167962024-02-12T01:41:44.804Z Error 0 0 0 0
167972024-02-12T01:41:44.804ZLast Flush: 1024 1024 0
167982024-02-12T01:41:44.804ZDownstairs last five completed:
167992024-02-12T01:41:44.804ZUpstairs last five completed: 30 29 28 27 26
168002024-02-12T01:41:44.804Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:43.673052774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1743}
168012024-02-12T01:41:44.804ZWrote out file "/tmp/test_repair_verify.data"
168022024-02-12T01:41:44.808Z
168032024-02-12T01:41:44.811ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168042024-02-12T01:41:44.825ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168052024-02-12T01:41:44.825Z 0 000-019 144 144 144 251 251 251 F F F
168062024-02-12T01:41:44.825Z 1 020-039 152 152 147 268 268 259 F F T <---
168072024-02-12T01:41:44.825Z 2 040-059 148 148 148 260 260 260 F F F
168082024-02-12T01:41:44.825Z 3 060-079 151 151 151 267 267 267 F F F
168092024-02-12T01:41:44.825Z 4 080-099 151 151 151 267 267 267 F F F
168102024-02-12T01:41:44.825Z 5 100-119 148 148 148 260 260 260 F F F
168112024-02-12T01:41:44.826Z 6 120-139 151 151 151 267 267 267 F F F
168122024-02-12T01:41:44.826Z 7 140-159 148 148 148 260 260 260 F F F
168132024-02-12T01:41:44.826Z 8 160-179 148 148 148 260 260 260 F F F
168142024-02-12T01:41:44.826Z 9 180-199 150 150 150 264 264 264 F F F
168152024-02-12T01:41:44.826Z 10 200-219 146 146 146 258 258 258 F F F
168162024-02-12T01:41:44.826Z 11 220-239 152 152 146 270 270 258 F F F <---
168172024-02-12T01:41:44.826Z 12 240-259 152 152 140 270 270 244 F F F <---
168182024-02-12T01:41:44.826Z 13 260-279 152 152 148 270 270 260 F F F <---
168192024-02-12T01:41:44.826Z 14 280-299 152 152 148 268 268 260 F F F <---
168202024-02-12T01:41:44.826Z 15 300-319 152 152 138 270 270 242 F F T <---
168212024-02-12T01:41:44.826Z 16 320-339 152 152 150 268 268 261 F F F <---
168222024-02-12T01:41:44.826Z 17 340-359 150 150 150 264 264 264 F F F
168232024-02-12T01:41:44.826Z 18 360-379 148 148 148 260 260 260 F F F
168242024-02-12T01:41:44.826Z 19 380-399 152 152 150 270 270 261 T T F <---
168252024-02-12T01:41:44.826Z 20 400-419 150 150 150 261 261 261 T T F <---
168262024-02-12T01:41:44.829Z 21 420-439 150 150 150 266 266 266 F F F
168272024-02-12T01:41:44.829Z 22 440-459 150 150 150 265 265 265 F F F
168282024-02-12T01:41:44.829Z 23 460-479 150 150 150 265 265 265 F F F
168292024-02-12T01:41:44.830Z 24 480-499 141 141 141 247 247 247 F F F
168302024-02-12T01:41:44.830Z 25 500-519 138 138 138 242 242 242 F F F
168312024-02-12T01:41:44.830Z 26 520-539 150 150 150 266 266 266 F F F
168322024-02-12T01:41:44.830Z 27 540-559 146 146 146 254 254 254 F F F
168332024-02-12T01:41:44.830Z 28 560-579 150 150 150 261 261 261 F F F
168342024-02-12T01:41:44.830Z 29 580-599 140 140 140 246 246 246 F F F
168352024-02-12T01:41:44.830ZMax gen: 152, Max flush: 270
168362024-02-12T01:41:44.830ZError: Difference in extent metadata found!
168372024-02-12T01:41:44.830ZA Difference in extent metadata is expected here
168382024-02-12T01:41:44.830ZOn loop 76
168392024-02-12T01:41:44.830Z
168402024-02-12T01:41:44.833ZVerifying data now
168412024-02-12T01:41:50.313ZNo change after verify
168422024-02-12T01:41:50.313ZLoop: 76 Downstairs dump after verify (and repair):
168432024-02-12T01:41:50.325ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168442024-02-12T01:41:50.325Z 0 000-019 144 144 144 251 251 251 F F F
168452024-02-12T01:41:50.325Z 1 020-039 152 152 152 268 268 268 F F F
168462024-02-12T01:41:50.325Z 2 040-059 148 148 148 260 260 260 F F F
168472024-02-12T01:41:50.325Z 3 060-079 151 151 151 267 267 267 F F F
168482024-02-12T01:41:50.325Z 4 080-099 151 151 151 267 267 267 F F F
168492024-02-12T01:41:50.325Z 5 100-119 148 148 148 260 260 260 F F F
168502024-02-12T01:41:50.325Z 6 120-139 151 151 151 267 267 267 F F F
168512024-02-12T01:41:50.325Z 7 140-159 148 148 148 260 260 260 F F F
168522024-02-12T01:41:50.325Z 8 160-179 148 148 148 260 260 260 F F F
168532024-02-12T01:41:50.325Z 9 180-199 150 150 150 264 264 264 F F F
168542024-02-12T01:41:50.325Z 10 200-219 146 146 146 258 258 258 F F F
168552024-02-12T01:41:50.325Z 11 220-239 152 152 152 270 270 270 F F F
168562024-02-12T01:41:50.325Z 12 240-259 152 152 152 270 270 270 F F F
168572024-02-12T01:41:50.325Z 13 260-279 152 152 152 270 270 270 F F F
168582024-02-12T01:41:50.325Z 14 280-299 152 152 152 268 268 268 F F F
168592024-02-12T01:41:50.329Z 15 300-319 152 152 152 270 270 270 F F F
168602024-02-12T01:41:50.329Z 16 320-339 152 152 152 268 268 268 F F F
168612024-02-12T01:41:50.329Z 17 340-359 150 150 150 264 264 264 F F F
168622024-02-12T01:41:50.329Z 18 360-379 148 148 148 260 260 260 F F F
168632024-02-12T01:41:50.329Z 19 380-399 153 153 153 271 271 271 F F F
168642024-02-12T01:41:50.329Z 20 400-419 153 153 153 271 271 271 F F F
168652024-02-12T01:41:50.329Z 21 420-439 150 150 150 266 266 266 F F F
168662024-02-12T01:41:50.329Z 22 440-459 150 150 150 265 265 265 F F F
168672024-02-12T01:41:50.329Z 23 460-479 150 150 150 265 265 265 F F F
168682024-02-12T01:41:50.329Z 24 480-499 141 141 141 247 247 247 F F F
168692024-02-12T01:41:50.329Z 25 500-519 138 138 138 242 242 242 F F F
168702024-02-12T01:41:50.329Z 26 520-539 150 150 150 266 266 266 F F F
168712024-02-12T01:41:50.329Z 27 540-559 146 146 146 254 254 254 F F F
168722024-02-12T01:41:50.329Z 28 560-579 150 150 150 261 261 261 F F F
168732024-02-12T01:41:50.330Z 29 580-599 140 140 140 246 246 246 F F F
168742024-02-12T01:41:50.330ZMax gen: 153, Max flush: 271
168752024-02-12T01:41:50.330Z
168762024-02-12T01:41:50.330ZBegin loop 77
168772024-02-12T01:41:50.330ZDownstairs to restart: 0
168782024-02-12T01:41:50.349Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.221795329Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
168792024-02-12T01:41:50.354Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\nCrucible runtime is spawned
168802024-02-12T01:41:50.354Z rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222165771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
168812024-02-12T01:41:50.354Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222212759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
168822024-02-12T01:41:50.354Z{"msg":"Crucible d56f47c3-e3e6-445e-a567-66e1cb81f24d has session id: 801c6946-8ea8-4434-8258-83e30a95eaa9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222237388Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168832024-02-12T01:41:50.355Z{"msg":"Upstairs opts: Upstairs UUID: d56f47c3-e3e6-445e-a567-66e1cb81f24d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222297185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168842024-02-12T01:41:50.355Z{"msg":"Crucible stats registered with UUID: d56f47c3-e3e6-445e-a567-66e1cb81f24d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222341413Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168852024-02-12T01:41:50.355Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222381661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
168862024-02-12T01:41:50.355Z{"msg":"d56f47c3-e3e6-445e-a567-66e1cb81f24d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.22240934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168872024-02-12T01:41:50.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222455038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168882024-02-12T01:41:50.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222487116Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168892024-02-12T01:41:50.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222519555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168902024-02-12T01:41:50.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222592351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168912024-02-12T01:41:50.355Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222640739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168922024-02-12T01:41:50.355Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222689236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168932024-02-12T01:41:50.355Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222716365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168942024-02-12T01:41:50.355Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222763743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168952024-02-12T01:41:50.355Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222791551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168962024-02-12T01:41:50.355Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.22282369Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168972024-02-12T01:41:50.355Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222848949Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168982024-02-12T01:41:50.355Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.222877407Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
168992024-02-12T01:41:50.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.223434151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169002024-02-12T01:41:50.355Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.223482198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169012024-02-12T01:41:50.355Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.223508247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169022024-02-12T01:41:50.355Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.223535516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169032024-02-12T01:41:50.355Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224024263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169042024-02-12T01:41:50.355Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224234913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169052024-02-12T01:41:50.356Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224280231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169062024-02-12T01:41:50.356Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224305549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169072024-02-12T01:41:50.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224523189Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169082024-02-12T01:41:50.356Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224593936Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169092024-02-12T01:41:50.356Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224633794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169102024-02-12T01:41:50.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224786016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169112024-02-12T01:41:50.356Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224818265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169122024-02-12T01:41:50.356Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224833154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169132024-02-12T01:41:50.356Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.224865083Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169142024-02-12T01:41:50.356Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225302322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169152024-02-12T01:41:50.356Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.22534201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169162024-02-12T01:41:50.356Z{"msg":"[0]R flush_numbers[0..12]: [251, 268, 260, 267, 267, 260, 267, 260, 260, 264, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225359999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169172024-02-12T01:41:50.356Z{"msg":"[0]R generation[0..12]: [144, 152, 148, 151, 151, 148, 151, 148, 148, 150, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225382418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169182024-02-12T01:41:50.356Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225398247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169192024-02-12T01:41:50.356Z{"msg":"[1]R flush_numbers[0..12]: [251, 268, 260, 267, 267, 260, 267, 260, 260, 264, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225419556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169202024-02-12T01:41:50.356Z{"msg":"[1]R generation[0..12]: [144, 152, 148, 151, 151, 148, 151, 148, 148, 150, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225434015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169212024-02-12T01:41:50.356Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225448145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169222024-02-12T01:41:50.356Z{"msg":"[2]R flush_numbers[0..12]: [251, 268, 260, 267, 267, 260, 267, 260, 260, 264, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225462324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169232024-02-12T01:41:50.368Z{"msg":"[2]R generation[0..12]: [144, 152, 148, 151, 151, 148, 151, 148, 148, 150, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225476043Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169242024-02-12T01:41:50.368ZWait for a query_work_queue command to finish before sending IO
169252024-02-12T01:41:50.369Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225501582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169262024-02-12T01:41:50.369Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225516032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169272024-02-12T01:41:50.369Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225530161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169282024-02-12T01:41:50.369Z{"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.2255435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169292024-02-12T01:41:50.369Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225570899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169302024-02-12T01:41:50.369Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225586668Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169312024-02-12T01:41:50.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225600817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169322024-02-12T01:41:50.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225629006Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169332024-02-12T01:41:50.369Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225644355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169342024-02-12T01:41:50.369Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225658425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169352024-02-12T01:41:50.369Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225678454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169362024-02-12T01:41:50.369Z{"msg":"d56f47c3-e3e6-445e-a567-66e1cb81f24d is now active with session: 801c6946-8ea8-4434-8258-83e30a95eaa9","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225691993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169372024-02-12T01:41:50.369Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225704652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
169382024-02-12T01:41:50.369Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:49.225717712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
169392024-02-12T01:41:50.369ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169402024-02-12T01:41:50.369ZLoading write count information from file "/tmp/test_repair_verify.data"
169412024-02-12T01:41:50.369ZRun Repair workload
169422024-02-12T01:41:50.369Z01/30 Write block 68 len 4096 data: 16
169432024-02-12T01:41:50.369Z02/30 Write block 305 len 28672 data: 11 12 14 15 14 14 13
169442024-02-12T01:41:50.369Z03/30 Write block 589 len 36864 data: 9 12 9 10 7 7 9 9 7
169452024-02-12T01:41:50.369Z04/30 Read block 385 len 4096
169462024-02-12T01:41:50.370Z05/30 Flush
169472024-02-12T01:41:50.370Z06/30 Write block 468 len 8192 data: 12 11
169482024-02-12T01:41:50.370Z07/30 Write block 378 len 8192 data: 14 15
169492024-02-12T01:41:50.370Z08/30 Read block 539 len 8192
169502024-02-12T01:41:50.370Z09/30 Read block 88 len 16384
169512024-02-12T01:41:50.370Z10/30 Write block 474 len 20480 data: 15 13 10 11 10
169522024-02-12T01:41:50.370Z11/30 Read block 91 len 4096
169532024-02-12T01:41:50.370Z12/30 Write block 170 len 40960 data: 14 14 11 11 10 13 14 13 14 15
169542024-02-12T01:41:50.370Z13/30 Write block 174 len 28672 data: 11 14 15 14 15 16 12
169552024-02-12T01:41:50.370Z14/30 Read block 459 len 24576
169562024-02-12T01:41:50.370Z15/30 Read block 308 len 16384
169572024-02-12T01:41:50.370Z16/30 Read block 593 len 12288
169582024-02-12T01:41:50.374Z17/30 Write block 504 len 16384 data: 9 9 8 9
169592024-02-12T01:41:50.374Z18/30 Read block 552 len 24576
169602024-02-12T01:41:50.374Z19/30 Flush
169612024-02-12T01:41:50.374Z20/30 Write block 187 len 28672 data: 12 14 14 12 11 12 11
169622024-02-12T01:41:50.374Z21/30 Read block 192 len 8192
169632024-02-12T01:41:50.374Z22/30 Write block 282 len 32768 data: 15 9 8 9 8 10 13 12
169642024-02-12T01:41:50.377Z23/30 Write block 47 len 28672 data: 15 16 17 16 18 16 12
169652024-02-12T01:41:50.377Z24/30 Read block 536 len 12288
169662024-02-12T01:41:50.378Z25/30 Flush
169672024-02-12T01:41:50.380Z26/30 Write block 190 len 32768 data: 13 12 13 12 10 9 11 12
169682024-02-12T01:41:50.380Z27/30 Read block 440 len 8192
169692024-02-12T01:41:50.384Z28/30 Write block 584 len 40960 data: 12 12 12 9 9 10 13 10 11 8
169702024-02-12T01:41:50.384Z29/30 Write block 25 len 28672 data: 12 12 13 15 17 15 15
169712024-02-12T01:41:50.384Z30/30 Write block 173 len 16384 data: 12 12 15 16
169722024-02-12T01:41:50.384Z----------------------------------------------------------------
169732024-02-12T01:41:50.384Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
169742024-02-12T01:41:50.384ZStates: Active Active Active
169752024-02-12T01:41:50.384ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169762024-02-12T01:41:50.384Z 1 Acked 1000 Write 1 Sent Done Done false
169772024-02-12T01:41:50.384Z 2 Acked 1001 Write 7 Sent Done Done false
169782024-02-12T01:41:50.384Z 3 Acked 1002 Write 9 Sent Done Done false
169792024-02-12T01:41:50.384Z 4 Acked 1003 Read 1 Sent Done Done false
169802024-02-12T01:41:50.384Z 5 Acked 1004 Flush 0 Sent Done Done false
169812024-02-12T01:41:50.384Z 6 Acked 1005 Write 2 Sent Done Done false
169822024-02-12T01:41:50.384Z 7 Acked 1006 Write 2 Sent Done Done false
169832024-02-12T01:41:50.385Z 8 Acked 1007 Read 2 Sent Done Done false
169842024-02-12T01:41:50.385Z 9 Acked 1008 Read 4 Sent Done Done false
169852024-02-12T01:41:50.385Z 10 Acked 1009 Write 5 Sent Done Done false
169862024-02-12T01:41:50.385Z 11 Acked 1010 Read 1 Sent Done Done false
169872024-02-12T01:41:50.385Z 12 Acked 1011 Write 10 Sent Done Done false
169882024-02-12T01:41:50.385Z 13 Acked 1012 Write 7 Sent Done Done false
169892024-02-12T01:41:50.385Z 14 Acked 1013 Read 6 Sent Done Done false
169902024-02-12T01:41:50.385Z 15 Acked 1014 Read 4 Sent Done Done false
169912024-02-12T01:41:50.385Z 16 Acked 1015 Read 3 Sent Done Done false
169922024-02-12T01:41:50.385Z 17 Acked 1016 Write 4 Sent Done Done false
169932024-02-12T01:41:50.385Z 18 Acked 1017 Read 6 Sent Done Done false
169942024-02-12T01:41:50.385Z 19 Acked 1018 Flush 0 Sent Done Done false
169952024-02-12T01:41:50.385Z 20 Acked 1019 Write 7 Sent Done Done false
169962024-02-12T01:41:50.385Z 21 Acked 1020 Read 2 Sent Done Done false
169972024-02-12T01:41:50.385Z 22 Acked 1021 Write 8 Sent Done Done false
169982024-02-12T01:41:50.385Z 23 Acked 1022 Write 7 Sent Done Done false
169992024-02-12T01:41:50.385Z 24 Acked 1023 Read 3 Sent Done Done false
170002024-02-12T01:41:50.385Z 25 Acked 1024 Flush 0 Sent Done Done false
170012024-02-12T01:41:50.385Z 26 Acked 1025 Write 8 Sent Done Done false
170022024-02-12T01:41:50.385Z 27 Acked 1026 Read 2 Sent Done Done false
170032024-02-12T01:41:50.385Z 28 Acked 1027 Write 10 Sent Done Done false
170042024-02-12T01:41:50.385Z 29 Acked 1028 Write 7 Sent Sent Sent false
170052024-02-12T01:41:50.385Z 30 Acked 1029 Write 4 Sent Sent Sent false
170062024-02-12T01:41:50.385Z STATES DS:0 DS:1 DS:2 TOTAL
170072024-02-12T01:41:50.386Z New 0 0 0 0
170082024-02-12T01:41:50.386Z Sent 30 2 2 34
170092024-02-12T01:41:50.386Z Done 0 28 28 56
170102024-02-12T01:41:50.386Z Skipped 0 0 0 0
170112024-02-12T01:41:50.386Z Error 0 0 0 0
170122024-02-12T01:41:50.386ZLast Flush: 0 1024 1024
170132024-02-12T01:41:50.386ZDownstairs last five completed:
170142024-02-12T01:41:50.386ZUpstairs last five completed: 30 29 28 27 26
170152024-02-12T01:41:50.386Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.256182109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751}
170162024-02-12T01:41:50.393ZWrote out file "/tmp/test_repair_verify.data"
170172024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.256866766Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170182024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.256920303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170192024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.256948112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170202024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.256973121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170212024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257010069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170222024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257047227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170232024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257074386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170242024-02-12T01:41:50.393Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257118194Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170252024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257154642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170262024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25719077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170272024-02-12T01:41:50.393Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257227208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170282024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257263947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170292024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257300135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170302024-02-12T01:41:50.393Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257337213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170312024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257362932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170322024-02-12T01:41:50.393Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25740001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170332024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257436079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170342024-02-12T01:41:50.393Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257472327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170352024-02-12T01:41:50.393Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257509195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170362024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257545363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170372024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257580922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170382024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25761672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170392024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257652408Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170402024-02-12T01:41:50.394Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257688317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170412024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257725695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170422024-02-12T01:41:50.394Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257762213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170432024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257799201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170442024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25783551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170452024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257871698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170462024-02-12T01:41:50.394Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257908496Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170472024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257945204Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170482024-02-12T01:41:50.394Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.257981253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170492024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258019031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170502024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258056549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170512024-02-12T01:41:50.394Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258093237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170522024-02-12T01:41:50.394Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258129356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170532024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258164844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170542024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258201112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170552024-02-12T01:41:50.394Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25823715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170562024-02-12T01:41:50.394Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258275258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170572024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258312007Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170582024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258348225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170592024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258384993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170602024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258421622Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170612024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25845782Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170622024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258493688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170632024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258529917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170642024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258565635Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170652024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258602603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170662024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258640371Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170672024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25867651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170682024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258712988Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170692024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258749996Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170702024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258786084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170712024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258822163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170722024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258858581Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170732024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258894739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170742024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258930627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170752024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.258967336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170762024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259003614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170772024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259039872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170782024-02-12T01:41:50.395Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25907655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170792024-02-12T01:41:50.395Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259115329Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170802024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259149897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170812024-02-12T01:41:50.395Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259183335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170822024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259212774Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170832024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259227003Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170842024-02-12T01:41:50.396Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259247442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170852024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259259732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170862024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259279561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170872024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25929276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170882024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259311999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170892024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259324639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170902024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259343808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170912024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259357067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170922024-02-12T01:41:50.396Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259376946Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170932024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259389765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170942024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259408844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170952024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259421874Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170962024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259440853Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170972024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259453882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170982024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259472761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
170992024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259485681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171002024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25950477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171012024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259517039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171022024-02-12T01:41:50.396Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259535878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171032024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259548838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171042024-02-12T01:41:50.396Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259568557Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171052024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259581076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171062024-02-12T01:41:50.396Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259600575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171072024-02-12T01:41:50.397Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259613195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171082024-02-12T01:41:50.397Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259633084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171092024-02-12T01:41:50.397Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259645263Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171102024-02-12T01:41:50.397Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259664402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171112024-02-12T01:41:50.397Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259677272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171122024-02-12T01:41:50.397Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259696171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171132024-02-12T01:41:50.397Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25970918Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171142024-02-12T01:41:50.397Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259728399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171152024-02-12T01:41:50.397Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259741459Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171162024-02-12T01:41:50.401Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259760958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171172024-02-12T01:41:50.401Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259773917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171182024-02-12T01:41:50.401Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259793406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171192024-02-12T01:41:50.401Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259806876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171202024-02-12T01:41:50.401Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259825765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171212024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259838694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171222024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259858193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171232024-02-12T01:41:50.402Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259870493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171242024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259890592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171252024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259902961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171262024-02-12T01:41:50.402Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.25992241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171272024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259934809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171282024-02-12T01:41:50.402Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259954289Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171292024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259966678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171302024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259985437Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171312024-02-12T01:41:50.402Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.259998626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171322024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260011416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171332024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260030525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171342024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260051094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171352024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260071773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171362024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260084492Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171372024-02-12T01:41:50.402Z{"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260104081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171382024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260117061Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171392024-02-12T01:41:50.402Z{"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.26013649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171402024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260148829Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171412024-02-12T01:41:50.402Z{"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:49.260167668Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1751,"session_id":"801c6946-8ea8-4434-8258-83e30a95eaa9"}
171422024-02-12T01:41:50.402Z
171432024-02-12T01:41:50.403ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171442024-02-12T01:41:50.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171452024-02-12T01:41:50.410Z 0 000-019 144 144 144 251 251 251 F F F
171462024-02-12T01:41:50.410Z 1 020-039 152 152 152 268 268 268 F T T <---
171472024-02-12T01:41:50.410Z 2 040-059 148 154 154 260 274 274 F F F <---
171482024-02-12T01:41:50.410Z 3 060-079 151 154 154 267 272 272 F F F <---
171492024-02-12T01:41:50.410Z 4 080-099 151 151 151 267 267 267 F F F
171502024-02-12T01:41:50.410Z 5 100-119 148 148 148 260 260 260 F F F
171512024-02-12T01:41:50.410Z 6 120-139 151 151 151 267 267 267 F F F
171522024-02-12T01:41:50.410Z 7 140-159 148 148 148 260 260 260 F F F
171532024-02-12T01:41:50.414Z 8 160-179 148 154 154 260 273 273 F T T <---
171542024-02-12T01:41:50.414Z 9 180-199 150 154 154 264 274 274 F T T <---
171552024-02-12T01:41:50.414Z 10 200-219 146 146 146 258 258 258 F F F
171562024-02-12T01:41:50.414Z 11 220-239 152 152 152 270 270 270 F F F
171572024-02-12T01:41:50.414Z 12 240-259 152 152 152 270 270 270 F F F
171582024-02-12T01:41:50.414Z 13 260-279 152 152 152 270 270 270 F F F
171592024-02-12T01:41:50.414Z 14 280-299 152 154 154 268 274 274 F F F <---
171602024-02-12T01:41:50.414Z 15 300-319 152 154 154 270 272 272 F F F <---
171612024-02-12T01:41:50.414Z 16 320-339 152 152 152 268 268 268 F F F
171622024-02-12T01:41:50.414Z 17 340-359 150 150 150 264 264 264 F F F
171632024-02-12T01:41:50.414Z 18 360-379 148 154 154 260 273 273 F F F <---
171642024-02-12T01:41:50.414Z 19 380-399 153 153 153 271 271 271 F F F
171652024-02-12T01:41:50.414Z 20 400-419 153 153 153 271 271 271 F F F
171662024-02-12T01:41:50.414Z 21 420-439 150 150 150 266 266 266 F F F
171672024-02-12T01:41:50.414Z 22 440-459 150 150 150 265 265 265 F F F
171682024-02-12T01:41:50.414Z 23 460-479 150 154 154 265 273 273 F F F <---
171692024-02-12T01:41:50.414Z 24 480-499 141 141 141 247 247 247 F F F
171702024-02-12T01:41:50.414Z 25 500-519 138 154 154 242 273 273 F F F <---
171712024-02-12T01:41:50.414Z 26 520-539 150 150 150 266 266 266 F F F
171722024-02-12T01:41:50.414Z 27 540-559 146 146 146 254 254 254 F F F
171732024-02-12T01:41:50.415Z 28 560-579 150 150 150 261 261 261 F F F
171742024-02-12T01:41:50.415Z 29 580-599 140 154 154 246 272 272 F T T <---
171752024-02-12T01:41:50.415ZMax gen: 154, Max flush: 274
171762024-02-12T01:41:50.415ZError: Difference in extent metadata found!
171772024-02-12T01:41:50.415ZA Difference in extent metadata is expected here
171782024-02-12T01:41:50.415ZOn loop 77
171792024-02-12T01:41:50.415Z
171802024-02-12T01:41:50.418ZVerifying data now
171812024-02-12T01:41:54.963ZNo change after verify
171822024-02-12T01:41:54.964ZLoop: 77 Downstairs dump after verify (and repair):
171832024-02-12T01:41:54.981ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171842024-02-12T01:41:54.981Z 0 000-019 144 144 144 251 251 251 F F F
171852024-02-12T01:41:54.981Z 1 020-039 155 155 155 275 275 275 F F F
171862024-02-12T01:41:54.981Z 2 040-059 154 154 154 274 274 274 F F F
171872024-02-12T01:41:54.981Z 3 060-079 154 154 154 272 272 272 F F F
171882024-02-12T01:41:54.981Z 4 080-099 151 151 151 267 267 267 F F F
171892024-02-12T01:41:54.982Z 5 100-119 148 148 148 260 260 260 F F F
171902024-02-12T01:41:54.982Z 6 120-139 151 151 151 267 267 267 F F F
171912024-02-12T01:41:54.982Z 7 140-159 148 148 148 260 260 260 F F F
171922024-02-12T01:41:54.982Z 8 160-179 155 155 155 275 275 275 F F F
171932024-02-12T01:41:54.985Z 9 180-199 155 155 155 275 275 275 F F F
171942024-02-12T01:41:54.985Z 10 200-219 146 146 146 258 258 258 F F F
171952024-02-12T01:41:54.985Z 11 220-239 152 152 152 270 270 270 F F F
171962024-02-12T01:41:54.985Z 12 240-259 152 152 152 270 270 270 F F F
171972024-02-12T01:41:54.985Z 13 260-279 152 152 152 270 270 270 F F F
171982024-02-12T01:41:54.985Z 14 280-299 154 154 154 274 274 274 F F F
171992024-02-12T01:41:54.985Z 15 300-319 154 154 154 272 272 272 F F F
172002024-02-12T01:41:54.985Z 16 320-339 152 152 152 268 268 268 F F F
172012024-02-12T01:41:54.985Z 17 340-359 150 150 150 264 264 264 F F F
172022024-02-12T01:41:54.986Z 18 360-379 154 154 154 273 273 273 F F F
172032024-02-12T01:41:54.986Z 19 380-399 153 153 153 271 271 271 F F F
172042024-02-12T01:41:54.986Z 20 400-419 153 153 153 271 271 271 F F F
172052024-02-12T01:41:54.986Z 21 420-439 150 150 150 266 266 266 F F F
172062024-02-12T01:41:54.986Z 22 440-459 150 150 150 265 265 265 F F F
172072024-02-12T01:41:54.986Z 23 460-479 154 154 154 273 273 273 F F F
172082024-02-12T01:41:54.986Z 24 480-499 141 141 141 247 247 247 F F F
172092024-02-12T01:41:54.986Z 25 500-519 154 154 154 273 273 273 F F F
172102024-02-12T01:41:54.986Z 26 520-539 150 150 150 266 266 266 F F F
172112024-02-12T01:41:54.986Z 27 540-559 146 146 146 254 254 254 F F F
172122024-02-12T01:41:54.986Z 28 560-579 150 150 150 261 261 261 F F F
172132024-02-12T01:41:54.986Z 29 580-599 155 155 155 275 275 275 F F F
172142024-02-12T01:41:54.986ZMax gen: 155, Max flush: 275
172152024-02-12T01:41:54.986Z
172162024-02-12T01:41:54.986ZBegin loop 78
172172024-02-12T01:41:54.986ZDownstairs to restart: 2
172182024-02-12T01:41:55.001Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873129888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
172192024-02-12T01:41:55.001Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\nCrucible runtime is spawned
172202024-02-12T01:41:55.006Z git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873420364Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
172212024-02-12T01:41:55.006Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873484021Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
172222024-02-12T01:41:55.006Z{"msg":"Crucible 51803dd0-9ce2-48d9-9e29-7911f6bf8803 has session id: aab60928-058a-44dc-8667-2d8c165bd128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.87352009Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172232024-02-12T01:41:55.006Z{"msg":"Upstairs opts: Upstairs UUID: 51803dd0-9ce2-48d9-9e29-7911f6bf8803, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873544488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172242024-02-12T01:41:55.007Z{"msg":"Crucible stats registered with UUID: 51803dd0-9ce2-48d9-9e29-7911f6bf8803","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873567177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172252024-02-12T01:41:55.007Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873580787Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
172262024-02-12T01:41:55.007Z{"msg":"51803dd0-9ce2-48d9-9e29-7911f6bf8803 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873673502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172272024-02-12T01:41:55.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873699991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172282024-02-12T01:41:55.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.87371556Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172292024-02-12T01:41:55.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.87373001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172302024-02-12T01:41:55.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873754298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172312024-02-12T01:41:55.007Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873778367Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172322024-02-12T01:41:55.007Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873802576Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172332024-02-12T01:41:55.007Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873833505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172342024-02-12T01:41:55.007Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873870063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172352024-02-12T01:41:55.007Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.873897392Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172362024-02-12T01:41:55.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874014196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172372024-02-12T01:41:55.007Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874064734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172382024-02-12T01:41:55.007Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:53.874099612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172392024-02-12T01:41:55.007Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:53.87412902Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172402024-02-12T01:41:55.007Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:53.874283583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172412024-02-12T01:41:55.007Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874316232Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172422024-02-12T01:41:55.007Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874331671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172432024-02-12T01:41:55.007Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874517742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172442024-02-12T01:41:55.007Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874951001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172452024-02-12T01:41:55.007Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.8749789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172462024-02-12T01:41:55.008Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.874995039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172472024-02-12T01:41:55.008Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.875009609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172482024-02-12T01:41:55.008Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.875571322Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172492024-02-12T01:41:55.008Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.87561317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172502024-02-12T01:41:55.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.876210281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172512024-02-12T01:41:55.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.87624761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172522024-02-12T01:41:55.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.876263539Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172532024-02-12T01:41:55.008Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.876284648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172542024-02-12T01:41:55.008Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.876298857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172552024-02-12T01:41:55.008Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:53.876320026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172562024-02-12T01:41:56.004Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.87584861Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172572024-02-12T01:41:56.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.875917256Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172582024-02-12T01:41:56.009Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.876814014Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172592024-02-12T01:41:56.009Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.876852402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172602024-02-12T01:41:56.010Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.877391956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172612024-02-12T01:41:56.010Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.877901952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172622024-02-12T01:41:56.010Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.87793198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172632024-02-12T01:41:56.010Z{"msg":"[0]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 260, 267, 260, 275, 275, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.877949629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172642024-02-12T01:41:56.010Z{"msg":"[0]R generation[0..12]: [144, 155, 154, 154, 151, 148, 151, 148, 155, 155, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.877964699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172652024-02-12T01:41:56.010Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.877984328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172662024-02-12T01:41:56.010Z{"msg":"[1]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 260, 267, 260, 275, 275, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878025206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172672024-02-12T01:41:56.010Z{"msg":"[1]R generation[0..12]: [144, 155, 154, 154, 151, 148, 151, 148, 155, 155, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878041405Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172682024-02-12T01:41:56.010Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878056254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172692024-02-12T01:41:56.010Z{"msg":"[2]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 260, 267, 260, 275, 275, 258, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878070474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172702024-02-12T01:41:56.010Z{"msg":"[2]R generation[0..12]: [144, 155, 154, 154, 151, 148, 151, 148, 155, 155, 146, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878093042Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172712024-02-12T01:41:56.010Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878108282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172722024-02-12T01:41:56.010Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
172732024-02-12T01:41:56.010Z,"time":"2024-02-12T01:41:54.878121741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172742024-02-12T01:41:56.010Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.87813751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172752024-02-12T01:41:56.010Z{"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.87815126Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172762024-02-12T01:41:56.010Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878172069Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172772024-02-12T01:41:56.010Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878186068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172782024-02-12T01:41:56.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878198208Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172792024-02-12T01:41:56.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878218267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172802024-02-12T01:41:56.010Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878231786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172812024-02-12T01:41:56.010Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878245265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172822024-02-12T01:41:56.010Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878264694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172832024-02-12T01:41:56.010Z{"msg":"51803dd0-9ce2-48d9-9e29-7911f6bf8803 is now active with session: aab60928-058a-44dc-8667-2d8c165bd128","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878277944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172842024-02-12T01:41:56.011Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878290093Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759,"session_id":"aab60928-058a-44dc-8667-2d8c165bd128"}
172852024-02-12T01:41:56.011Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:41:54.878308832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
172862024-02-12T01:41:56.011ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172872024-02-12T01:41:56.011ZLoading write count information from file "/tmp/test_repair_verify.data"
172882024-02-12T01:41:56.011ZRun Repair workload
172892024-02-12T01:41:56.011Z01/30 Write block 411 len 4096 data: 14
172902024-02-12T01:41:56.015Z02/30 Read block 191 len 28672
172912024-02-12T01:41:56.015Z03/30 Read block 198 len 8192
172922024-02-12T01:41:56.015Z04/30 Read block 411 len 4096
172932024-02-12T01:41:56.015Z05/30 Write block 156 len 20480 data: 7 8 9 10 13
172942024-02-12T01:41:56.015Z06/30 Write block 152 len 8192 data: 13 10
172952024-02-12T01:41:56.015Z07/30 Write block 544 len 32768 data: 7 8 11 11 11 13 17 16
172962024-02-12T01:41:56.015Z08/30 Write block 139 len 24576 data: 15 16 18 20 18 18
172972024-02-12T01:41:56.015Z09/30 Read block 393 len 28672
172982024-02-12T01:41:56.021Z10/30 Write block 556 len 32768 data: 8 7 7 8 7 10 9 12
172992024-02-12T01:41:56.021Z11/30 Write block 181 len 36864 data: 14 13 14 15 12 12 13 15 15
173002024-02-12T01:41:56.021Z12/30 Write block 198 len 36864 data: 10 14 13 11 13 14 12 11 11
173012024-02-12T01:41:56.021Z13/30 Read block 414 len 12288
173022024-02-12T01:41:56.024Z14/30 Read block 16 len 4096
173032024-02-12T01:41:56.024Z15/30 Read block 280 len 4096
173042024-02-12T01:41:56.024Z16/30 Write block 523 len 20480 data: 13 12 8 8 7
173052024-02-12T01:41:56.024Z17/30 Write block 463 len 8192 data: 11 10
173062024-02-12T01:41:56.024Z18/30 Read block 405 len 4096
173072024-02-12T01:41:56.024Z19/30 Read block 92 len 28672
173082024-02-12T01:41:56.025Z20/30 Flush
173092024-02-12T01:41:56.028Z21/30 Write block 365 len 32768 data: 14 13 18 17 17 16 13 16
173102024-02-12T01:41:56.031Z22/30 Read block 553 len 16384
173112024-02-12T01:41:56.031Z23/30 Read block 385 len 36864
173122024-02-12T01:41:56.031Z24/30 Flush
173132024-02-12T01:41:56.034Z25/30 Write block 264 len 16384 data: 9 12 14 15
173142024-02-12T01:41:56.034Z26/30 Read block 164 len 40960
173152024-02-12T01:41:56.037Z27/30 Write block 186 len 8192 data: 13 14
173162024-02-12T01:41:56.037Z28/30 Write block 104 len 24576 data: 12 15 13 14 14 13
173172024-02-12T01:41:56.037Z29/30 Read block 153 len 8192
173182024-02-12T01:41:56.037Z30/30 Read block 527 len 8192
173192024-02-12T01:41:56.040Z----------------------------------------------------------------
173202024-02-12T01:41:56.040Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
173212024-02-12T01:41:56.040ZStates: Active Active Active
173222024-02-12T01:41:56.040ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
173232024-02-12T01:41:56.040Z 1 Acked 1000 Write 1 Done Done Done false
173242024-02-12T01:41:56.040Z 2 Acked 1001 Read 7 Done Done Done false
173252024-02-12T01:41:56.040Z 3 Acked 1002 Read 2 Done Done Done false
173262024-02-12T01:41:56.040Z 4 Acked 1003 Read 1 Done Done Sent false
173272024-02-12T01:41:56.040Z 5 Acked 1004 Write 5 Done Done Sent false
173282024-02-12T01:41:56.040Z 6 Acked 1005 Write 2 Done Done Sent false
173292024-02-12T01:41:56.041Z 7 Acked 1006 Write 8 Done Done Sent false
173302024-02-12T01:41:56.041Z 8 Acked 1007 Write 6 Done Done Sent false
173312024-02-12T01:41:56.041Z 9 Acked 1008 Read 7 Done Done Sent false
173322024-02-12T01:41:56.041Z 10 Acked 1009 Write 8 Done Done Sent false
173332024-02-12T01:41:56.041Z 11 Acked 1010 Write 9 Done Done Sent false
173342024-02-12T01:41:56.041Z 12 Acked 1011 Write 9 Done Done Sent false
173352024-02-12T01:41:56.041Z 13 Acked 1012 Read 3 Done Done Sent false
173362024-02-12T01:41:56.041Z 14 Acked 1013 Read 1 Done Done Sent false
173372024-02-12T01:41:56.041Z 15 Acked 1014 Read 1 Done Done Sent false
173382024-02-12T01:41:56.041Z 16 Acked 1015 Write 5 Done Done Sent false
173392024-02-12T01:41:56.041Z 17 Acked 1016 Write 2 Done Done Sent false
173402024-02-12T01:41:56.041Z 18 Acked 1017 Read 1 Done Done Sent false
173412024-02-12T01:41:56.041Z 19 Acked 1018 Read 7 Done Done Sent false
173422024-02-12T01:41:56.041Z 20 Acked 1019 Flush 0 Done Done Sent false
173432024-02-12T01:41:56.041Z 21 Acked 1020 Write 8 Done Done Sent false
173442024-02-12T01:41:56.041Z 22 Acked 1021 Read 4 Done Done Sent false
173452024-02-12T01:41:56.041Z 23 Acked 1022 Read 9 Done Done Sent false
173462024-02-12T01:41:56.041Z 24 Acked 1023 Flush 0 Done Done Sent false
173472024-02-12T01:41:56.041Z 25 Acked 1024 Write 4 Done Done Sent false
173482024-02-12T01:41:56.041Z 26 Acked 1025 Read 10 Done Done Sent false
173492024-02-12T01:41:56.041Z 27 Acked 1026 Write 2 Done Done Sent false
173502024-02-12T01:41:56.041Z 28 Acked 1027 Write 6 Done Done Sent false
173512024-02-12T01:41:56.041Z 29 Acked 1028 Read 2 Done Done Sent false
173522024-02-12T01:41:56.041Z 30 Acked 1029 Read 2 Done Done Sent false
173532024-02-12T01:41:56.042Z STATES DS:0 DS:1 DS:2 TOTAL
173542024-02-12T01:41:56.042Z New 0 0 0 0
173552024-02-12T01:41:56.042Z Sent 0 0 27 27
173562024-02-12T01:41:56.042Z Done 30 30 3 63
173572024-02-12T01:41:56.042Z Skipped 0 0 0 0
173582024-02-12T01:41:56.042Z Error 0 0 0 0
173592024-02-12T01:41:56.042ZLast Flush: 1023 1023 0
173602024-02-12T01:41:56.042ZDownstairs last five completed:
173612024-02-12T01:41:56.042ZUpstairs last five completed: 30 29 28 27 26
173622024-02-12T01:41:56.042Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:41:54.909783361Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1759}
173632024-02-12T01:41:56.042ZWrote out file "/tmp/test_repair_verify.data"
173642024-02-12T01:41:56.045Z
173652024-02-12T01:41:56.048ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173662024-02-12T01:41:56.063ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173672024-02-12T01:41:56.063Z 0 000-019 144 144 144 251 251 251 F F F
173682024-02-12T01:41:56.063Z 1 020-039 155 155 155 275 275 275 F F F
173692024-02-12T01:41:56.063Z 2 040-059 154 154 154 274 274 274 F F F
173702024-02-12T01:41:56.063Z 3 060-079 154 154 154 272 272 272 F F F
173712024-02-12T01:41:56.063Z 4 080-099 151 151 151 267 267 267 F F F
173722024-02-12T01:41:56.063Z 5 100-119 148 148 148 260 260 260 T T F <---
173732024-02-12T01:41:56.063Z 6 120-139 156 156 151 276 276 267 F F F <---
173742024-02-12T01:41:56.063Z 7 140-159 156 156 148 276 276 260 F F F <---
173752024-02-12T01:41:56.063Z 8 160-179 156 156 155 276 276 275 F F F <---
173762024-02-12T01:41:56.067Z 9 180-199 156 156 155 276 276 275 T T F <---
173772024-02-12T01:41:56.067Z 10 200-219 156 156 146 276 276 258 F F F <---
173782024-02-12T01:41:56.067Z 11 220-239 152 152 152 270 270 270 F F F
173792024-02-12T01:41:56.067Z 12 240-259 152 152 152 270 270 270 F F F
173802024-02-12T01:41:56.067Z 13 260-279 152 152 152 270 270 270 T T F <---
173812024-02-12T01:41:56.067Z 14 280-299 154 154 154 274 274 274 F F F
173822024-02-12T01:41:56.067Z 15 300-319 154 154 154 272 272 272 F F F
173832024-02-12T01:41:56.067Z 16 320-339 152 152 152 268 268 268 F F F
173842024-02-12T01:41:56.067Z 17 340-359 150 150 150 264 264 264 F F F
173852024-02-12T01:41:56.067Z 18 360-379 156 156 154 277 277 273 F F F <---
173862024-02-12T01:41:56.067Z 19 380-399 153 153 153 271 271 271 F F F
173872024-02-12T01:41:56.067Z 20 400-419 156 156 153 276 276 271 F F T <---
173882024-02-12T01:41:56.067Z 21 420-439 150 150 150 266 266 266 F F F
173892024-02-12T01:41:56.067Z 22 440-459 150 150 150 265 265 265 F F F
173902024-02-12T01:41:56.067Z 23 460-479 156 156 154 276 276 273 F F F <---
173912024-02-12T01:41:56.067Z 24 480-499 141 141 141 247 247 247 F F F
173922024-02-12T01:41:56.067Z 25 500-519 154 154 154 273 273 273 F F F
173932024-02-12T01:41:56.067Z 26 520-539 156 156 150 276 276 266 F F F <---
173942024-02-12T01:41:56.067Z 27 540-559 156 156 146 276 276 254 F F F <---
173952024-02-12T01:41:56.067Z 28 560-579 156 156 150 276 276 261 F F F <---
173962024-02-12T01:41:56.067Z 29 580-599 155 155 155 275 275 275 F F F
173972024-02-12T01:41:56.067ZMax gen: 156, Max flush: 277
173982024-02-12T01:41:56.068ZError: Difference in extent metadata found!
173992024-02-12T01:41:56.068ZA Difference in extent metadata is expected here
174002024-02-12T01:41:56.068ZOn loop 78
174012024-02-12T01:41:56.068Z
174022024-02-12T01:41:56.071ZVerifying data now
174032024-02-12T01:42:01.713ZNo change after verify
174042024-02-12T01:42:01.713ZLoop: 78 Downstairs dump after verify (and repair):
174052024-02-12T01:42:01.729ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
174062024-02-12T01:42:01.729Z 0 000-019 144 144 144 251 251 251 F F F
174072024-02-12T01:42:01.729Z 1 020-039 155 155 155 275 275 275 F F F
174082024-02-12T01:42:01.729Z 2 040-059 154 154 154 274 274 274 F F F
174092024-02-12T01:42:01.729Z 3 060-079 154 154 154 272 272 272 F F F
174102024-02-12T01:42:01.729Z 4 080-099 151 151 151 267 267 267 F F F
174112024-02-12T01:42:01.729Z 5 100-119 157 157 157 278 278 278 F F F
174122024-02-12T01:42:01.729Z 6 120-139 156 156 156 276 276 276 F F F
174132024-02-12T01:42:01.729Z 7 140-159 156 156 156 276 276 276 F F F
174142024-02-12T01:42:01.729Z 8 160-179 156 156 156 276 276 276 F F F
174152024-02-12T01:42:01.729Z 9 180-199 157 157 157 278 278 278 F F F
174162024-02-12T01:42:01.729Z 10 200-219 156 156 156 276 276 276 F F F
174172024-02-12T01:42:01.729Z 11 220-239 152 152 152 270 270 270 F F F
174182024-02-12T01:42:01.729Z 12 240-259 152 152 152 270 270 270 F F F
174192024-02-12T01:42:01.729Z 13 260-279 157 157 157 278 278 278 F F F
174202024-02-12T01:42:01.735Z 14 280-299 154 154 154 274 274 274 F F F
174212024-02-12T01:42:01.735Z 15 300-319 154 154 154 272 272 272 F F F
174222024-02-12T01:42:01.735Z 16 320-339 152 152 152 268 268 268 F F F
174232024-02-12T01:42:01.735Z 17 340-359 150 150 150 264 264 264 F F F
174242024-02-12T01:42:01.735Z 18 360-379 156 156 156 277 277 277 F F F
174252024-02-12T01:42:01.735Z 19 380-399 153 153 153 271 271 271 F F F
174262024-02-12T01:42:01.735Z 20 400-419 156 156 156 276 276 276 F F F
174272024-02-12T01:42:01.735Z 21 420-439 150 150 150 266 266 266 F F F
174282024-02-12T01:42:01.735Z 22 440-459 150 150 150 265 265 265 F F F
174292024-02-12T01:42:01.735Z 23 460-479 156 156 156 276 276 276 F F F
174302024-02-12T01:42:01.735Z 24 480-499 141 141 141 247 247 247 F F F
174312024-02-12T01:42:01.735Z 25 500-519 154 154 154 273 273 273 F F F
174322024-02-12T01:42:01.735Z 26 520-539 156 156 156 276 276 276 F F F
174332024-02-12T01:42:01.735Z 27 540-559 156 156 156 276 276 276 F F F
174342024-02-12T01:42:01.735Z 28 560-579 156 156 156 276 276 276 F F F
174352024-02-12T01:42:01.735Z 29 580-599 155 155 155 275 275 275 F F F
174362024-02-12T01:42:01.735ZMax gen: 157, Max flush: 278
174372024-02-12T01:42:01.735Z
174382024-02-12T01:42:01.735ZBegin loop 79
174392024-02-12T01:42:01.735ZDownstairs to restart: 2
174402024-02-12T01:42:01.753Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624104625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
174412024-02-12T01:42:01.753Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624366692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
174422024-02-12T01:42:01.758Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624391891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
174432024-02-12T01:42:01.758Z{"msg":"Crucible 9b79494c-6e6f-4adf-bfd0-c76e074264d2 has session id: cb377d1f-e5e9-4624-bca2-1b02623caf00",Crucible runtime is spawned
174442024-02-12T01:42:01.758Z"v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624468967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174452024-02-12T01:42:01.758Z{"msg":"Upstairs opts: Upstairs UUID: 9b79494c-6e6f-4adf-bfd0-c76e074264d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624526415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174462024-02-12T01:42:01.758Z{"msg":"Crucible stats registered with UUID: 9b79494c-6e6f-4adf-bfd0-c76e074264d2","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624568822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174472024-02-12T01:42:01.759Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624609471Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
174482024-02-12T01:42:01.759Z{"msg":"9b79494c-6e6f-4adf-bfd0-c76e074264d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624636429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174492024-02-12T01:42:01.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624682947Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174502024-02-12T01:42:01.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624715115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174512024-02-12T01:42:01.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624745094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174522024-02-12T01:42:01.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624782362Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174532024-02-12T01:42:01.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.62482235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174542024-02-12T01:42:01.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624871858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174552024-02-12T01:42:01.759Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624932715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174562024-02-12T01:42:01.759Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.624969403Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174572024-02-12T01:42:01.759Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625000452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174582024-02-12T01:42:01.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.62504767Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174592024-02-12T01:42:01.759Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:00.625107087Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174602024-02-12T01:42:01.759Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:00.625161554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174612024-02-12T01:42:01.759Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:00.625213892Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174622024-02-12T01:42:01.759Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625260869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174632024-02-12T01:42:01.759Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625289928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174642024-02-12T01:42:01.759Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625320587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174652024-02-12T01:42:01.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625357835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174662024-02-12T01:42:01.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625658801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174672024-02-12T01:42:01.759Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625734407Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174682024-02-12T01:42:01.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625785184Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174692024-02-12T01:42:01.759Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.625824743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174702024-02-12T01:42:01.760Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626253932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174712024-02-12T01:42:01.760Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626413445Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174722024-02-12T01:42:01.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.62671239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174732024-02-12T01:42:01.760Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626767308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174742024-02-12T01:42:01.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626817175Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174752024-02-12T01:42:01.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626853644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174762024-02-12T01:42:01.760Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626883072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174772024-02-12T01:42:01.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:00.626905791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174782024-02-12T01:42:02.756Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.626494312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174792024-02-12T01:42:02.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.626577238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174802024-02-12T01:42:02.762Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.627469385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174812024-02-12T01:42:02.762Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.627516053Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174822024-02-12T01:42:02.762Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628017339Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174832024-02-12T01:42:02.762Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628578952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174842024-02-12T01:42:02.762Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.62862002Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174852024-02-12T01:42:02.763Z{"msg":"[0]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 278, 276, 276, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628649689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174862024-02-12T01:42:02.763Z{"msg":"[0]R generation[0..12]: [144, 155, 154, 154, 151, 157, 156, 156, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628687077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174872024-02-12T01:42:02.763Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628707016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174882024-02-12T01:42:02.763Z{"msg":"[1]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 278, 276, 276, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628730905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174892024-02-12T01:42:02.763Z{"msg":"[1]R generation[0..12]: [144, 155, 154, 154, 151, 157, 156, 156, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628746704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174902024-02-12T01:42:02.763Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628762273Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174912024-02-12T01:42:02.763Z{"msg":"Wait for a query_work_queue command to finish before sending IO
174922024-02-12T01:42:02.763Z[2]R flush_numbers[0..12]: [251, 275, 274, 272, 267, 278, 276, 276, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628785982Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174932024-02-12T01:42:02.763Z{"msg":"[2]R generation[0..12]: [144, 155, 154, 154, 151, 157, 156, 156, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628801442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174942024-02-12T01:42:02.763Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.62882303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174952024-02-12T01:42:02.763Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.62883785Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174962024-02-12T01:42:02.763Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628873378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174972024-02-12T01:42:02.763Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628907037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174982024-02-12T01:42:02.763Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628932415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
174992024-02-12T01:42:02.763Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628956924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175002024-02-12T01:42:02.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.628977013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175012024-02-12T01:42:02.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629009842Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175022024-02-12T01:42:02.763Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629030611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175032024-02-12T01:42:02.763Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629063979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175042024-02-12T01:42:02.763Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629088508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175052024-02-12T01:42:02.763Z{"msg":"9b79494c-6e6f-4adf-bfd0-c76e074264d2 is now active with session: cb377d1f-e5e9-4624-bca2-1b02623caf00","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629123816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175062024-02-12T01:42:02.763Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
175072024-02-12T01:42:02.763Z,"time":"2024-02-12T01:42:01.629157795Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767,"session_id":"cb377d1f-e5e9-4624-bca2-1b02623caf00"}
175082024-02-12T01:42:02.763Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:01.629182724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
175092024-02-12T01:42:02.763ZLoading write count information from file "/tmp/test_repair_verify.data"
175102024-02-12T01:42:02.764ZRun Repair workload
175112024-02-12T01:42:02.764Z01/30 Write block 337 len 32768 data: 8 7 9 10 7 7 10 8
175122024-02-12T01:42:02.767Z02/30 Read block 426 len 20480
175132024-02-12T01:42:02.767Z03/30 Read block 355 len 8192
175142024-02-12T01:42:02.767Z04/30 Read block 293 len 36864
175152024-02-12T01:42:02.767Z05/30 Read block 149 len 16384
175162024-02-12T01:42:02.767Z06/30 Write block 291 len 28672 data: 15 13 14 11 12 10 10
175172024-02-12T01:42:02.767Z07/30 Read block 302 len 20480
175182024-02-12T01:42:02.767Z08/30 Write block 332 len 32768 data: 9 7 7 7 7 9 8 10
175192024-02-12T01:42:02.767Z09/30 Read block 318 len 32768
175202024-02-12T01:42:02.771Z10/30 Flush
175212024-02-12T01:42:02.774Z11/30 Write block 489 len 40960 data: 10 12 10 10 10 10 14 12 12 13
175222024-02-12T01:42:02.774Z12/30 Flush
175232024-02-12T01:42:02.777Z13/30 Write block 350 len 20480 data: 13 15 17 13 9
175242024-02-12T01:42:02.778Z14/30 Read block 255 len 4096
175252024-02-12T01:42:02.778Z15/30 Read block 559 len 8192
175262024-02-12T01:42:02.778Z16/30 Read block 211 len 40960
175272024-02-12T01:42:02.782Z17/30 Read block 73 len 40960
175282024-02-12T01:42:02.782Z18/30 Read block 209 len 20480
175292024-02-12T01:42:02.782Z19/30 Read block 380 len 4096
175302024-02-12T01:42:02.782Z20/30 Write block 555 len 4096 data: 9
175312024-02-12T01:42:02.783Z21/30 Write block 63 len 16384 data: 15 15 17 16
175322024-02-12T01:42:02.783Z22/30 Read block 164 len 16384
175332024-02-12T01:42:02.786Z23/30 Write block 489 len 40960 data: 11 13 11 11 11 11 15 13 13 14
175342024-02-12T01:42:02.786Z24/30 Write block 151 len 32768 data: 15 14 11 11 8 8 9 10
175352024-02-12T01:42:02.786Z25/30 Write block 319 len 36864 data: 10 10 10 12 10 11 10 12 11
175362024-02-12T01:42:02.786Z26/30 Write block 131 len 24576 data: 15 14 12 12 10 13
175372024-02-12T01:42:02.790Z27/30 Write block 455 len 8192 data: 9 9
175382024-02-12T01:42:02.790Z28/30 Write block 54 len 20480 data: 12 12 12 14 17
175392024-02-12T01:42:02.790Z29/30 Write block 367 len 20480 data: 19 18 18 17 14
175402024-02-12T01:42:02.790Z30/30 Read block 568 len 20480
175412024-02-12T01:42:02.790Z----------------------------------------------------------------
175422024-02-12T01:42:02.790Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
175432024-02-12T01:42:02.790ZStates: Active Active Active
175442024-02-12T01:42:02.790ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
175452024-02-12T01:42:02.790Z 1 Acked 1000 Write 8 Done Done Done false
175462024-02-12T01:42:02.790Z 2 Acked 1001 Read 5 Done Done Done false
175472024-02-12T01:42:02.790Z 3 Acked 1002 Read 2 Done Done Sent false
175482024-02-12T01:42:02.790Z 4 Acked 1003 Read 9 Done Done Sent false
175492024-02-12T01:42:02.790Z 5 Acked 1004 Read 4 Done Done Sent false
175502024-02-12T01:42:02.790Z 6 Acked 1005 Write 7 Done Done Sent false
175512024-02-12T01:42:02.790Z 7 Acked 1006 Read 5 Done Done Sent false
175522024-02-12T01:42:02.790Z 8 Acked 1007 Write 8 Done Done Sent false
175532024-02-12T01:42:02.790Z 9 Acked 1008 Read 8 Done Done Sent false
175542024-02-12T01:42:02.790Z 10 Acked 1009 Flush 0 Done Done Sent false
175552024-02-12T01:42:02.790Z 11 Acked 1010 Write 10 Done Done Sent false
175562024-02-12T01:42:02.791Z 12 Acked 1011 Flush 0 Done Done Sent false
175572024-02-12T01:42:02.791Z 13 Acked 1012 Write 5 Done Done Sent false
175582024-02-12T01:42:02.791Z 14 Acked 1013 Read 1 Done Done Sent false
175592024-02-12T01:42:02.791Z 15 Acked 1014 Read 2 Done Done Sent false
175602024-02-12T01:42:02.791Z 16 Acked 1015 Read 10 Done Done Sent false
175612024-02-12T01:42:02.791Z 17 Acked 1016 Read 10 Done Done Sent false
175622024-02-12T01:42:02.791Z 18 Acked 1017 Read 5 Done Done Sent false
175632024-02-12T01:42:02.791Z 19 Acked 1018 Read 1 Done Done Sent false
175642024-02-12T01:42:02.791Z 20 Acked 1019 Write 1 Done Done Sent false
175652024-02-12T01:42:02.791Z 21 Acked 1020 Write 4 Done Done Sent false
175662024-02-12T01:42:02.791Z 22 Acked 1021 Read 4 Done Done Sent false
175672024-02-12T01:42:02.791Z 23 Acked 1022 Write 10 Done Done Sent false
175682024-02-12T01:42:02.791Z 24 Acked 1023 Write 8 Done Done Sent false
175692024-02-12T01:42:02.791Z 25 Acked 1024 Write 9 Done Done Sent false
175702024-02-12T01:42:02.791Z 26 Acked 1025 Write 6 Done Done Sent false
175712024-02-12T01:42:02.791Z 27 Acked 1026 Write 2 Done Done Sent false
175722024-02-12T01:42:02.791Z 28 Acked 1027 Write 5 Done Done Sent false
175732024-02-12T01:42:02.791Z 29 Acked 1028 Write 5 Done Done Sent false
175742024-02-12T01:42:02.791Z 30 Acked 1029 Read 5 Done Sent Sent false
175752024-02-12T01:42:02.791Z STATES DS:0 DS:1 DS:2 TOTAL
175762024-02-12T01:42:02.791Z New 0 0 0 0
175772024-02-12T01:42:02.791Z Sent 0 1 28 29
175782024-02-12T01:42:02.791Z Done 30 29 2 61
175792024-02-12T01:42:02.791Z Skipped 0 0 0 0
175802024-02-12T01:42:02.791Z Error 0 0 0 0
175812024-02-12T01:42:02.795ZLast Flush: 1011 1011 0
175822024-02-12T01:42:02.795ZDownstairs last five completed:
175832024-02-12T01:42:02.795ZUpstairs last five completed: 30 29 28 27 26
175842024-02-12T01:42:02.795Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:01.66110441Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1767}
175852024-02-12T01:42:02.795ZWrote out file "/tmp/test_repair_verify.data"
175862024-02-12T01:42:02.795Z
175872024-02-12T01:42:02.799ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175882024-02-12T01:42:02.820ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175892024-02-12T01:42:02.820Z 0 000-019 144 144 144 251 251 251 F F F
175902024-02-12T01:42:02.820Z 1 020-039 155 155 155 275 275 275 F F F
175912024-02-12T01:42:02.820Z 2 040-059 154 154 154 274 274 274 T T F <---
175922024-02-12T01:42:02.820Z 3 060-079 154 154 154 272 272 272 T T F <---
175932024-02-12T01:42:02.820Z 4 080-099 151 151 151 267 267 267 F F F
175942024-02-12T01:42:02.820Z 5 100-119 157 157 157 278 278 278 F F F
175952024-02-12T01:42:02.820Z 6 120-139 156 156 156 276 276 276 T T F <---
175962024-02-12T01:42:02.820Z 7 140-159 156 156 156 276 276 276 T T F <---
175972024-02-12T01:42:02.820Z 8 160-179 156 156 156 276 276 276 F F F
175982024-02-12T01:42:02.820Z 9 180-199 157 157 157 278 278 278 F F F
175992024-02-12T01:42:02.820Z 10 200-219 156 156 156 276 276 276 F F F
176002024-02-12T01:42:02.820Z 11 220-239 152 152 152 270 270 270 F F F
176012024-02-12T01:42:02.820Z 12 240-259 152 152 152 270 270 270 F F F
176022024-02-12T01:42:02.820Z 13 260-279 157 157 157 278 278 278 F F F
176032024-02-12T01:42:02.820Z 14 280-299 158 158 154 279 279 274 F F F <---
176042024-02-12T01:42:02.820Z 15 300-319 154 154 154 272 272 272 T T F <---
176052024-02-12T01:42:02.821Z 16 320-339 158 158 152 279 279 268 T T T <---
176062024-02-12T01:42:02.821Z 17 340-359 158 158 150 279 279 264 T T T <---
176072024-02-12T01:42:02.821Z 18 360-379 156 156 156 277 277 277 T T F <---
176082024-02-12T01:42:02.821Z 19 380-399 153 153 153 271 271 271 F F F
176092024-02-12T01:42:02.821Z 20 400-419 156 156 156 276 276 276 F F F
176102024-02-12T01:42:02.821Z 21 420-439 150 150 150 266 266 266 F F F
176112024-02-12T01:42:02.821Z 22 440-459 150 150 150 265 265 265 T T F <---
176122024-02-12T01:42:02.821Z 23 460-479 156 156 156 276 276 276 F F F
176132024-02-12T01:42:02.821Z 24 480-499 158 158 141 280 280 247 T T F <---
176142024-02-12T01:42:02.821Z 25 500-519 154 154 154 273 273 273 F F F
176152024-02-12T01:42:02.821Z 26 520-539 156 156 156 276 276 276 F F F
176162024-02-12T01:42:02.825Z 27 540-559 156 156 156 276 276 276 T T F <---
176172024-02-12T01:42:02.825Z 28 560-579 156 156 156 276 276 276 F F F
176182024-02-12T01:42:02.826Z 29 580-599 155 155 155 275 275 275 F F F
176192024-02-12T01:42:02.826ZMax gen: 158, Max flush: 280
176202024-02-12T01:42:02.826ZError: Difference in extent metadata found!
176212024-02-12T01:42:02.826ZA Difference in extent metadata is expected here
176222024-02-12T01:42:02.826ZOn loop 79
176232024-02-12T01:42:02.826Z
176242024-02-12T01:42:02.829ZVerifying data now
176252024-02-12T01:42:08.465ZNo change after verify
176262024-02-12T01:42:08.465ZLoop: 79 Downstairs dump after verify (and repair):
176272024-02-12T01:42:08.481ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
176282024-02-12T01:42:08.481Z 0 000-019 144 144 144 251 251 251 F F F
176292024-02-12T01:42:08.481Z 1 020-039 155 155 155 275 275 275 F F F
176302024-02-12T01:42:08.481Z 2 040-059 159 159 159 281 281 281 F F F
176312024-02-12T01:42:08.481Z 3 060-079 159 159 159 281 281 281 F F F
176322024-02-12T01:42:08.481Z 4 080-099 151 151 151 267 267 267 F F F
176332024-02-12T01:42:08.481Z 5 100-119 157 157 157 278 278 278 F F F
176342024-02-12T01:42:08.481Z 6 120-139 159 159 159 281 281 281 F F F
176352024-02-12T01:42:08.481Z 7 140-159 159 159 159 281 281 281 F F F
176362024-02-12T01:42:08.481Z 8 160-179 156 156 156 276 276 276 F F F
176372024-02-12T01:42:08.481Z 9 180-199 157 157 157 278 278 278 F F F
176382024-02-12T01:42:08.481Z 10 200-219 156 156 156 276 276 276 F F F
176392024-02-12T01:42:08.481Z 11 220-239 152 152 152 270 270 270 F F F
176402024-02-12T01:42:08.481Z 12 240-259 152 152 152 270 270 270 F F F
176412024-02-12T01:42:08.481Z 13 260-279 157 157 157 278 278 278 F F F
176422024-02-12T01:42:08.481Z 14 280-299 158 158 158 279 279 279 F F F
176432024-02-12T01:42:08.484Z 15 300-319 159 159 159 281 281 281 F F F
176442024-02-12T01:42:08.484Z 16 320-339 159 159 159 281 281 281 F F F
176452024-02-12T01:42:08.484Z 17 340-359 159 159 159 281 281 281 F F F
176462024-02-12T01:42:08.484Z 18 360-379 159 159 159 281 281 281 F F F
176472024-02-12T01:42:08.484Z 19 380-399 153 153 153 271 271 271 F F F
176482024-02-12T01:42:08.484Z 20 400-419 156 156 156 276 276 276 F F F
176492024-02-12T01:42:08.484Z 21 420-439 150 150 150 266 266 266 F F F
176502024-02-12T01:42:08.485Z 22 440-459 159 159 159 281 281 281 F F F
176512024-02-12T01:42:08.485Z 23 460-479 156 156 156 276 276 276 F F F
176522024-02-12T01:42:08.485Z 24 480-499 159 159 159 281 281 281 F F F
176532024-02-12T01:42:08.485Z 25 500-519 154 154 154 273 273 273 F F F
176542024-02-12T01:42:08.485Z 26 520-539 156 156 156 276 276 276 F F F
176552024-02-12T01:42:08.485Z 27 540-559 159 159 159 281 281 281 F F F
176562024-02-12T01:42:08.485Z 28 560-579 156 156 156 276 276 276 F F F
176572024-02-12T01:42:08.485Z 29 580-599 155 155 155 275 275 275 F F F
176582024-02-12T01:42:08.485ZMax gen: 159, Max flush: 281
176592024-02-12T01:42:08.485Z
176602024-02-12T01:42:08.485ZBegin loop 80
176612024-02-12T01:42:08.485ZDownstairs to restart: 2
176622024-02-12T01:42:08.507Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.376710765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
176632024-02-12T01:42:08.507Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.37701764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
176642024-02-12T01:42:08.512Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377059318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
176652024-02-12T01:42:08.512Z{"msg":"Crucible b4d12b24-6db6-45b2-bd5f-9360f12b2f37 has session id: 52947e9f-0930-4465-a697-1a613053fa6f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377091257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176662024-02-12T01:42:08.512Z{"msg":"Upstairs opts: Upstairs UUID: b4d12b24-6db6-45b2-bd5f-9360f12b2f37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377135474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176672024-02-12T01:42:08.512ZCrucible runtime is spawned
176682024-02-12T01:42:08.512Z{"msg":"Crucible stats registered with UUID: b4d12b24-6db6-45b2-bd5f-9360f12b2f37","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377170603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176692024-02-12T01:42:08.512Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.37723597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
176702024-02-12T01:42:08.512Z{"msg":"b4d12b24-6db6-45b2-bd5f-9360f12b2f37 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377357004Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176712024-02-12T01:42:08.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377396012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176722024-02-12T01:42:08.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377423101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176732024-02-12T01:42:08.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377465739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176742024-02-12T01:42:08.512Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377502817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176752024-02-12T01:42:08.512Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377546235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176762024-02-12T01:42:08.513Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377587173Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176772024-02-12T01:42:08.513Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377608402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176782024-02-12T01:42:08.513Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377657279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176792024-02-12T01:42:08.513Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377694648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176802024-02-12T01:42:08.513Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:07.377726816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176812024-02-12T01:42:08.513Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:07.377781504Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176822024-02-12T01:42:08.513Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377818872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176832024-02-12T01:42:08.513Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:07.37785507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176842024-02-12T01:42:08.513Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377870639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176852024-02-12T01:42:08.513Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377886728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176862024-02-12T01:42:08.513Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.377926997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176872024-02-12T01:42:08.513Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.378049921Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176882024-02-12T01:42:08.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.378625963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176892024-02-12T01:42:08.513Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.378666941Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176902024-02-12T01:42:08.513Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.37868642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176912024-02-12T01:42:08.513Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.378722909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176922024-02-12T01:42:08.513Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379248414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176932024-02-12T01:42:08.513Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379424595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176942024-02-12T01:42:08.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379705972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176952024-02-12T01:42:08.513Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.37975309Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176962024-02-12T01:42:08.513Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379801297Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176972024-02-12T01:42:08.513Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379868164Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176982024-02-12T01:42:08.513Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379915082Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
176992024-02-12T01:42:08.514Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:07.379933121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177002024-02-12T01:42:09.509Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.379654205Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177012024-02-12T01:42:09.509Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.379748151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177022024-02-12T01:42:09.514Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.38060389Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177032024-02-12T01:42:09.514Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.380645228Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177042024-02-12T01:42:09.514Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381122765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177052024-02-12T01:42:09.514Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381633901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177062024-02-12T01:42:09.514Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381676739Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177072024-02-12T01:42:09.514Z{"msg":"[0]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 281, 281, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381696778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177082024-02-12T01:42:09.514Z{"msg":"[0]R generation[0..12]: [144, 155, 159, 159, 151, 157, 159, 159, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381724347Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177092024-02-12T01:42:09.514Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381743966Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177102024-02-12T01:42:09.514Z{"msg":"[1]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 281, 281, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381771074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177112024-02-12T01:42:09.514Z{"msg":"[1]R generation[0..12]: [144, 155, 159, 159, 151, 157, 159, 159, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381789783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177122024-02-12T01:42:09.514Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381816052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177132024-02-12T01:42:09.514Z{"msg":"[2]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 281, 281, 276, 278, 276, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381834001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177142024-02-12T01:42:09.514Z{"msg":"[2]R generation[0..12]: [144, 155, 159, 159, 151, 157, 159, 159, 156, 157, 156, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.38185927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177152024-02-12T01:42:09.515Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381877299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177162024-02-12T01:42:09.515ZWait for a query_work_queue command to finish before sending IO
177172024-02-12T01:42:09.515Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381904298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177182024-02-12T01:42:09.515Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381922727Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177192024-02-12T01:42:09.515Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381946986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177202024-02-12T01:42:09.515Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381964875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177212024-02-12T01:42:09.515Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.381991924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177222024-02-12T01:42:09.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382009353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177232024-02-12T01:42:09.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382041841Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177242024-02-12T01:42:09.515Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382080959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177252024-02-12T01:42:09.515Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382102758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177262024-02-12T01:42:09.515Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382116538Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177272024-02-12T01:42:09.515Z{"msg":"b4d12b24-6db6-45b2-bd5f-9360f12b2f37 is now active with session: 52947e9f-0930-4465-a697-1a613053fa6f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382138597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177282024-02-12T01:42:09.515Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382151526Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775,"session_id":"52947e9f-0930-4465-a697-1a613053fa6f"}
177292024-02-12T01:42:09.515Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:08.382163906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
177302024-02-12T01:42:09.515ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
177312024-02-12T01:42:09.515ZLoading write count information from file "/tmp/test_repair_verify.data"
177322024-02-12T01:42:09.515ZRun Repair workload
177332024-02-12T01:42:09.515Z01/30 Write block 342 len 12288 data: 8 11 9
177342024-02-12T01:42:09.519Z02/30 Flush
177352024-02-12T01:42:09.519Z03/30 Flush
177362024-02-12T01:42:09.519Z04/30 Write block 309 len 28672 data: 15 15 14 11 13 11 12
177372024-02-12T01:42:09.519Z05/30 Read block 136 len 12288
177382024-02-12T01:42:09.522Z06/30 Write block 499 len 20480 data: 13 10 11 12 10
177392024-02-12T01:42:09.522Z07/30 Read block 474 len 4096
177402024-02-12T01:42:09.522Z08/30 Read block 75 len 36864
177412024-02-12T01:42:09.522Z09/30 Read block 161 len 8192
177422024-02-12T01:42:09.525Z10/30 Read block 52 len 28672
177432024-02-12T01:42:09.525Z11/30 Write block 472 len 24576 data: 11 15 16 14 11 12
177442024-02-12T01:42:09.525Z12/30 Read block 541 len 40960
177452024-02-12T01:42:09.527Z13/30 Write block 121 len 40960 data: 14 13 15 16 15 17 16 14 13 12
177462024-02-12T01:42:09.528Z14/30 Flush
177472024-02-12T01:42:09.530Z15/30 Write block 561 len 4096 data: 11
177482024-02-12T01:42:09.533Z16/30 Write block 529 len 36864 data: 9 13 12 13 16 15 17 17 16
177492024-02-12T01:42:09.533Z17/30 Read block 214 len 8192
177502024-02-12T01:42:09.533Z18/30 Read block 167 len 32768
177512024-02-12T01:42:09.537Z19/30 Write block 198 len 36864 data: 11 15 14 12 14 15 13 12 12
177522024-02-12T01:42:09.537Z20/30 Write block 533 len 20480 data: 17 16 18 18 17
177532024-02-12T01:42:09.537Z21/30 Write block 357 len 36864 data: 7 6 8 10 11 10 11 15 15
177542024-02-12T01:42:09.537Z22/30 Write block 169 len 24576 data: 15 15 15 12 13 13
177552024-02-12T01:42:09.540Z23/30 Write block 297 len 24576 data: 11 9 8 8 7 6
177562024-02-12T01:42:09.540Z24/30 Read block 447 len 4096
177572024-02-12T01:42:09.540Z25/30 Flush
177582024-02-12T01:42:09.543Z26/30 Flush
177592024-02-12T01:42:09.545Z27/30 Write block 169 len 4096 data: 16
177602024-02-12T01:42:09.546Z28/30 Read block 343 len 20480
177612024-02-12T01:42:09.546Z29/30 Write block 550 len 12288 data: 18 17 13
177622024-02-12T01:42:09.546Z30/30 Read block 227 len 36864
177632024-02-12T01:42:09.549Z----------------------------------------------------------------
177642024-02-12T01:42:09.549Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
177652024-02-12T01:42:09.549ZStates: Active Active Active
177662024-02-12T01:42:09.550ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177672024-02-12T01:42:09.550Z 1 Acked 1000 Write 3 Done Done Done false
177682024-02-12T01:42:09.550Z 2 Acked 1001 Flush 0 Done Done Sent false
177692024-02-12T01:42:09.550Z 3 Acked 1002 Flush 0 Done Done Sent false
177702024-02-12T01:42:09.550Z 4 Acked 1003 Write 7 Done Done Sent false
177712024-02-12T01:42:09.550Z 5 Acked 1004 Read 3 Done Done Sent false
177722024-02-12T01:42:09.550Z 6 Acked 1005 Write 5 Done Done Sent false
177732024-02-12T01:42:09.550Z 7 Acked 1006 Read 1 Done Done Sent false
177742024-02-12T01:42:09.550Z 8 Acked 1007 Read 9 Done Done Sent false
177752024-02-12T01:42:09.550Z 9 Acked 1008 Read 2 Done Done Sent false
177762024-02-12T01:42:09.550Z 10 Acked 1009 Read 7 Done Done Sent false
177772024-02-12T01:42:09.550Z 11 Acked 1010 Write 6 Done Done Sent false
177782024-02-12T01:42:09.550Z 12 Acked 1011 Read 10 Done Done Sent false
177792024-02-12T01:42:09.550Z 13 Acked 1012 Write 10 Done Done Sent false
177802024-02-12T01:42:09.550Z 14 Acked 1013 Flush 0 Done Done Sent false
177812024-02-12T01:42:09.550Z 15 Acked 1014 Write 1 Done Done Sent false
177822024-02-12T01:42:09.550Z 16 Acked 1015 Write 9 Done Done Sent false
177832024-02-12T01:42:09.550Z 17 Acked 1016 Read 2 Done Done Sent false
177842024-02-12T01:42:09.550Z 18 Acked 1017 Read 8 Done Done Sent false
177852024-02-12T01:42:09.550Z 19 Acked 1018 Write 9 Done Done Sent false
177862024-02-12T01:42:09.550Z 20 Acked 1019 Write 5 Done Done Sent false
177872024-02-12T01:42:09.550Z 21 Acked 1020 Write 9 Done Done Sent false
177882024-02-12T01:42:09.550Z 22 Acked 1021 Write 6 Done Done Sent false
177892024-02-12T01:42:09.550Z 23 Acked 1022 Write 6 Done Done Sent false
177902024-02-12T01:42:09.551Z 24 Acked 1023 Read 1 Done Done Sent false
177912024-02-12T01:42:09.551Z 25 Acked 1024 Flush 0 Done Done Sent false
177922024-02-12T01:42:09.551Z 26 Acked 1025 Flush 0 Done Done Sent false
177932024-02-12T01:42:09.551Z 27 Acked 1026 Write 1 Done Done Sent false
177942024-02-12T01:42:09.551Z 28 Acked 1027 Read 5 Done Done Sent false
177952024-02-12T01:42:09.551Z 29 Acked 1028 Write 3 Done Done Sent false
177962024-02-12T01:42:09.551Z 30 Acked 1029 Read 9 Done Sent Sent false
177972024-02-12T01:42:09.551Z STATES DS:0 DS:1 DS:2 TOTAL
177982024-02-12T01:42:09.551Z New 0 0 0 0
177992024-02-12T01:42:09.551Z Sent 0 1 29 30
178002024-02-12T01:42:09.551Z Done 30 29 1 60
178012024-02-12T01:42:09.551Z Skipped 0 0 0 0
178022024-02-12T01:42:09.551Z Error 0 0 0 0
178032024-02-12T01:42:09.551ZLast Flush: 1025 1025 0
178042024-02-12T01:42:09.551ZDownstairs last five completed:
178052024-02-12T01:42:09.551ZUpstairs last five completed: 30 29 28 27 26
178062024-02-12T01:42:09.551Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:08.417857832Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1775}
178072024-02-12T01:42:09.551ZWrote out file "/tmp/test_repair_verify.data"
178082024-02-12T01:42:09.554Z
178092024-02-12T01:42:09.557ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
178102024-02-12T01:42:09.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178112024-02-12T01:42:09.573Z 0 000-019 144 144 144 251 251 251 F F F
178122024-02-12T01:42:09.573Z 1 020-039 155 155 155 275 275 275 F F F
178132024-02-12T01:42:09.573Z 2 040-059 159 159 159 281 281 281 F F F
178142024-02-12T01:42:09.574Z 3 060-079 159 159 159 281 281 281 F F F
178152024-02-12T01:42:09.574Z 4 080-099 151 151 151 267 267 267 F F F
178162024-02-12T01:42:09.574Z 5 100-119 157 157 157 278 278 278 F F F
178172024-02-12T01:42:09.574Z 6 120-139 160 160 159 284 284 281 F F F <---
178182024-02-12T01:42:09.574Z 7 140-159 159 159 159 281 281 281 F F F
178192024-02-12T01:42:09.574Z 8 160-179 160 160 156 285 285 276 T T F <---
178202024-02-12T01:42:09.574Z 9 180-199 160 160 157 285 285 278 F F F <---
178212024-02-12T01:42:09.574Z 10 200-219 160 160 156 285 285 276 F F F <---
178222024-02-12T01:42:09.574Z 11 220-239 152 152 152 270 270 270 F F F
178232024-02-12T01:42:09.574Z 12 240-259 152 152 152 270 270 270 F F F
178242024-02-12T01:42:09.574Z 13 260-279 157 157 157 278 278 278 F F F
178252024-02-12T01:42:09.574Z 14 280-299 160 160 158 285 285 279 F F F <---
178262024-02-12T01:42:09.574Z 15 300-319 160 160 159 285 285 281 F F F <---
178272024-02-12T01:42:09.574Z 16 320-339 159 159 159 281 281 281 F F F
178282024-02-12T01:42:09.574Z 17 340-359 160 160 159 285 285 281 F F T <---
178292024-02-12T01:42:09.574Z 18 360-379 160 160 159 285 285 281 F F F <---
178302024-02-12T01:42:09.574Z 19 380-399 153 153 153 271 271 271 F F F
178312024-02-12T01:42:09.577Z 20 400-419 156 156 156 276 276 276 F F F
178322024-02-12T01:42:09.577Z 21 420-439 150 150 150 266 266 266 F F F
178332024-02-12T01:42:09.577Z 22 440-459 159 159 159 281 281 281 F F F
178342024-02-12T01:42:09.578Z 23 460-479 160 160 156 284 284 276 F F F <---
178352024-02-12T01:42:09.578Z 24 480-499 160 160 159 284 284 281 F F F <---
178362024-02-12T01:42:09.578Z 25 500-519 160 160 154 284 284 273 F F F <---
178372024-02-12T01:42:09.578Z 26 520-539 160 160 156 285 285 276 F F F <---
178382024-02-12T01:42:09.578Z 27 540-559 159 159 159 281 281 281 T T F <---
178392024-02-12T01:42:09.578Z 28 560-579 160 160 156 285 285 276 F F F <---
178402024-02-12T01:42:09.578Z 29 580-599 155 155 155 275 275 275 F F F
178412024-02-12T01:42:09.578ZMax gen: 160, Max flush: 285
178422024-02-12T01:42:09.578ZError: Difference in extent metadata found!
178432024-02-12T01:42:09.581ZA Difference in extent metadata is expected here
178442024-02-12T01:42:09.581ZOn loop 80
178452024-02-12T01:42:09.581Z
178462024-02-12T01:42:09.581ZVerifying data now
178472024-02-12T01:42:15.281ZNo change after verify
178482024-02-12T01:42:15.281ZLoop: 80 Downstairs dump after verify (and repair):
178492024-02-12T01:42:15.296ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178502024-02-12T01:42:15.296Z 0 000-019 144 144 144 251 251 251 F F F
178512024-02-12T01:42:15.297Z 1 020-039 155 155 155 275 275 275 F F F
178522024-02-12T01:42:15.297Z 2 040-059 159 159 159 281 281 281 F F F
178532024-02-12T01:42:15.297Z 3 060-079 159 159 159 281 281 281 F F F
178542024-02-12T01:42:15.297Z 4 080-099 151 151 151 267 267 267 F F F
178552024-02-12T01:42:15.297Z 5 100-119 157 157 157 278 278 278 F F F
178562024-02-12T01:42:15.297Z 6 120-139 160 160 160 284 284 284 F F F
178572024-02-12T01:42:15.297Z 7 140-159 159 159 159 281 281 281 F F F
178582024-02-12T01:42:15.297Z 8 160-179 161 161 161 286 286 286 F F F
178592024-02-12T01:42:15.297Z 9 180-199 160 160 160 285 285 285 F F F
178602024-02-12T01:42:15.297Z 10 200-219 160 160 160 285 285 285 F F F
178612024-02-12T01:42:15.297Z 11 220-239 152 152 152 270 270 270 F F F
178622024-02-12T01:42:15.297Z 12 240-259 152 152 152 270 270 270 F F F
178632024-02-12T01:42:15.297Z 13 260-279 157 157 157 278 278 278 F F F
178642024-02-12T01:42:15.297Z 14 280-299 160 160 160 285 285 285 F F F
178652024-02-12T01:42:15.297Z 15 300-319 160 160 160 285 285 285 F F F
178662024-02-12T01:42:15.297Z 16 320-339 159 159 159 281 281 281 F F F
178672024-02-12T01:42:15.297Z 17 340-359 160 160 160 285 285 285 F F F
178682024-02-12T01:42:15.297Z 18 360-379 160 160 160 285 285 285 F F F
178692024-02-12T01:42:15.300Z 19 380-399 153 153 153 271 271 271 F F F
178702024-02-12T01:42:15.301Z 20 400-419 156 156 156 276 276 276 F F F
178712024-02-12T01:42:15.301Z 21 420-439 150 150 150 266 266 266 F F F
178722024-02-12T01:42:15.301Z 22 440-459 159 159 159 281 281 281 F F F
178732024-02-12T01:42:15.301Z 23 460-479 160 160 160 284 284 284 F F F
178742024-02-12T01:42:15.301Z 24 480-499 160 160 160 284 284 284 F F F
178752024-02-12T01:42:15.301Z 25 500-519 160 160 160 284 284 284 F F F
178762024-02-12T01:42:15.301Z 26 520-539 160 160 160 285 285 285 F F F
178772024-02-12T01:42:15.301Z 27 540-559 161 161 161 286 286 286 F F F
178782024-02-12T01:42:15.301Z 28 560-579 160 160 160 285 285 285 F F F
178792024-02-12T01:42:15.301Z 29 580-599 155 155 155 275 275 275 F F F
178802024-02-12T01:42:15.301ZMax gen: 161, Max flush: 286
178812024-02-12T01:42:15.301Z
178822024-02-12T01:42:15.301ZBegin loop 81
178832024-02-12T01:42:15.301ZDownstairs to restart: 1
178842024-02-12T01:42:15.322Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.191600813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
178852024-02-12T01:42:15.322Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.19187202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
178862024-02-12T01:42:15.322Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.191904718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
178872024-02-12T01:42:15.327Z{"msg":"Crucible 6f7bb179-302c-46b6-9513-607ba7517606 has session id: 1d8a32b4-3263-45f0-a30b-01d93c2481e1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.191941337Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178882024-02-12T01:42:15.327Z{Crucible runtime is spawned
178892024-02-12T01:42:15.327Z"msg":"Upstairs opts: Upstairs UUID: 6f7bb179-302c-46b6-9513-607ba7517606, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.191997404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178902024-02-12T01:42:15.327Z{"msg":"Crucible stats registered with UUID: 6f7bb179-302c-46b6-9513-607ba7517606","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192038072Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178912024-02-12T01:42:15.327Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192067101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
178922024-02-12T01:42:15.327Z{"msg":"6f7bb179-302c-46b6-9513-607ba7517606 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192134298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178932024-02-12T01:42:15.327Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192175416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178942024-02-12T01:42:15.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192196924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178952024-02-12T01:42:15.327Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192229633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178962024-02-12T01:42:15.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192249452Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178972024-02-12T01:42:15.327Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.19228784Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178982024-02-12T01:42:15.327Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192308029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
178992024-02-12T01:42:15.327Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192340438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179002024-02-12T01:42:15.327Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192363307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179012024-02-12T01:42:15.327Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192389515Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179022024-02-12T01:42:15.327Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:14.192544868Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179032024-02-12T01:42:15.327Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192608655Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179042024-02-12T01:42:15.327Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:14.192666732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179052024-02-12T01:42:15.328Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192694851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179062024-02-12T01:42:15.328Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:14.192728489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179072024-02-12T01:42:15.328Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192745348Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179082024-02-12T01:42:15.328Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192779397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179092024-02-12T01:42:15.328Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.192954158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179102024-02-12T01:42:15.328Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.193477274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179112024-02-12T01:42:15.328Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.193528341Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179122024-02-12T01:42:15.328Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.19355909Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179132024-02-12T01:42:15.328Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.193601517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179142024-02-12T01:42:15.328Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194160651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179152024-02-12T01:42:15.328Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194201899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179162024-02-12T01:42:15.328Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194713064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179172024-02-12T01:42:15.328Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194764292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179182024-02-12T01:42:15.328Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.19481155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179192024-02-12T01:42:15.328Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194856768Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179202024-02-12T01:42:15.328Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194884146Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179212024-02-12T01:42:15.328Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:14.194919265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179222024-02-12T01:42:16.325Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.194509105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179232024-02-12T01:42:16.325Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.19462644Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179242024-02-12T01:42:16.330Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.195501248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179252024-02-12T01:42:16.330Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.195573705Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179262024-02-12T01:42:16.330Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196039922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179272024-02-12T01:42:16.330Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196568057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179282024-02-12T01:42:16.330Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196613915Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179292024-02-12T01:42:16.330Z{"msg":"[0]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 284, 281, 286, 285, 285, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196651333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179302024-02-12T01:42:16.330Z{"msg":"[0]R generation[0..12]: [144, 155, 159, 159, 151, 157, 160, 159, 161, 160, 160, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196687132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179312024-02-12T01:42:16.330Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.19672433Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179322024-02-12T01:42:16.330Z{"msg":"[1]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 284, 281, 286, 285, 285, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196760808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179332024-02-12T01:42:16.330Z{"msg":"[1]R generation[0..12]: [144, 155, 159, 159, 151, 157, 160, 159, 161, 160, 160, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196794207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179342024-02-12T01:42:16.331ZWait for a query_work_queue command to finish before sending IO
179352024-02-12T01:42:16.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196837974Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179362024-02-12T01:42:16.331Z{"msg":"[2]R flush_numbers[0..12]: [251, 275, 281, 281, 267, 278, 284, 281, 286, 285, 285, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196879152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179372024-02-12T01:42:16.331Z{"msg":"[2]R generation[0..12]: [144, 155, 159, 159, 151, 157, 160, 159, 161, 160, 160, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.19692136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179382024-02-12T01:42:16.331Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196960499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179392024-02-12T01:42:16.331Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.196992017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179402024-02-12T01:42:16.331Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197051734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179412024-02-12T01:42:16.331Z{"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197079803Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179422024-02-12T01:42:16.331Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197115051Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179432024-02-12T01:42:16.331Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197149999Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179442024-02-12T01:42:16.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197165979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179452024-02-12T01:42:16.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197186668Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179462024-02-12T01:42:16.331Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197200237Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179472024-02-12T01:42:16.331ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179482024-02-12T01:42:16.331Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197226286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179492024-02-12T01:42:16.331Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197239985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179502024-02-12T01:42:16.331Z{"msg":"6f7bb179-302c-46b6-9513-607ba7517606 is now active with session: 1d8a32b4-3263-45f0-a30b-01d93c2481e1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197253494Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179512024-02-12T01:42:16.331Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197266254Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783,"session_id":"1d8a32b4-3263-45f0-a30b-01d93c2481e1"}
179522024-02-12T01:42:16.331Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:15.197280133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
179532024-02-12T01:42:16.331ZLoading write count information from file "/tmp/test_repair_verify.data"
179542024-02-12T01:42:16.331ZRun Repair workload
179552024-02-12T01:42:16.331Z01/30 Flush
179562024-02-12T01:42:16.335Z02/30 Write block 532 len 4096 data: 14
179572024-02-12T01:42:16.335Z03/30 Read block 486 len 36864
179582024-02-12T01:42:16.335Z04/30 Read block 162 len 20480
179592024-02-12T01:42:16.335Z05/30 Read block 568 len 28672
179602024-02-12T01:42:16.335Z06/30 Write block 559 len 4096 data: 9
179612024-02-12T01:42:16.335Z07/30 Flush
179622024-02-12T01:42:16.338Z08/30 Write block 303 len 24576 data: 7 9 12 13 15 16
179632024-02-12T01:42:16.338Z09/30 Write block 569 len 12288 data: 16 16 14
179642024-02-12T01:42:16.338Z10/30 Write block 583 len 8192 data: 11 13
179652024-02-12T01:42:16.338Z11/30 Read block 409 len 24576
179662024-02-12T01:42:16.341Z12/30 Write block 593 len 24576 data: 9 8 10 10 8 5
179672024-02-12T01:42:16.341Z13/30 Write block 5 len 12288 data: 8 9 8
179682024-02-12T01:42:16.341Z14/30 Read block 143 len 8192
179692024-02-12T01:42:16.341Z15/30 Write block 250 len 16384 data: 7 9 7 9
179702024-02-12T01:42:16.342Z16/30 Read block 135 len 12288
179712024-02-12T01:42:16.344Z17/30 Write block 206 len 36864 data: 13 13 13 11 9 11 10 9 10
179722024-02-12T01:42:16.344Z18/30 Flush
179732024-02-12T01:42:16.347Z19/30 Write block 202 len 8192 data: 15 16
179742024-02-12T01:42:16.347Z20/30 Write block 178 len 28672 data: 16 17 13 15 14 15 16
179752024-02-12T01:42:16.347Z21/30 Write block 241 len 20480 data: 11 13 15 12 10
179762024-02-12T01:42:16.347Z22/30 Read block 72 len 40960
179772024-02-12T01:42:16.350Z23/30 Read block 369 len 32768
179782024-02-12T01:42:16.350Z24/30 Read block 18 len 40960
179792024-02-12T01:42:16.355Z25/30 Read block 201 len 28672
179802024-02-12T01:42:16.355Z26/30 Read block 583 len 4096
179812024-02-12T01:42:16.355Z27/30 Write block 504 len 16384 data: 10 10 9 10
179822024-02-12T01:42:16.355Z28/30 Read block 83 len 16384
179832024-02-12T01:42:16.358Z29/30 Write block 409 len 20480 data: 13 15 15 15 17
179842024-02-12T01:42:16.358Z30/30 Read block 570 len 8192
179852024-02-12T01:42:16.358Z----------------------------------------------------------------
179862024-02-12T01:42:16.358Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:29
179872024-02-12T01:42:16.358ZStates: Active Active Active
179882024-02-12T01:42:16.358ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179892024-02-12T01:42:16.358Z 2 Acked 1001 Write 1 Done Sent Done false
179902024-02-12T01:42:16.358Z 3 Acked 1002 Read 9 Done Sent Done false
179912024-02-12T01:42:16.358Z 4 Acked 1003 Read 5 Done Sent Done false
179922024-02-12T01:42:16.358Z 5 Acked 1004 Read 7 Done Sent Done false
179932024-02-12T01:42:16.358Z 6 Acked 1005 Write 1 Done Sent Done false
179942024-02-12T01:42:16.359Z 7 Acked 1006 Flush 0 Done Sent Done false
179952024-02-12T01:42:16.359Z 8 Acked 1007 Write 6 Done Sent Done false
179962024-02-12T01:42:16.359Z 9 Acked 1008 Write 3 Done Sent Done false
179972024-02-12T01:42:16.359Z 10 Acked 1009 Write 2 Done Sent Done false
179982024-02-12T01:42:16.359Z 11 Acked 1010 Read 6 Done Sent Done false
179992024-02-12T01:42:16.359Z 12 Acked 1011 Write 6 Done Sent Done false
180002024-02-12T01:42:16.359Z 13 Acked 1012 Write 3 Done Sent Done false
180012024-02-12T01:42:16.359Z 14 Acked 1013 Read 2 Done Sent Done false
180022024-02-12T01:42:16.359Z 15 Acked 1014 Write 4 Done Sent Done false
180032024-02-12T01:42:16.359Z 16 Acked 1015 Read 3 Done Sent Done false
180042024-02-12T01:42:16.359Z 17 Acked 1016 Write 9 Done Sent Done false
180052024-02-12T01:42:16.359Z 18 Acked 1017 Flush 0 Done Sent Done false
180062024-02-12T01:42:16.359Z 19 Acked 1018 Write 2 Done Sent Done false
180072024-02-12T01:42:16.359Z 20 Acked 1019 Write 7 Done Sent Done false
180082024-02-12T01:42:16.359Z 21 Acked 1020 Write 5 Done Sent Done false
180092024-02-12T01:42:16.359Z 22 Acked 1021 Read 10 Done Sent Done false
180102024-02-12T01:42:16.359Z 23 Acked 1022 Read 8 Done Sent Done false
180112024-02-12T01:42:16.359Z 24 Acked 1023 Read 10 Done Sent Done false
180122024-02-12T01:42:16.359Z 25 Acked 1024 Read 7 Done Sent Done false
180132024-02-12T01:42:16.359Z 26 Acked 1025 Read 1 Done Sent Done false
180142024-02-12T01:42:16.359Z 27 Acked 1026 Write 4 Done Sent Done false
180152024-02-12T01:42:16.359Z 28 Acked 1027 Read 4 Done Sent Done false
180162024-02-12T01:42:16.359Z 29 Acked 1028 Write 5 Done Sent Done false
180172024-02-12T01:42:16.359Z 30 Acked 1029 Read 2 Sent Sent Done false
180182024-02-12T01:42:16.359Z STATES DS:0 DS:1 DS:2 TOTAL
180192024-02-12T01:42:16.360Z New 0 0 0 0
180202024-02-12T01:42:16.360Z Sent 1 29 0 30
180212024-02-12T01:42:16.360Z Done 28 0 29 57
180222024-02-12T01:42:16.360Z Skipped 0 0 0 0
180232024-02-12T01:42:16.360Z Error 0 0 0 0
180242024-02-12T01:42:16.360ZLast Flush: 1017 1000 1017
180252024-02-12T01:42:16.360ZDownstairs last five completed: 1000
180262024-02-12T01:42:16.360ZUpstairs last five completed: 30 29 28 27 26
180272024-02-12T01:42:16.360Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:15.226186614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1783}
180282024-02-12T01:42:16.360ZWrote out file "/tmp/test_repair_verify.data"
180292024-02-12T01:42:16.363Z
180302024-02-12T01:42:16.366ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180312024-02-12T01:42:16.383ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180322024-02-12T01:42:16.383Z 0 000-019 162 144 162 289 251 289 F F F <---
180332024-02-12T01:42:16.383Z 1 020-039 155 155 155 275 275 275 F F F
180342024-02-12T01:42:16.383Z 2 040-059 159 159 159 281 281 281 F F F
180352024-02-12T01:42:16.383Z 3 060-079 159 159 159 281 281 281 F F F
180362024-02-12T01:42:16.383Z 4 080-099 151 151 151 267 267 267 F F F
180372024-02-12T01:42:16.383Z 5 100-119 157 157 157 278 278 278 F F F
180382024-02-12T01:42:16.384Z 6 120-139 160 160 160 284 284 284 F F F
180392024-02-12T01:42:16.384Z 7 140-159 159 159 159 281 281 281 F F F
180402024-02-12T01:42:16.384Z 8 160-179 161 161 161 286 286 286 T F T <---
180412024-02-12T01:42:16.387Z 9 180-199 160 160 160 285 285 285 T F T <---
180422024-02-12T01:42:16.387Z 10 200-219 162 160 162 289 285 289 T F T <---
180432024-02-12T01:42:16.387Z 11 220-239 152 152 152 270 270 270 F F F
180442024-02-12T01:42:16.387Z 12 240-259 162 152 162 289 270 289 T F T <---
180452024-02-12T01:42:16.387Z 13 260-279 157 157 157 278 278 278 F F F
180462024-02-12T01:42:16.387Z 14 280-299 160 160 160 285 285 285 F F F
180472024-02-12T01:42:16.387Z 15 300-319 162 160 162 289 285 289 F F F <---
180482024-02-12T01:42:16.387Z 16 320-339 159 159 159 281 281 281 F F F
180492024-02-12T01:42:16.387Z 17 340-359 160 160 160 285 285 285 F F F
180502024-02-12T01:42:16.387Z 18 360-379 160 160 160 285 285 285 F F F
180512024-02-12T01:42:16.387Z 19 380-399 153 153 153 271 271 271 F F F
180522024-02-12T01:42:16.387Z 20 400-419 156 156 156 276 276 276 T F T <---
180532024-02-12T01:42:16.387Z 21 420-439 150 150 150 266 266 266 F F F
180542024-02-12T01:42:16.387Z 22 440-459 159 159 159 281 281 281 F F F
180552024-02-12T01:42:16.387Z 23 460-479 160 160 160 284 284 284 F F F
180562024-02-12T01:42:16.387Z 24 480-499 160 160 160 284 284 284 F F F
180572024-02-12T01:42:16.387Z 25 500-519 160 160 160 284 284 284 T F T <---
180582024-02-12T01:42:16.388Z 26 520-539 162 160 162 288 285 288 F F F <---
180592024-02-12T01:42:16.388Z 27 540-559 162 161 162 288 286 288 F F F <---
180602024-02-12T01:42:16.388Z 28 560-579 162 160 162 289 285 289 F F F <---
180612024-02-12T01:42:16.388Z 29 580-599 162 155 162 289 275 289 F F F <---
180622024-02-12T01:42:16.388ZMax gen: 162, Max flush: 289
180632024-02-12T01:42:16.388ZError: Difference in extent metadata found!
180642024-02-12T01:42:16.388ZA Difference in extent metadata is expected here
180652024-02-12T01:42:16.388ZOn loop 81
180662024-02-12T01:42:16.388Z
180672024-02-12T01:42:16.400ZVerifying data now
180682024-02-12T01:42:20.994ZNo change after verify
180692024-02-12T01:42:20.994ZLoop: 81 Downstairs dump after verify (and repair):
180702024-02-12T01:42:21.011ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180712024-02-12T01:42:21.011Z 0 000-019 162 162 162 289 289 289 F F F
180722024-02-12T01:42:21.011Z 1 020-039 155 155 155 275 275 275 F F F
180732024-02-12T01:42:21.011Z 2 040-059 159 159 159 281 281 281 F F F
180742024-02-12T01:42:21.011Z 3 060-079 159 159 159 281 281 281 F F F
180752024-02-12T01:42:21.011Z 4 080-099 151 151 151 267 267 267 F F F
180762024-02-12T01:42:21.011Z 5 100-119 157 157 157 278 278 278 F F F
180772024-02-12T01:42:21.011Z 6 120-139 160 160 160 284 284 284 F F F
180782024-02-12T01:42:21.011Z 7 140-159 159 159 159 281 281 281 F F F
180792024-02-12T01:42:21.011Z 8 160-179 163 163 163 290 290 290 F F F
180802024-02-12T01:42:21.011Z 9 180-199 163 163 163 290 290 290 F F F
180812024-02-12T01:42:21.011Z 10 200-219 163 163 163 290 290 290 F F F
180822024-02-12T01:42:21.011Z 11 220-239 152 152 152 270 270 270 F F F
180832024-02-12T01:42:21.011Z 12 240-259 163 163 163 290 290 290 F F F
180842024-02-12T01:42:21.011Z 13 260-279 157 157 157 278 278 278 F F F
180852024-02-12T01:42:21.011Z 14 280-299 160 160 160 285 285 285 F F F
180862024-02-12T01:42:21.012Z 15 300-319 162 162 162 289 289 289 F F F
180872024-02-12T01:42:21.012Z 16 320-339 159 159 159 281 281 281 F F F
180882024-02-12T01:42:21.012Z 17 340-359 160 160 160 285 285 285 F F F
180892024-02-12T01:42:21.012Z 18 360-379 160 160 160 285 285 285 F F F
180902024-02-12T01:42:21.012Z 19 380-399 153 153 153 271 271 271 F F F
180912024-02-12T01:42:21.012Z 20 400-419 163 163 163 290 290 290 F F F
180922024-02-12T01:42:21.012Z 21 420-439 150 150 150 266 266 266 F F F
180932024-02-12T01:42:21.012Z 22 440-459 159 159 159 281 281 281 F F F
180942024-02-12T01:42:21.012Z 23 460-479 160 160 160 284 284 284 F F F
180952024-02-12T01:42:21.015Z 24 480-499 160 160 160 284 284 284 F F F
180962024-02-12T01:42:21.015Z 25 500-519 163 163 163 290 290 290 F F F
180972024-02-12T01:42:21.015Z 26 520-539 162 162 162 288 288 288 F F F
180982024-02-12T01:42:21.015Z 27 540-559 162 162 162 288 288 288 F F F
180992024-02-12T01:42:21.015Z 28 560-579 162 162 162 289 289 289 F F F
181002024-02-12T01:42:21.015Z 29 580-599 162 162 162 289 289 289 F F F
181012024-02-12T01:42:21.015ZMax gen: 163, Max flush: 290
181022024-02-12T01:42:21.016Z
181032024-02-12T01:42:21.016ZBegin loop 82
181042024-02-12T01:42:21.016ZDownstairs to restart: 1
181052024-02-12T01:42:21.036Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905094888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
181062024-02-12T01:42:21.036Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
181072024-02-12T01:42:21.041Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905391814Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
181082024-02-12T01:42:21.041Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905453231Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
181092024-02-12T01:42:21.041Z{"msg":"Crucible 255c3479-2e4c-4be0-921d-0e5d19762d5f has session id: e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905486279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181102024-02-12T01:42:21.041Z{"msg":"Upstairs opts: Upstairs UUID: 255c3479-2e4c-4be0-921d-0e5d19762d5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905529747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181112024-02-12T01:42:21.041Z{"msg":"Crucible stats registered with UUID: 255c3479-2e4c-4be0-921d-0e5d19762d5f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905553616Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181122024-02-12T01:42:21.041Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905567305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
181132024-02-12T01:42:21.041Z{"msg":"255c3479-2e4c-4be0-921d-0e5d19762d5f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905586564Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181142024-02-12T01:42:21.041Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905606563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181152024-02-12T01:42:21.041Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905626573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181162024-02-12T01:42:21.041Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905659251Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181172024-02-12T01:42:21.041Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905694609Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181182024-02-12T01:42:21.041Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905722488Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181192024-02-12T01:42:21.041Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905758016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181202024-02-12T01:42:21.041Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905789625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181212024-02-12T01:42:21.041Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905817204Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181222024-02-12T01:42:21.041Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.905872631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181232024-02-12T01:42:21.041Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906020544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181242024-02-12T01:42:21.041Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906073871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181252024-02-12T01:42:21.041Z{"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:19.906175036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181262024-02-12T01:42:21.042Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:19.906225314Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181272024-02-12T01:42:21.042Z{"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:19.906249703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181282024-02-12T01:42:21.042Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906274772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181292024-02-12T01:42:21.042Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906289671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181302024-02-12T01:42:21.042Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906387286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181312024-02-12T01:42:21.042Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906710031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181322024-02-12T01:42:21.042Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906748659Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181332024-02-12T01:42:21.042Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906843464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181342024-02-12T01:42:21.042Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.906889922Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181352024-02-12T01:42:21.042Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907317182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181362024-02-12T01:42:21.042Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907445776Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181372024-02-12T01:42:21.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907808638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181382024-02-12T01:42:21.042Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907853976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181392024-02-12T01:42:21.042Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907874985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181402024-02-12T01:42:21.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907909924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181412024-02-12T01:42:21.042Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907943742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181422024-02-12T01:42:21.042Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:19.907968081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181432024-02-12T01:42:22.039Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.907677156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181442024-02-12T01:42:22.039Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.907773421Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181452024-02-12T01:42:22.044Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.908678568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181462024-02-12T01:42:22.044Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.908734145Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181472024-02-12T01:42:22.044Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909198073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181482024-02-12T01:42:22.044Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909716698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181492024-02-12T01:42:22.044Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909778285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181502024-02-12T01:42:22.044Z{"msg":"[0]R flush_numbers[0..12]: [289, 275, 281, 281, 267, 278, 284, 281, 290, 290, 290, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909807694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181512024-02-12T01:42:22.044Z{"msg":"[0]R generation[0..12]: [162, 155, 159, 159, 151, 157, 160, 159, 163, 163, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909837592Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181522024-02-12T01:42:22.044Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909856392Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181532024-02-12T01:42:22.044Z{"msg":"[1]R flush_numbers[0..12]: [289, 275, 281, 281, 267, 278, 284, 281, 290, 290, 290, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.90988269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181542024-02-12T01:42:22.045Z{"msg":"[1]R generation[0..12]: [162, 155, 159, 159, 151, 157, 160, 159, 163, 163, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909901269Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181552024-02-12T01:42:22.045Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909927698Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181562024-02-12T01:42:22.045Z{"msg":"Wait for a query_work_queue command to finish before sending IO
181572024-02-12T01:42:22.045Z[2]R flush_numbers[0..12]: [289, 275, 281, 281, 267, 278, 284, 281, 290, 290, 290, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909949307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181582024-02-12T01:42:22.045Z{"msg":"[2]R generation[0..12]: [162, 155, 159, 159, 151, 157, 160, 159, 163, 163, 163, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909968186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181592024-02-12T01:42:22.045Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.909985896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181602024-02-12T01:42:22.045Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910011704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181612024-02-12T01:42:22.045Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910029613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181622024-02-12T01:42:22.045Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910046153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181632024-02-12T01:42:22.045Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910063442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181642024-02-12T01:42:22.045Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910080411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181652024-02-12T01:42:22.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.9101039Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181662024-02-12T01:42:22.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910120939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181672024-02-12T01:42:22.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910151857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181682024-02-12T01:42:22.045Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910168757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181692024-02-12T01:42:22.045Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910193226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181702024-02-12T01:42:22.045Z{"msg":"255c3479-2e4c-4be0-921d-0e5d19762d5f is now active with session: e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910219814Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181712024-02-12T01:42:22.045Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910235584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791,"session_id":"e8f5c96b-4ff0-4fa5-a3ea-8720edc04fab"}
181722024-02-12T01:42:22.045Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:20.910265552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
181732024-02-12T01:42:22.045ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181742024-02-12T01:42:22.045ZLoading write count information from file "/tmp/test_repair_verify.data"
181752024-02-12T01:42:22.045ZRun Repair workload
181762024-02-12T01:42:22.045Z01/30 Write block 201 len 20480 data: 13 16 17 14 13
181772024-02-12T01:42:22.049Z02/30 Read block 363 len 4096
181782024-02-12T01:42:22.049Z03/30 Flush
181792024-02-12T01:42:22.049Z04/30 Write block 91 len 32768 data: 12 10 8 7 7 9 10 10
181802024-02-12T01:42:22.049Z05/30 Flush
181812024-02-12T01:42:22.049Z06/30 Flush
181822024-02-12T01:42:22.052Z07/30 Write block 287 len 36864 data: 11 14 13 14 16 14 15 12 13
181832024-02-12T01:42:22.052Z08/30 Read block 344 len 40960
181842024-02-12T01:42:22.052Z09/30 Read block 418 len 16384
181852024-02-12T01:42:22.055Z10/30 Read block 369 len 40960
181862024-02-12T01:42:22.055Z11/30 Read block 527 len 8192
181872024-02-12T01:42:22.055Z12/30 Write block 511 len 4096 data: 11
181882024-02-12T01:42:22.055Z13/30 Read block 507 len 4096
181892024-02-12T01:42:22.058Z14/30 Write block 4 len 12288 data: 6 9 10
181902024-02-12T01:42:22.058Z15/30 Flush
181912024-02-12T01:42:22.061Z16/30 Flush
181922024-02-12T01:42:22.061Z17/30 Write block 441 len 8192 data: 13 11
181932024-02-12T01:42:22.067Z18/30 Read block 368 len 36864
181942024-02-12T01:42:22.067Z19/30 Write block 42 len 24576 data: 8 12 12 12 14 16
181952024-02-12T01:42:22.067Z20/30 Write block 554 len 20480 data: 11 10 9 8 8
181962024-02-12T01:42:22.071Z21/30 Read block 272 len 36864
181972024-02-12T01:42:22.071Z22/30 Write block 538 len 24576 data: 15 12 12 10 10 7
181982024-02-12T01:42:22.071Z23/30 Read block 55 len 24576
181992024-02-12T01:42:22.071Z24/30 Read block 508 len 32768
182002024-02-12T01:42:22.071Z25/30 Flush
182012024-02-12T01:42:22.076Z26/30 Write block 426 len 20480 data: 9 7 6 8 9
182022024-02-12T01:42:22.076Z27/30 Write block 34 len 40960 data: 17 14 13 11 15 11 11 10 9 13
182032024-02-12T01:42:22.076Z28/30 Read block 324 len 28672
182042024-02-12T01:42:22.077Z29/30 Read block 238 len 4096
182052024-02-12T01:42:22.077Z30/30 Read block 543 len 28672
182062024-02-12T01:42:22.077Z----------------------------------------------------------------
182072024-02-12T01:42:22.077Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
182082024-02-12T01:42:22.077ZStates: Active Active Active
182092024-02-12T01:42:22.077ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182102024-02-12T01:42:22.077Z 1 Acked 1000 Write 5 Done Sent Done false
182112024-02-12T01:42:22.077Z 2 Acked 1001 Read 1 Done Sent Done false
182122024-02-12T01:42:22.077Z 3 Acked 1002 Flush 0 Done Sent Done false
182132024-02-12T01:42:22.077Z 4 Acked 1003 Write 8 Done Sent Done false
182142024-02-12T01:42:22.077Z 5 Acked 1004 Flush 0 Done Sent Done false
182152024-02-12T01:42:22.077Z 6 Acked 1005 Flush 0 Done Sent Done false
182162024-02-12T01:42:22.077Z 7 Acked 1006 Write 9 Done Sent Done false
182172024-02-12T01:42:22.077Z 8 Acked 1007 Read 10 Done Sent Done false
182182024-02-12T01:42:22.077Z 9 Acked 1008 Read 4 Done Sent Done false
182192024-02-12T01:42:22.077Z 10 Acked 1009 Read 10 Done Sent Done false
182202024-02-12T01:42:22.077Z 11 Acked 1010 Read 2 Done Sent Done false
182212024-02-12T01:42:22.077Z 12 Acked 1011 Write 1 Done Sent Done false
182222024-02-12T01:42:22.077Z 13 Acked 1012 Read 1 Done Sent Done false
182232024-02-12T01:42:22.077Z 14 Acked 1013 Write 3 Done Sent Done false
182242024-02-12T01:42:22.077Z 15 Acked 1014 Flush 0 Done Sent Done false
182252024-02-12T01:42:22.077Z 16 Acked 1015 Flush 0 Done Sent Done false
182262024-02-12T01:42:22.077Z 17 Acked 1016 Write 2 Done Sent Done false
182272024-02-12T01:42:22.077Z 18 Acked 1017 Read 9 Done Sent Done false
182282024-02-12T01:42:22.078Z 19 Acked 1018 Write 6 Done Sent Done false
182292024-02-12T01:42:22.078Z 20 Acked 1019 Write 5 Done Sent Done false
182302024-02-12T01:42:22.078Z 21 Acked 1020 Read 9 Done Sent Done false
182312024-02-12T01:42:22.078Z 22 Acked 1021 Write 6 Done Sent Done false
182322024-02-12T01:42:22.078Z 23 Acked 1022 Read 6 Done Sent Done false
182332024-02-12T01:42:22.078Z 24 Acked 1023 Read 8 Done Sent Done false
182342024-02-12T01:42:22.078Z 25 Acked 1024 Flush 0 Done Sent Done false
182352024-02-12T01:42:22.078Z 26 Acked 1025 Write 5 Done Sent Done false
182362024-02-12T01:42:22.078Z 27 Acked 1026 Write 10 Done Sent Done false
182372024-02-12T01:42:22.078Z 28 Acked 1027 Read 7 Done Sent Done false
182382024-02-12T01:42:22.078Z 29 Acked 1028 Read 1 Done Sent Done false
182392024-02-12T01:42:22.078Z 30 Acked 1029 Read 7 Sent Sent Done false
182402024-02-12T01:42:22.078Z STATES DS:0 DS:1 DS:2 TOTAL
182412024-02-12T01:42:22.078Z New 0 0 0 0
182422024-02-12T01:42:22.078Z Sent 1 30 0 31
182432024-02-12T01:42:22.078Z Done 29 0 30 59
182442024-02-12T01:42:22.078Z Skipped 0 0 0 0
182452024-02-12T01:42:22.078Z Error 0 0 0 0
182462024-02-12T01:42:22.078ZLast Flush: 1024 0 1024
182472024-02-12T01:42:22.078ZDownstairs last five completed:
182482024-02-12T01:42:22.078ZUpstairs last five completed: 30 29 28 27 26
182492024-02-12T01:42:22.078Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:20.944928838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1791}
182502024-02-12T01:42:22.078ZWrote out file "/tmp/test_repair_verify.data"
182512024-02-12T01:42:22.082Z
182522024-02-12T01:42:22.085ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182532024-02-12T01:42:22.100ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182542024-02-12T01:42:22.101Z 0 000-019 164 162 164 294 289 294 F F F <---
182552024-02-12T01:42:22.101Z 1 020-039 155 155 155 275 275 275 T F T <---
182562024-02-12T01:42:22.101Z 2 040-059 164 159 164 296 281 296 T F T <---
182572024-02-12T01:42:22.101Z 3 060-079 159 159 159 281 281 281 F F F
182582024-02-12T01:42:22.101Z 4 080-099 164 151 164 292 267 292 F F F <---
182592024-02-12T01:42:22.101Z 5 100-119 157 157 157 278 278 278 F F F
182602024-02-12T01:42:22.101Z 6 120-139 160 160 160 284 284 284 F F F
182612024-02-12T01:42:22.101Z 7 140-159 159 159 159 281 281 281 F F F
182622024-02-12T01:42:22.101Z 8 160-179 163 163 163 290 290 290 F F F
182632024-02-12T01:42:22.101Z 9 180-199 163 163 163 290 290 290 F F F
182642024-02-12T01:42:22.101Z 10 200-219 164 163 164 291 290 291 F F F <---
182652024-02-12T01:42:22.101Z 11 220-239 152 152 152 270 270 270 F F F
182662024-02-12T01:42:22.101Z 12 240-259 163 163 163 290 290 290 F F F
182672024-02-12T01:42:22.101Z 13 260-279 157 157 157 278 278 278 F F F
182682024-02-12T01:42:22.104Z 14 280-299 164 160 164 294 285 294 F F F <---
182692024-02-12T01:42:22.105Z 15 300-319 162 162 162 289 289 289 F F F
182702024-02-12T01:42:22.105Z 16 320-339 159 159 159 281 281 281 F F F
182712024-02-12T01:42:22.105Z 17 340-359 160 160 160 285 285 285 F F F
182722024-02-12T01:42:22.105Z 18 360-379 160 160 160 285 285 285 F F F
182732024-02-12T01:42:22.105Z 19 380-399 153 153 153 271 271 271 F F F
182742024-02-12T01:42:22.105Z 20 400-419 163 163 163 290 290 290 F F F
182752024-02-12T01:42:22.105Z 21 420-439 150 150 150 266 266 266 T F T <---
182762024-02-12T01:42:22.105Z 22 440-459 164 159 164 296 281 296 F F F <---
182772024-02-12T01:42:22.105Z 23 460-479 160 160 160 284 284 284 F F F
182782024-02-12T01:42:22.105Z 24 480-499 160 160 160 284 284 284 F F F
182792024-02-12T01:42:22.105Z 25 500-519 164 163 164 294 290 294 F F F <---
182802024-02-12T01:42:22.105Z 26 520-539 164 162 164 296 288 296 F F F <---
182812024-02-12T01:42:22.105Z 27 540-559 164 162 164 296 288 296 F F F <---
182822024-02-12T01:42:22.105Z 28 560-579 162 162 162 289 289 289 F F F
182832024-02-12T01:42:22.105Z 29 580-599 162 162 162 289 289 289 F F F
182842024-02-12T01:42:22.105ZMax gen: 164, Max flush: 296
182852024-02-12T01:42:22.105ZError: Difference in extent metadata found!
182862024-02-12T01:42:22.108ZA Difference in extent metadata is expected here
182872024-02-12T01:42:22.108ZOn loop 82
182882024-02-12T01:42:22.108Z
182892024-02-12T01:42:22.108ZVerifying data now
182902024-02-12T01:42:26.666ZNo change after verify
182912024-02-12T01:42:26.666ZLoop: 82 Downstairs dump after verify (and repair):
182922024-02-12T01:42:26.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182932024-02-12T01:42:26.681Z 0 000-019 164 164 164 294 294 294 F F F
182942024-02-12T01:42:26.681Z 1 020-039 165 165 165 297 297 297 F F F
182952024-02-12T01:42:26.681Z 2 040-059 165 165 165 297 297 297 F F F
182962024-02-12T01:42:26.681Z 3 060-079 159 159 159 281 281 281 F F F
182972024-02-12T01:42:26.681Z 4 080-099 164 164 164 292 292 292 F F F
182982024-02-12T01:42:26.681Z 5 100-119 157 157 157 278 278 278 F F F
182992024-02-12T01:42:26.682Z 6 120-139 160 160 160 284 284 284 F F F
183002024-02-12T01:42:26.682Z 7 140-159 159 159 159 281 281 281 F F F
183012024-02-12T01:42:26.685Z 8 160-179 163 163 163 290 290 290 F F F
183022024-02-12T01:42:26.685Z 9 180-199 163 163 163 290 290 290 F F F
183032024-02-12T01:42:26.685Z 10 200-219 164 164 164 291 291 291 F F F
183042024-02-12T01:42:26.685Z 11 220-239 152 152 152 270 270 270 F F F
183052024-02-12T01:42:26.685Z 12 240-259 163 163 163 290 290 290 F F F
183062024-02-12T01:42:26.685Z 13 260-279 157 157 157 278 278 278 F F F
183072024-02-12T01:42:26.685Z 14 280-299 164 164 164 294 294 294 F F F
183082024-02-12T01:42:26.685Z 15 300-319 162 162 162 289 289 289 F F F
183092024-02-12T01:42:26.685Z 16 320-339 159 159 159 281 281 281 F F F
183102024-02-12T01:42:26.685Z 17 340-359 160 160 160 285 285 285 F F F
183112024-02-12T01:42:26.685Z 18 360-379 160 160 160 285 285 285 F F F
183122024-02-12T01:42:26.685Z 19 380-399 153 153 153 271 271 271 F F F
183132024-02-12T01:42:26.685Z 20 400-419 163 163 163 290 290 290 F F F
183142024-02-12T01:42:26.685Z 21 420-439 165 165 165 297 297 297 F F F
183152024-02-12T01:42:26.686Z 22 440-459 164 164 164 296 296 296 F F F
183162024-02-12T01:42:26.686Z 23 460-479 160 160 160 284 284 284 F F F
183172024-02-12T01:42:26.686Z 24 480-499 160 160 160 284 284 284 F F F
183182024-02-12T01:42:26.686Z 25 500-519 164 164 164 294 294 294 F F F
183192024-02-12T01:42:26.686Z 26 520-539 164 164 164 296 296 296 F F F
183202024-02-12T01:42:26.686Z 27 540-559 164 164 164 296 296 296 F F F
183212024-02-12T01:42:26.686Z 28 560-579 162 162 162 289 289 289 F F F
183222024-02-12T01:42:26.686Z 29 580-599 162 162 162 289 289 289 F F F
183232024-02-12T01:42:26.686ZMax gen: 165, Max flush: 297
183242024-02-12T01:42:26.686Z
183252024-02-12T01:42:26.686ZBegin loop 83
183262024-02-12T01:42:26.686ZDownstairs to restart: 2
183272024-02-12T01:42:26.705Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57338594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
183282024-02-12T01:42:26.709Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.573690565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
183292024-02-12T01:42:26.709Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.573730423Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
183302024-02-12T01:42:26.709Z{"msg":"Crucible 2dfa7d8e-7da6-4a92-967c-dc4c796a7655 has session id: 5e0d7eef-a3ab-4ffd-b243-06455737633d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.573753122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183312024-02-12T01:42:26.709Z{"msg":"Upstairs opts: Upstairs UUID: 2dfa7d8e-7da6-4a92-967c-dc4c796a7655, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57379399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183322024-02-12T01:42:26.709ZCrucible runtime is spawned
183332024-02-12T01:42:26.709Z{"msg":"Crucible stats registered with UUID: 2dfa7d8e-7da6-4a92-967c-dc4c796a7655","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.573834568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183342024-02-12T01:42:26.710Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.573909215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
183352024-02-12T01:42:26.710Z{"msg":"2dfa7d8e-7da6-4a92-967c-dc4c796a7655 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574061378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183362024-02-12T01:42:26.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574100165Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183372024-02-12T01:42:26.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574139094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183382024-02-12T01:42:26.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574172212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183392024-02-12T01:42:26.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57421791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183402024-02-12T01:42:26.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574245279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183412024-02-12T01:42:26.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574275347Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183422024-02-12T01:42:26.710Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574324255Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183432024-02-12T01:42:26.710Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574376032Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183442024-02-12T01:42:26.710Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574403431Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183452024-02-12T01:42:26.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574442899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183462024-02-12T01:42:26.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574680078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183472024-02-12T01:42:26.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.574735895Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183482024-02-12T01:42:26.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.575217682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183492024-02-12T01:42:26.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57525735Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183502024-02-12T01:42:26.710Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.575300178Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183512024-02-12T01:42:26.710Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.575326207Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183522024-02-12T01:42:26.710Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.575843533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183532024-02-12T01:42:26.710Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.575985096Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183542024-02-12T01:42:26.710Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57630792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183552024-02-12T01:42:26.710Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576405865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183562024-02-12T01:42:26.711Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576451223Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183572024-02-12T01:42:26.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576466572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183582024-02-12T01:42:26.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576480932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183592024-02-12T01:42:26.711Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57651583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183602024-02-12T01:42:26.711Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576530719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183612024-02-12T01:42:26.711Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576544199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183622024-02-12T01:42:26.715Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.576889682Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183632024-02-12T01:42:26.715Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577329351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183642024-02-12T01:42:26.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577369899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183652024-02-12T01:42:26.715Z{"msg":"[0]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 278, 284, 281, 290, 290, 291, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577386548Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183662024-02-12T01:42:26.715Z{"msg":"[0]R generation[0..12]: [164, 165, 165, 159, 164, 157, 160, 159, 163, 163, 164, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577408027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183672024-02-12T01:42:26.715Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577422587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183682024-02-12T01:42:26.715Z{"msg":"[1]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 278, 284, 281, 290, 290, 291, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577443186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183692024-02-12T01:42:26.715Z{"msg":"[1]R generation[0..12]: [164, 165, 165, 159, 164, 157, 160, 159, 163, 163, 164, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577457475Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183702024-02-12T01:42:26.715Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577478064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183712024-02-12T01:42:26.715Z{"msg":"[2]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 278, 284, 281, 290, 290, 291, 270]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577492434Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO
183722024-02-12T01:42:26.715Z","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183732024-02-12T01:42:26.715Z{"msg":"[2]R generation[0..12]: [164, 165, 165, 159, 164, 157, 160, 159, 163, 163, 164, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577520712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183742024-02-12T01:42:26.715Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577534561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183752024-02-12T01:42:26.715Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57755428Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183762024-02-12T01:42:26.715Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.57756805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183772024-02-12T01:42:26.715Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577581129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183782024-02-12T01:42:26.715Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577594839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183792024-02-12T01:42:26.715Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577608708Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183802024-02-12T01:42:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577627357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183812024-02-12T01:42:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577640886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183822024-02-12T01:42:26.716Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577667645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183832024-02-12T01:42:26.716Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577682154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183842024-02-12T01:42:26.716Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577696304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183852024-02-12T01:42:26.716Z{"msg":"2dfa7d8e-7da6-4a92-967c-dc4c796a7655 is now active with session: 5e0d7eef-a3ab-4ffd-b243-06455737633d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577725282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183862024-02-12T01:42:26.716Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577739852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799,"session_id":"5e0d7eef-a3ab-4ffd-b243-06455737633d"}
183872024-02-12T01:42:26.716Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:25.577758831Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
183882024-02-12T01:42:26.716ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183892024-02-12T01:42:26.716ZLoading write count information from file "/tmp/test_repair_verify.data"
183902024-02-12T01:42:26.716ZRun Repair workload
183912024-02-12T01:42:26.716Z01/30 Write block 133 len 36864 data: 13 13 11 14 12 14 16 17 19
183922024-02-12T01:42:26.716Z02/30 Flush
183932024-02-12T01:42:26.716Z03/30 Flush
183942024-02-12T01:42:26.719Z04/30 Write block 541 len 36864 data: 11 11 8 8 9 12 12 12 14
183952024-02-12T01:42:26.719Z05/30 Read block 210 len 28672
183962024-02-12T01:42:26.719Z06/30 Read block 595 len 16384
183972024-02-12T01:42:26.719Z07/30 Read block 112 len 12288
183982024-02-12T01:42:26.720Z08/30 Flush
183992024-02-12T01:42:26.722Z09/30 Write block 363 len 40960 data: 12 16 16 14 20 19 19 18 15 17
184002024-02-12T01:42:26.723Z10/30 Read block 544 len 8192
184012024-02-12T01:42:26.726Z11/30 Read block 465 len 16384
184022024-02-12T01:42:26.726Z12/30 Read block 516 len 36864
184032024-02-12T01:42:26.726Z13/30 Read block 82 len 40960
184042024-02-12T01:42:26.732Z14/30 Write block 441 len 8192 data: 14 12
184052024-02-12T01:42:26.732Z15/30 Read block 172 len 40960
184062024-02-12T01:42:26.732Z16/30 Read block 179 len 8192
184072024-02-12T01:42:26.732Z17/30 Read block 99 len 20480
184082024-02-12T01:42:26.732Z18/30 Read block 195 len 40960
184092024-02-12T01:42:26.735Z19/30 Flush
184102024-02-12T01:42:26.735Z20/30 Write block 505 len 28672 data: 11 10 11 8 7 10 12
184112024-02-12T01:42:26.735Z21/30 Read block 206 len 36864
184122024-02-12T01:42:26.735Z22/30 Write block 223 len 20480 data: 13 13 12 8 9
184132024-02-12T01:42:26.738Z23/30 Write block 361 len 32768 data: 12 11 13 17 17 15 21 20
184142024-02-12T01:42:26.738Z24/30 Write block 147 len 20480 data: 13 13 16 17 16
184152024-02-12T01:42:26.738Z25/30 Write block 113 len 12288 data: 11 13 13
184162024-02-12T01:42:26.738Z26/30 Flush
184172024-02-12T01:42:26.741Z27/30 Write block 251 len 20480 data: 10 8 10 9 11
184182024-02-12T01:42:26.741Z28/30 Read block 174 len 16384
184192024-02-12T01:42:26.741Z29/30 Write block 257 len 20480 data: 12 14 12 16 12
184202024-02-12T01:42:26.745Z30/30 Read block 93 len 20480
184212024-02-12T01:42:26.745Z----------------------------------------------------------------
184222024-02-12T01:42:26.745Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:28
184232024-02-12T01:42:26.745ZStates: Active Active Active
184242024-02-12T01:42:26.745ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184252024-02-12T01:42:26.745Z 3 Acked 1002 Flush 0 Done Done Sent false
184262024-02-12T01:42:26.745Z 4 Acked 1003 Write 9 Done Done Sent false
184272024-02-12T01:42:26.745Z 5 Acked 1004 Read 7 Done Done Sent false
184282024-02-12T01:42:26.745Z 6 Acked 1005 Read 4 Done Done Sent false
184292024-02-12T01:42:26.745Z 7 Acked 1006 Read 3 Done Done Sent false
184302024-02-12T01:42:26.745Z 8 Acked 1007 Flush 0 Done Done Sent false
184312024-02-12T01:42:26.745Z 9 Acked 1008 Write 10 Done Done Sent false
184322024-02-12T01:42:26.745Z 10 Acked 1009 Read 2 Done Done Sent false
184332024-02-12T01:42:26.745Z 11 Acked 1010 Read 4 Done Done Sent false
184342024-02-12T01:42:26.745Z 12 Acked 1011 Read 9 Done Done Sent false
184352024-02-12T01:42:26.745Z 13 Acked 1012 Read 10 Done Done Sent false
184362024-02-12T01:42:26.745Z 14 Acked 1013 Write 2 Done Done Sent false
184372024-02-12T01:42:26.745Z 15 Acked 1014 Read 10 Done Done Sent false
184382024-02-12T01:42:26.745Z 16 Acked 1015 Read 2 Done Done Sent false
184392024-02-12T01:42:26.745Z 17 Acked 1016 Read 5 Done Done Sent false
184402024-02-12T01:42:26.745Z 18 Acked 1017 Read 10 Done Done Sent false
184412024-02-12T01:42:26.746Z 19 Acked 1018 Flush 0 Done Done Sent false
184422024-02-12T01:42:26.746Z 20 Acked 1019 Write 7 Done Done Sent false
184432024-02-12T01:42:26.746Z 21 Acked 1020 Read 9 Done Done Sent false
184442024-02-12T01:42:26.746Z 22 Acked 1021 Write 5 Done Done Sent false
184452024-02-12T01:42:26.746Z 23 Acked 1022 Write 8 Done Done Sent false
184462024-02-12T01:42:26.746Z 24 Acked 1023 Write 5 Done Done Sent false
184472024-02-12T01:42:26.746Z 25 Acked 1024 Write 3 Done Done Sent false
184482024-02-12T01:42:26.746Z 26 Acked 1025 Flush 0 Done Done Sent false
184492024-02-12T01:42:26.746Z 27 Acked 1026 Write 5 Done Done Sent false
184502024-02-12T01:42:26.746Z 28 Acked 1027 Read 4 Done Done Sent false
184512024-02-12T01:42:26.746Z 29 Acked 1028 Write 5 Done Done Sent false
184522024-02-12T01:42:26.746Z 30 Acked 1029 Read 5 Done Done Sent false
184532024-02-12T01:42:26.746Z STATES DS:0 DS:1 DS:2 TOTAL
184542024-02-12T01:42:26.746Z New 0 0 0 0
184552024-02-12T01:42:26.746Z Sent 0 0 28 28
184562024-02-12T01:42:26.746Z Done 28 28 0 56
184572024-02-12T01:42:26.746Z Skipped 0 0 0 0
184582024-02-12T01:42:26.746Z Error 0 0 0 0
184592024-02-12T01:42:26.746ZLast Flush: 1025 1025 1001
184602024-02-12T01:42:26.746ZDownstairs last five completed: 1001 1000
184612024-02-12T01:42:26.746ZUpstairs last five completed: 30 29 28 27 26
184622024-02-12T01:42:26.746Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:25.61194654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1799}
184632024-02-12T01:42:26.746ZWrote out file "/tmp/test_repair_verify.data"
184642024-02-12T01:42:26.750Z
184652024-02-12T01:42:26.753ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
184662024-02-12T01:42:26.766ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184672024-02-12T01:42:26.766Z 0 000-019 164 164 164 294 294 294 F F F
184682024-02-12T01:42:26.766Z 1 020-039 165 165 165 297 297 297 F F F
184692024-02-12T01:42:26.766Z 2 040-059 165 165 165 297 297 297 F F F
184702024-02-12T01:42:26.766Z 3 060-079 159 159 159 281 281 281 F F F
184712024-02-12T01:42:26.766Z 4 080-099 164 164 164 292 292 292 F F F
184722024-02-12T01:42:26.766Z 5 100-119 166 166 157 302 302 278 F F F <---
184732024-02-12T01:42:26.766Z 6 120-139 166 166 166 298 298 298 F F F
184742024-02-12T01:42:26.766Z 7 140-159 166 166 166 302 302 298 F F F <---
184752024-02-12T01:42:26.766Z 8 160-179 163 163 163 290 290 290 F F F
184762024-02-12T01:42:26.766Z 9 180-199 163 163 163 290 290 290 F F F
184772024-02-12T01:42:26.766Z 10 200-219 164 164 164 291 291 291 F F F
184782024-02-12T01:42:26.766Z 11 220-239 166 166 152 302 302 270 F F F <---
184792024-02-12T01:42:26.766Z 12 240-259 163 163 163 290 290 290 T T F <---
184802024-02-12T01:42:26.766Z 13 260-279 157 157 157 278 278 278 T T F <---
184812024-02-12T01:42:26.770Z 14 280-299 164 164 164 294 294 294 F F F
184822024-02-12T01:42:26.770Z 15 300-319 162 162 162 289 289 289 F F F
184832024-02-12T01:42:26.770Z 16 320-339 159 159 159 281 281 281 F F F
184842024-02-12T01:42:26.770Z 17 340-359 160 160 160 285 285 285 F F F
184852024-02-12T01:42:26.770Z 18 360-379 166 166 160 302 302 285 F F F <---
184862024-02-12T01:42:26.770Z 19 380-399 153 153 153 271 271 271 F F F
184872024-02-12T01:42:26.770Z 20 400-419 163 163 163 290 290 290 F F F
184882024-02-12T01:42:26.770Z 21 420-439 165 165 165 297 297 297 F F F
184892024-02-12T01:42:26.770Z 22 440-459 166 166 164 301 301 296 F F F <---
184902024-02-12T01:42:26.770Z 23 460-479 160 160 160 284 284 284 F F F
184912024-02-12T01:42:26.770Z 24 480-499 160 160 160 284 284 284 F F F
184922024-02-12T01:42:26.770Z 25 500-519 166 166 164 302 302 294 F F F <---
184932024-02-12T01:42:26.770Z 26 520-539 164 164 164 296 296 296 F F F
184942024-02-12T01:42:26.770Z 27 540-559 166 166 164 300 300 296 F F F <---
184952024-02-12T01:42:26.770Z 28 560-579 162 162 162 289 289 289 F F F
184962024-02-12T01:42:26.770Z 29 580-599 162 162 162 289 289 289 F F F
184972024-02-12T01:42:26.770ZMax gen: 166, Max flush: 302
184982024-02-12T01:42:26.770ZError: Difference in extent metadata found!
184992024-02-12T01:42:26.770ZA Difference in extent metadata is expected here
185002024-02-12T01:42:26.771ZOn loop 83
185012024-02-12T01:42:26.771Z
185022024-02-12T01:42:26.774ZVerifying data now
185032024-02-12T01:42:31.239ZNo change after verify
185042024-02-12T01:42:31.239ZLoop: 83 Downstairs dump after verify (and repair):
185052024-02-12T01:42:31.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185062024-02-12T01:42:31.253Z 0 000-019 164 164 164 294 294 294 F F F
185072024-02-12T01:42:31.253Z 1 020-039 165 165 165 297 297 297 F F F
185082024-02-12T01:42:31.253Z 2 040-059 165 165 165 297 297 297 F F F
185092024-02-12T01:42:31.253Z 3 060-079 159 159 159 281 281 281 F F F
185102024-02-12T01:42:31.253Z 4 080-099 164 164 164 292 292 292 F F F
185112024-02-12T01:42:31.254Z 5 100-119 166 166 166 302 302 302 F F F
185122024-02-12T01:42:31.254Z 6 120-139 166 166 166 298 298 298 F F F
185132024-02-12T01:42:31.257Z 7 140-159 166 166 166 302 302 302 F F F
185142024-02-12T01:42:31.257Z 8 160-179 163 163 163 290 290 290 F F F
185152024-02-12T01:42:31.257Z 9 180-199 163 163 163 290 290 290 F F F
185162024-02-12T01:42:31.257Z 10 200-219 164 164 164 291 291 291 F F F
185172024-02-12T01:42:31.257Z 11 220-239 166 166 166 302 302 302 F F F
185182024-02-12T01:42:31.257Z 12 240-259 167 167 167 303 303 303 F F F
185192024-02-12T01:42:31.257Z 13 260-279 167 167 167 303 303 303 F F F
185202024-02-12T01:42:31.257Z 14 280-299 164 164 164 294 294 294 F F F
185212024-02-12T01:42:31.257Z 15 300-319 162 162 162 289 289 289 F F F
185222024-02-12T01:42:31.257Z 16 320-339 159 159 159 281 281 281 F F F
185232024-02-12T01:42:31.257Z 17 340-359 160 160 160 285 285 285 F F F
185242024-02-12T01:42:31.257Z 18 360-379 166 166 166 302 302 302 F F F
185252024-02-12T01:42:31.257Z 19 380-399 153 153 153 271 271 271 F F F
185262024-02-12T01:42:31.258Z 20 400-419 163 163 163 290 290 290 F F F
185272024-02-12T01:42:31.258Z 21 420-439 165 165 165 297 297 297 F F F
185282024-02-12T01:42:31.258Z 22 440-459 166 166 166 301 301 301 F F F
185292024-02-12T01:42:31.258Z 23 460-479 160 160 160 284 284 284 F F F
185302024-02-12T01:42:31.258Z 24 480-499 160 160 160 284 284 284 F F F
185312024-02-12T01:42:31.258Z 25 500-519 166 166 166 302 302 302 F F F
185322024-02-12T01:42:31.258Z 26 520-539 164 164 164 296 296 296 F F F
185332024-02-12T01:42:31.258Z 27 540-559 166 166 166 300 300 300 F F F
185342024-02-12T01:42:31.258Z 28 560-579 162 162 162 289 289 289 F F F
185352024-02-12T01:42:31.258Z 29 580-599 162 162 162 289 289 289 F F F
185362024-02-12T01:42:31.258ZMax gen: 167, Max flush: 303
185372024-02-12T01:42:31.258Z
185382024-02-12T01:42:31.258ZBegin loop 84
185392024-02-12T01:42:31.258ZDownstairs to restart: 2
185402024-02-12T01:42:31.274Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.1424963Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
185412024-02-12T01:42:31.274ZCrucible runtime is spawned
185422024-02-12T01:42:31.279Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.142851313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
185432024-02-12T01:42:31.279Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.142888962Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
185442024-02-12T01:42:31.279Z{"msg":"Crucible 75436247-b68b-45b9-8e81-6b16b09ab8fe has session id: e1fb5e94-5ccb-427b-843d-025a256b1c70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.142910451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185452024-02-12T01:42:31.279Z{"msg":"Upstairs opts: Upstairs UUID: 75436247-b68b-45b9-8e81-6b16b09ab8fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.142961338Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185462024-02-12T01:42:31.279Z{"msg":"Crucible stats registered with UUID: 75436247-b68b-45b9-8e81-6b16b09ab8fe","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143027555Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185472024-02-12T01:42:31.279Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143086342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
185482024-02-12T01:42:31.280Z{"msg":"75436247-b68b-45b9-8e81-6b16b09ab8fe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143116521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185492024-02-12T01:42:31.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143156309Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185502024-02-12T01:42:31.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143195997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185512024-02-12T01:42:31.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143224416Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185522024-02-12T01:42:31.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143250714Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185532024-02-12T01:42:31.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143276153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185542024-02-12T01:42:31.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143300762Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185552024-02-12T01:42:31.280Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.14333439Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185562024-02-12T01:42:31.280Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143367109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185572024-02-12T01:42:31.280Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143390718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185582024-02-12T01:42:31.280Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:30.143419406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185592024-02-12T01:42:31.280Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:30.143464054Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185602024-02-12T01:42:31.280Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143511172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185612024-02-12T01:42:31.280Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143539451Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185622024-02-12T01:42:31.280Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:30.143568299Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185632024-02-12T01:42:31.280Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143620067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185642024-02-12T01:42:31.280Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143636226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185652024-02-12T01:42:31.280Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.143657155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185662024-02-12T01:42:31.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.144026547Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185672024-02-12T01:42:31.280Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.144060686Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185682024-02-12T01:42:31.280Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.144075685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185692024-02-12T01:42:31.280Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.144091854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185702024-02-12T01:42:31.281Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.14459235Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185712024-02-12T01:42:31.281Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.144724544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185722024-02-12T01:42:31.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.145130215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185732024-02-12T01:42:31.281Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.145174993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185742024-02-12T01:42:31.281Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.145202351Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185752024-02-12T01:42:31.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.14522968Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185762024-02-12T01:42:31.281Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.145254649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185772024-02-12T01:42:31.281Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:30.145285997Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185782024-02-12T01:42:32.276Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.143836977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185792024-02-12T01:42:32.281Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.14397506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185802024-02-12T01:42:32.281Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.144838969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185812024-02-12T01:42:32.281Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.144875018Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185822024-02-12T01:42:32.281Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.145383913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185832024-02-12T01:42:32.281Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.145892649Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185842024-02-12T01:42:32.281Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.145951396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185852024-02-12T01:42:32.281Z{"msg":"[0]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 302, 298, 302, 290, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.145988074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185862024-02-12T01:42:32.281Z{"msg":"[0]R generation[0..12]: [164, 165, 165, 159, 164, 166, 166, 166, 163, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146004313Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185872024-02-12T01:42:32.281Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146019743Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185882024-02-12T01:42:32.281Z{"msg":"[1]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 302, 298, 302, 290, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146040792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185892024-02-12T01:42:32.281Z{"msg":"[1]R generation[0..12]: [164, 165, 165, 159, 164, 166, 166, 166, 163, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146055571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185902024-02-12T01:42:32.281Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.1460765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185912024-02-12T01:42:32.281Z{"msg":"[2]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 302, 298, 302, 290, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146090849Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185922024-02-12T01:42:32.281Z{"msg":"[2]R generation[0..12]: [164, 165, 165, 159, 164, 166, 166, 166, 163, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146105309Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185932024-02-12T01:42:32.281Z{Wait for a query_work_queue command to finish before sending IO
185942024-02-12T01:42:32.281Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146129067Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185952024-02-12T01:42:32.281Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146144447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185962024-02-12T01:42:32.281Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146165446Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185972024-02-12T01:42:32.281Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146179375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185982024-02-12T01:42:32.281Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146211944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
185992024-02-12T01:42:32.282Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146240342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186002024-02-12T01:42:32.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146261781Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186012024-02-12T01:42:32.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.14629618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186022024-02-12T01:42:32.282Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146329778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186032024-02-12T01:42:32.282Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146366306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186042024-02-12T01:42:32.282Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186052024-02-12T01:42:32.282Z2024-02-12T01:42:31.146401395Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186062024-02-12T01:42:32.282Z{"msg":"75436247-b68b-45b9-8e81-6b16b09ab8fe is now active with session: e1fb5e94-5ccb-427b-843d-025a256b1c70","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146431123Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186072024-02-12T01:42:32.282Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146457712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807,"session_id":"e1fb5e94-5ccb-427b-843d-025a256b1c70"}
186082024-02-12T01:42:32.282Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:31.146471751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
186092024-02-12T01:42:32.282ZLoading write count information from file "/tmp/test_repair_verify.data"
186102024-02-12T01:42:32.282ZRun Repair workload
186112024-02-12T01:42:32.282Z01/30 Write block 467 len 40960 data: 11 13 12 11 11 12 16 17 15 12
186122024-02-12T01:42:32.285Z02/30 Read block 559 len 16384
186132024-02-12T01:42:32.285Z03/30 Write block 580 len 4096 data: 10
186142024-02-12T01:42:32.286Z04/30 Read block 569 len 4096
186152024-02-12T01:42:32.286Z05/30 Read block 299 len 36864
186162024-02-12T01:42:32.286Z06/30 Write block 262 len 24576 data: 9 9 10 13 15 16
186172024-02-12T01:42:32.286Z07/30 Write block 176 len 16384 data: 17 15 17 18
186182024-02-12T01:42:32.291Z08/30 Read block 368 len 12288
186192024-02-12T01:42:32.291Z09/30 Read block 10 len 4096
186202024-02-12T01:42:32.291Z10/30 Write block 139 len 28672 data: 17 18 20 21 19 19 18
186212024-02-12T01:42:32.292Z11/30 Write block 281 len 32768 data: 12 16 10 9 10 9 12 15
186222024-02-12T01:42:32.292Z12/30 Read block 268 len 20480
186232024-02-12T01:42:32.295Z13/30 Read block 43 len 16384
186242024-02-12T01:42:32.295Z14/30 Read block 438 len 16384
186252024-02-12T01:42:32.295Z15/30 Read block 288 len 8192
186262024-02-12T01:42:32.295Z16/30 Read block 198 len 32768
186272024-02-12T01:42:32.295Z17/30 Read block 440 len 12288
186282024-02-12T01:42:32.295Z18/30 Write block 506 len 40960 data: 11 12 9 8 11 13 11 10 9 12
186292024-02-12T01:42:32.298Z19/30 Write block 113 len 12288 data: 12 14 14
186302024-02-12T01:42:32.298Z20/30 Read block 498 len 16384
186312024-02-12T01:42:32.298Z21/30 Read block 20 len 12288
186322024-02-12T01:42:32.298Z22/30 Read block 42 len 16384
186332024-02-12T01:42:32.298Z23/30 Flush
186342024-02-12T01:42:32.302Z24/30 Write block 444 len 36864 data: 10 9 9 10 11 11 10 7 7
186352024-02-12T01:42:32.305Z25/30 Read block 360 len 28672
186362024-02-12T01:42:32.305Z26/30 Read block 553 len 8192
186372024-02-12T01:42:32.305Z27/30 Read block 516 len 32768
186382024-02-12T01:42:32.308Z28/30 Write block 571 len 32768 data: 15 16 19 18 16 12 12 10
186392024-02-12T01:42:32.308Z29/30 Write block 475 len 32768 data: 16 13 13 11 15 16 15 15
186402024-02-12T01:42:32.308Z30/30 Read block 146 len 8192
186412024-02-12T01:42:32.312Z----------------------------------------------------------------
186422024-02-12T01:42:32.312Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
186432024-02-12T01:42:32.312ZStates: Active Active Active
186442024-02-12T01:42:32.312ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
186452024-02-12T01:42:32.312Z 1 Acked 1000 Write 10 Done Done Done false
186462024-02-12T01:42:32.312Z 2 Acked 1001 Read 4 Done Done Done false
186472024-02-12T01:42:32.312Z 3 Acked 1002 Write 1 Done Done Done false
186482024-02-12T01:42:32.312Z 4 Acked 1003 Read 1 Done Done Done false
186492024-02-12T01:42:32.312Z 5 Acked 1004 Read 9 Done Done Done false
186502024-02-12T01:42:32.312Z 6 Acked 1005 Write 6 Done Done Done false
186512024-02-12T01:42:32.312Z 7 Acked 1006 Write 4 Done Done Sent false
186522024-02-12T01:42:32.312Z 8 Acked 1007 Read 3 Done Done Sent false
186532024-02-12T01:42:32.312Z 9 Acked 1008 Read 1 Done Done Sent false
186542024-02-12T01:42:32.312Z 10 Acked 1009 Write 7 Done Done Sent false
186552024-02-12T01:42:32.312Z 11 Acked 1010 Write 8 Done Done Sent false
186562024-02-12T01:42:32.312Z 12 Acked 1011 Read 5 Done Done Sent false
186572024-02-12T01:42:32.312Z 13 Acked 1012 Read 4 Done Done Sent false
186582024-02-12T01:42:32.312Z 14 Acked 1013 Read 4 Done Done Sent false
186592024-02-12T01:42:32.312Z 15 Acked 1014 Read 2 Done Done Sent false
186602024-02-12T01:42:32.312Z 16 Acked 1015 Read 8 Done Done Sent false
186612024-02-12T01:42:32.312Z 17 Acked 1016 Read 3 Done Done Sent false
186622024-02-12T01:42:32.313Z 18 Acked 1017 Write 10 Done Done Sent false
186632024-02-12T01:42:32.313Z 19 Acked 1018 Write 3 Done Done Sent false
186642024-02-12T01:42:32.313Z 20 Acked 1019 Read 4 Done Done Sent false
186652024-02-12T01:42:32.313Z 21 Acked 1020 Read 3 Done Done Sent false
186662024-02-12T01:42:32.313Z 22 Acked 1021 Read 4 Done Done Sent false
186672024-02-12T01:42:32.313Z 23 Acked 1022 Flush 0 Done Done Sent false
186682024-02-12T01:42:32.313Z 24 Acked 1023 Write 9 Done Done Sent false
186692024-02-12T01:42:32.313Z 25 Acked 1024 Read 7 Done Done Sent false
186702024-02-12T01:42:32.313Z 26 Acked 1025 Read 2 Done Done Sent false
186712024-02-12T01:42:32.313Z 27 Acked 1026 Read 8 Done Done Sent false
186722024-02-12T01:42:32.313Z 28 Acked 1027 Write 8 Done Done Sent false
186732024-02-12T01:42:32.313Z 29 Acked 1028 Write 8 Done Done Sent false
186742024-02-12T01:42:32.313Z 30 Acked 1029 Read 2 Sent Done Sent false
186752024-02-12T01:42:32.313Z STATES DS:0 DS:1 DS:2 TOTAL
186762024-02-12T01:42:32.313Z New 0 0 0 0
186772024-02-12T01:42:32.313Z Sent 1 0 24 25
186782024-02-12T01:42:32.313Z Done 29 30 6 65
186792024-02-12T01:42:32.313Z Skipped 0 0 0 0
186802024-02-12T01:42:32.313Z Error 0 0 0 0
186812024-02-12T01:42:32.313ZLast Flush: 1022 1022 0
186822024-02-12T01:42:32.313ZDownstairs last five completed:
186832024-02-12T01:42:32.313ZUpstairs last five completed: 30 29 28 27 26
186842024-02-12T01:42:32.313Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:31.177135498Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1807}
186852024-02-12T01:42:32.313ZWrote out file "/tmp/test_repair_verify.data"
186862024-02-12T01:42:32.317Z
186872024-02-12T01:42:32.320ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
186882024-02-12T01:42:32.334ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186892024-02-12T01:42:32.334Z 0 000-019 164 164 164 294 294 294 F F F
186902024-02-12T01:42:32.334Z 1 020-039 165 165 165 297 297 297 F F F
186912024-02-12T01:42:32.334Z 2 040-059 165 165 165 297 297 297 F F F
186922024-02-12T01:42:32.334Z 3 060-079 159 159 159 281 281 281 F F F
186932024-02-12T01:42:32.334Z 4 080-099 164 164 164 292 292 292 F F F
186942024-02-12T01:42:32.334Z 5 100-119 168 168 166 304 304 302 F F F <---
186952024-02-12T01:42:32.334Z 6 120-139 168 168 166 304 304 298 F F F <---
186962024-02-12T01:42:32.334Z 7 140-159 168 168 166 304 304 302 F F F <---
186972024-02-12T01:42:32.334Z 8 160-179 168 168 163 304 304 290 F F F <---
186982024-02-12T01:42:32.335Z 9 180-199 163 163 163 290 290 290 F F F
186992024-02-12T01:42:32.335Z 10 200-219 164 164 164 291 291 291 F F F
187002024-02-12T01:42:32.335Z 11 220-239 166 166 166 302 302 302 F F F
187012024-02-12T01:42:32.335Z 12 240-259 167 167 167 303 303 303 F F F
187022024-02-12T01:42:32.335Z 13 260-279 168 168 167 304 304 303 F F T <---
187032024-02-12T01:42:32.335Z 14 280-299 168 168 164 304 304 294 F F F <---
187042024-02-12T01:42:32.338Z 15 300-319 162 162 162 289 289 289 F F F
187052024-02-12T01:42:32.338Z 16 320-339 159 159 159 281 281 281 F F F
187062024-02-12T01:42:32.338Z 17 340-359 160 160 160 285 285 285 F F F
187072024-02-12T01:42:32.338Z 18 360-379 166 166 166 302 302 302 F F F
187082024-02-12T01:42:32.338Z 19 380-399 153 153 153 271 271 271 F F F
187092024-02-12T01:42:32.338Z 20 400-419 163 163 163 290 290 290 F F F
187102024-02-12T01:42:32.338Z 21 420-439 165 165 165 297 297 297 F F F
187112024-02-12T01:42:32.338Z 22 440-459 166 166 166 301 301 301 T T F <---
187122024-02-12T01:42:32.338Z 23 460-479 168 168 160 304 304 284 T T T <---
187132024-02-12T01:42:32.338Z 24 480-499 160 160 160 284 284 284 T T F <---
187142024-02-12T01:42:32.338Z 25 500-519 168 168 166 304 304 302 F F F <---
187152024-02-12T01:42:32.339Z 26 520-539 164 164 164 296 296 296 F F F
187162024-02-12T01:42:32.339Z 27 540-559 166 166 166 300 300 300 F F F
187172024-02-12T01:42:32.339Z 28 560-579 162 162 162 289 289 289 T T F <---
187182024-02-12T01:42:32.339Z 29 580-599 168 168 162 304 304 289 F F T <---
187192024-02-12T01:42:32.339ZMax gen: 168, Max flush: 304
187202024-02-12T01:42:32.339ZError: Difference in extent metadata found!
187212024-02-12T01:42:32.339ZA Difference in extent metadata is expected here
187222024-02-12T01:42:32.339ZOn loop 84
187232024-02-12T01:42:32.339Z
187242024-02-12T01:42:32.342ZVerifying data now
187252024-02-12T01:42:37.940ZNo change after verify
187262024-02-12T01:42:37.940ZLoop: 84 Downstairs dump after verify (and repair):
187272024-02-12T01:42:37.955ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187282024-02-12T01:42:37.956Z 0 000-019 164 164 164 294 294 294 F F F
187292024-02-12T01:42:37.956Z 1 020-039 165 165 165 297 297 297 F F F
187302024-02-12T01:42:37.956Z 2 040-059 165 165 165 297 297 297 F F F
187312024-02-12T01:42:37.956Z 3 060-079 159 159 159 281 281 281 F F F
187322024-02-12T01:42:37.956Z 4 080-099 164 164 164 292 292 292 F F F
187332024-02-12T01:42:37.956Z 5 100-119 168 168 168 304 304 304 F F F
187342024-02-12T01:42:37.956Z 6 120-139 168 168 168 304 304 304 F F F
187352024-02-12T01:42:37.956Z 7 140-159 168 168 168 304 304 304 F F F
187362024-02-12T01:42:37.956Z 8 160-179 168 168 168 304 304 304 F F F
187372024-02-12T01:42:37.956Z 9 180-199 163 163 163 290 290 290 F F F
187382024-02-12T01:42:37.959Z 10 200-219 164 164 164 291 291 291 F F F
187392024-02-12T01:42:37.959Z 11 220-239 166 166 166 302 302 302 F F F
187402024-02-12T01:42:37.959Z 12 240-259 167 167 167 303 303 303 F F F
187412024-02-12T01:42:37.959Z 13 260-279 168 168 168 304 304 304 F F F
187422024-02-12T01:42:37.959Z 14 280-299 168 168 168 304 304 304 F F F
187432024-02-12T01:42:37.959Z 15 300-319 162 162 162 289 289 289 F F F
187442024-02-12T01:42:37.959Z 16 320-339 159 159 159 281 281 281 F F F
187452024-02-12T01:42:37.959Z 17 340-359 160 160 160 285 285 285 F F F
187462024-02-12T01:42:37.960Z 18 360-379 166 166 166 302 302 302 F F F
187472024-02-12T01:42:37.960Z 19 380-399 153 153 153 271 271 271 F F F
187482024-02-12T01:42:37.960Z 20 400-419 163 163 163 290 290 290 F F F
187492024-02-12T01:42:37.960Z 21 420-439 165 165 165 297 297 297 F F F
187502024-02-12T01:42:37.960Z 22 440-459 169 169 169 305 305 305 F F F
187512024-02-12T01:42:37.960Z 23 460-479 169 169 169 305 305 305 F F F
187522024-02-12T01:42:37.960Z 24 480-499 169 169 169 305 305 305 F F F
187532024-02-12T01:42:37.960Z 25 500-519 168 168 168 304 304 304 F F F
187542024-02-12T01:42:37.960Z 26 520-539 164 164 164 296 296 296 F F F
187552024-02-12T01:42:37.960Z 27 540-559 166 166 166 300 300 300 F F F
187562024-02-12T01:42:37.960Z 28 560-579 169 169 169 305 305 305 F F F
187572024-02-12T01:42:37.960Z 29 580-599 168 168 168 304 304 304 F F F
187582024-02-12T01:42:37.960ZMax gen: 169, Max flush: 305
187592024-02-12T01:42:37.960Z
187602024-02-12T01:42:37.960ZBegin loop 85
187612024-02-12T01:42:37.960ZDownstairs to restart: 1
187622024-02-12T01:42:37.978Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.844873866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
187632024-02-12T01:42:37.978Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845159443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
187642024-02-12T01:42:37.978Z{"msg":"Crucible runtime is spawned
187652024-02-12T01:42:37.978ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84520335Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
187662024-02-12T01:42:37.982Z{"msg":"Crucible 9f4a6d93-a2d8-4266-9a06-58cd879070b7 has session id: d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845243328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187672024-02-12T01:42:37.982Z{"msg":"Upstairs opts: Upstairs UUID: 9f4a6d93-a2d8-4266-9a06-58cd879070b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845310085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187682024-02-12T01:42:37.982Z{"msg":"Crucible stats registered with UUID: 9f4a6d93-a2d8-4266-9a06-58cd879070b7","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845353363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187692024-02-12T01:42:37.982Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845379102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
187702024-02-12T01:42:37.982Z{"msg":"9f4a6d93-a2d8-4266-9a06-58cd879070b7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845403881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187712024-02-12T01:42:37.982Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845447589Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187722024-02-12T01:42:37.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845490877Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187732024-02-12T01:42:37.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845524715Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187742024-02-12T01:42:37.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845567693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187752024-02-12T01:42:37.983Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845621761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187762024-02-12T01:42:37.983Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845666838Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187772024-02-12T01:42:37.983Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845711916Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187782024-02-12T01:42:37.983Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845768694Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187792024-02-12T01:42:37.983Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845805642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187802024-02-12T01:42:37.983Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.8458366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187812024-02-12T01:42:37.983Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845876278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187822024-02-12T01:42:37.983Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.845895707Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187832024-02-12T01:42:37.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.846413343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187842024-02-12T01:42:37.983Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.846449321Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187852024-02-12T01:42:37.983Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84646466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187862024-02-12T01:42:37.983Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84647886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187872024-02-12T01:42:37.983Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847043443Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187882024-02-12T01:42:37.983Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847181866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187892024-02-12T01:42:37.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847535489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187902024-02-12T01:42:37.983Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847582917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187912024-02-12T01:42:37.983Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847610426Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187922024-02-12T01:42:37.983Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847648874Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187932024-02-12T01:42:37.983Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847675012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187942024-02-12T01:42:37.983Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847711161Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187952024-02-12T01:42:37.984Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847818016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187962024-02-12T01:42:37.984Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.847854994Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187972024-02-12T01:42:37.984Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848282583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187982024-02-12T01:42:37.988Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848680755Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
187992024-02-12T01:42:37.988Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848723333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188002024-02-12T01:42:37.988Z{"msg":"[0]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848748701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188012024-02-12T01:42:37.988Z{"msg":"[0]R generation[0..12]: [164, 165, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84878725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188022024-02-12T01:42:37.988Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848821038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188032024-02-12T01:42:37.988Z{"msg":"[1]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848837517Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188042024-02-12T01:42:37.988Z{"msg":"[1]R generation[0..12]: [164, 165, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
188052024-02-12T01:42:37.988Z2024-02-12T01:42:36.848858986Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188062024-02-12T01:42:37.988Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848930672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188072024-02-12T01:42:37.988Z{"msg":"[2]R flush_numbers[0..12]: [294, 297, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.848947982Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188082024-02-12T01:42:37.988Z{"msg":"[2]R generation[0..12]: [164, 165, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84897467Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188092024-02-12T01:42:37.988Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84898907Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188102024-02-12T01:42:37.988Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849013959Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188112024-02-12T01:42:37.988Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849027778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188122024-02-12T01:42:37.988Z{"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849047037Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188132024-02-12T01:42:37.988Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849060136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188142024-02-12T01:42:37.988Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849079425Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188152024-02-12T01:42:37.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849091525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188162024-02-12T01:42:37.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849105634Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188172024-02-12T01:42:37.989Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849125863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188182024-02-12T01:42:37.989Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849139393Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188192024-02-12T01:42:37.989Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849152632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188202024-02-12T01:42:37.989Z{"msg":"9f4a6d93-a2d8-4266-9a06-58cd879070b7 is now active with session: d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849172331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188212024-02-12T01:42:37.989Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.849185411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815,"session_id":"d1d71906-dcf4-43b8-bfbd-1bb23e74ce5c"}
188222024-02-12T01:42:37.989Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:36.84920521Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
188232024-02-12T01:42:37.989ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
188242024-02-12T01:42:37.989ZLoading write count information from file "/tmp/test_repair_verify.data"
188252024-02-12T01:42:37.989ZRun Repair workload
188262024-02-12T01:42:37.989Z01/30 Write block 19 len 28672 data: 7 8 10 11 11 10 13
188272024-02-12T01:42:37.989Z02/30 Read block 53 len 36864
188282024-02-12T01:42:37.989Z03/30 Read block 397 len 16384
188292024-02-12T01:42:37.989Z04/30 Read block 207 len 12288
188302024-02-12T01:42:37.992Z05/30 Write block 305 len 28672 data: 13 14 16 17 16 16 15
188312024-02-12T01:42:37.993Z06/30 Write block 23 len 16384 data: 12 11 14 13
188322024-02-12T01:42:37.993Z07/30 Flush
188332024-02-12T01:42:37.993Z08/30 Write block 457 len 12288 data: 10 14 11
188342024-02-12T01:42:37.993Z09/30 Read block 193 len 40960
188352024-02-12T01:42:37.995Z10/30 Read block 370 len 40960
188362024-02-12T01:42:37.996Z11/30 Read block 149 len 32768
188372024-02-12T01:42:37.996Z12/30 Read block 267 len 4096
188382024-02-12T01:42:37.999Z13/30 Write block 487 len 32768 data: 15 16 12 14 12 12 12 12
188392024-02-12T01:42:37.999Z14/30 Read block 550 len 8192
188402024-02-12T01:42:37.999Z15/30 Write block 313 len 32768 data: 14 12 13 9 10 11 11 11
188412024-02-12T01:42:37.999Z16/30 Read block 307 len 20480
188422024-02-12T01:42:38.002Z17/30 Write block 271 len 16384 data: 17 17 17 17
188432024-02-12T01:42:38.002Z18/30 Read block 346 len 12288
188442024-02-12T01:42:38.002Z19/30 Read block 71 len 36864
188452024-02-12T01:42:38.005Z20/30 Write block 296 len 32768 data: 11 12 10 9 9 8 7 8
188462024-02-12T01:42:38.005Z21/30 Read block 144 len 24576
188472024-02-12T01:42:38.005Z22/30 Read block 478 len 36864
188482024-02-12T01:42:38.011Z23/30 Flush
188492024-02-12T01:42:38.014Z24/30 Write block 265 len 32768 data: 14 16 17 15 14 16 18 18
188502024-02-12T01:42:38.014Z25/30 Write block 585 len 40960 data: 13 13 10 10 11 14 11 12 10 9
188512024-02-12T01:42:38.014Z26/30 Read block 243 len 32768
188522024-02-12T01:42:38.014Z27/30 Read block 276 len 24576
188532024-02-12T01:42:38.014Z28/30 Write block 489 len 8192 data: 13 15
188542024-02-12T01:42:38.014Z29/30 Read block 383 len 24576
188552024-02-12T01:42:38.014Z30/30 Read block 117 len 28672
188562024-02-12T01:42:38.017Z----------------------------------------------------------------
188572024-02-12T01:42:38.017Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
188582024-02-12T01:42:38.017ZStates: Active Active Active
188592024-02-12T01:42:38.017ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
188602024-02-12T01:42:38.017Z 1 Acked 1000 Write 7 Done Done Done false
188612024-02-12T01:42:38.017Z 2 Acked 1001 Read 9 Done Done Done false
188622024-02-12T01:42:38.017Z 3 Acked 1002 Read 4 Done Sent Done false
188632024-02-12T01:42:38.017Z 4 Acked 1003 Read 3 Done Sent Done false
188642024-02-12T01:42:38.017Z 5 Acked 1004 Write 7 Done Sent Done false
188652024-02-12T01:42:38.018Z 6 Acked 1005 Write 4 Done Sent Done false
188662024-02-12T01:42:38.018Z 7 Acked 1006 Flush 0 Done Sent Done false
188672024-02-12T01:42:38.018Z 8 Acked 1007 Write 3 Done Sent Done false
188682024-02-12T01:42:38.018Z 9 Acked 1008 Read 10 Done Sent Done false
188692024-02-12T01:42:38.018Z 10 Acked 1009 Read 10 Done Sent Done false
188702024-02-12T01:42:38.018Z 11 Acked 1010 Read 8 Done Sent Done false
188712024-02-12T01:42:38.018Z 12 Acked 1011 Read 1 Done Sent Done false
188722024-02-12T01:42:38.018Z 13 Acked 1012 Write 8 Done Sent Done false
188732024-02-12T01:42:38.018Z 14 Acked 1013 Read 2 Done Sent Done false
188742024-02-12T01:42:38.018Z 15 Acked 1014 Write 8 Done Sent Done false
188752024-02-12T01:42:38.018Z 16 Acked 1015 Read 5 Done Sent Done false
188762024-02-12T01:42:38.018Z 17 Acked 1016 Write 4 Done Sent Done false
188772024-02-12T01:42:38.018Z 18 Acked 1017 Read 3 Done Sent Done false
188782024-02-12T01:42:38.018Z 19 Acked 1018 Read 9 Done Sent Done false
188792024-02-12T01:42:38.018Z 20 Acked 1019 Write 8 Done Sent Done false
188802024-02-12T01:42:38.018Z 21 Acked 1020 Read 6 Done Sent Done false
188812024-02-12T01:42:38.018Z 22 Acked 1021 Read 9 Done Sent Done false
188822024-02-12T01:42:38.018Z 23 Acked 1022 Flush 0 Done Sent Done false
188832024-02-12T01:42:38.018Z 24 Acked 1023 Write 8 Done Sent Done false
188842024-02-12T01:42:38.018Z 25 Acked 1024 Write 10 Done Sent Done false
188852024-02-12T01:42:38.018Z 26 Acked 1025 Read 8 Done Sent Done false
188862024-02-12T01:42:38.018Z 27 Acked 1026 Read 6 Done Sent Done false
188872024-02-12T01:42:38.018Z 28 Acked 1027 Write 2 Done Sent Done false
188882024-02-12T01:42:38.019Z 29 Acked 1028 Read 6 Done Sent Done false
188892024-02-12T01:42:38.019Z 30 Acked 1029 Read 7 Done Sent Done false
188902024-02-12T01:42:38.019Z STATES DS:0 DS:1 DS:2 TOTAL
188912024-02-12T01:42:38.019Z New 0 0 0 0
188922024-02-12T01:42:38.019Z Sent 0 28 0 28
188932024-02-12T01:42:38.019Z Done 30 2 30 62
188942024-02-12T01:42:38.019Z Skipped 0 0 0 0
188952024-02-12T01:42:38.019Z Error 0 0 0 0
188962024-02-12T01:42:38.019ZLast Flush: 1022 0 1022
188972024-02-12T01:42:38.019ZDownstairs last five completed:
188982024-02-12T01:42:38.019ZUpstairs last five completed: 30 29 28 27 26
188992024-02-12T01:42:38.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:36.882541759Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1815}
189002024-02-12T01:42:38.019ZWrote out file "/tmp/test_repair_verify.data"
189012024-02-12T01:42:38.022Z
189022024-02-12T01:42:38.022ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189032024-02-12T01:42:38.039ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189042024-02-12T01:42:38.039Z 0 000-019 170 164 170 306 294 306 F T F <---
189052024-02-12T01:42:38.039Z 1 020-039 170 165 170 306 297 306 F T F <---
189062024-02-12T01:42:38.039Z 2 040-059 165 165 165 297 297 297 F F F
189072024-02-12T01:42:38.039Z 3 060-079 159 159 159 281 281 281 F F F
189082024-02-12T01:42:38.039Z 4 080-099 164 164 164 292 292 292 F F F
189092024-02-12T01:42:38.039Z 5 100-119 168 168 168 304 304 304 F F F
189102024-02-12T01:42:38.039Z 6 120-139 168 168 168 304 304 304 F F F
189112024-02-12T01:42:38.039Z 7 140-159 168 168 168 304 304 304 F F F
189122024-02-12T01:42:38.039Z 8 160-179 168 168 168 304 304 304 F F F
189132024-02-12T01:42:38.039Z 9 180-199 163 163 163 290 290 290 F F F
189142024-02-12T01:42:38.039Z 10 200-219 164 164 164 291 291 291 F F F
189152024-02-12T01:42:38.039Z 11 220-239 166 166 166 302 302 302 F F F
189162024-02-12T01:42:38.039Z 12 240-259 167 167 167 303 303 303 F F F
189172024-02-12T01:42:38.039Z 13 260-279 170 168 170 307 304 307 T F T <---
189182024-02-12T01:42:38.043Z 14 280-299 170 168 170 307 304 307 F F F <---
189192024-02-12T01:42:38.043Z 15 300-319 170 162 170 307 289 307 F F F <---
189202024-02-12T01:42:38.043Z 16 320-339 170 159 170 307 281 307 F F F <---
189212024-02-12T01:42:38.043Z 17 340-359 160 160 160 285 285 285 F F F
189222024-02-12T01:42:38.043Z 18 360-379 166 166 166 302 302 302 F F F
189232024-02-12T01:42:38.043Z 19 380-399 153 153 153 271 271 271 F F F
189242024-02-12T01:42:38.043Z 20 400-419 163 163 163 290 290 290 F F F
189252024-02-12T01:42:38.043Z 21 420-439 165 165 165 297 297 297 F F F
189262024-02-12T01:42:38.043Z 22 440-459 170 169 170 307 305 307 F F F <---
189272024-02-12T01:42:38.043Z 23 460-479 169 169 169 305 305 305 F F F
189282024-02-12T01:42:38.043Z 24 480-499 170 169 170 307 305 307 T F T <---
189292024-02-12T01:42:38.043Z 25 500-519 168 168 168 304 304 304 F F F
189302024-02-12T01:42:38.043Z 26 520-539 164 164 164 296 296 296 F F F
189312024-02-12T01:42:38.043Z 27 540-559 166 166 166 300 300 300 F F F
189322024-02-12T01:42:38.043Z 28 560-579 169 169 169 305 305 305 F F F
189332024-02-12T01:42:38.043Z 29 580-599 168 168 168 304 304 304 T F T <---
189342024-02-12T01:42:38.043ZMax gen: 170, Max flush: 307
189352024-02-12T01:42:38.043ZError: Difference in extent metadata found!
189362024-02-12T01:42:38.043ZA Difference in extent metadata is expected here
189372024-02-12T01:42:38.043ZOn loop 85
189382024-02-12T01:42:38.043Z
189392024-02-12T01:42:38.047ZVerifying data now
189402024-02-12T01:42:43.526ZNo change after verify
189412024-02-12T01:42:43.526ZLoop: 85 Downstairs dump after verify (and repair):
189422024-02-12T01:42:43.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189432024-02-12T01:42:43.542Z 0 000-019 170 170 170 306 306 306 F F F
189442024-02-12T01:42:43.542Z 1 020-039 170 170 170 306 306 306 F F F
189452024-02-12T01:42:43.542Z 2 040-059 165 165 165 297 297 297 F F F
189462024-02-12T01:42:43.542Z 3 060-079 159 159 159 281 281 281 F F F
189472024-02-12T01:42:43.542Z 4 080-099 164 164 164 292 292 292 F F F
189482024-02-12T01:42:43.542Z 5 100-119 168 168 168 304 304 304 F F F
189492024-02-12T01:42:43.542Z 6 120-139 168 168 168 304 304 304 F F F
189502024-02-12T01:42:43.542Z 7 140-159 168 168 168 304 304 304 F F F
189512024-02-12T01:42:43.545Z 8 160-179 168 168 168 304 304 304 F F F
189522024-02-12T01:42:43.545Z 9 180-199 163 163 163 290 290 290 F F F
189532024-02-12T01:42:43.545Z 10 200-219 164 164 164 291 291 291 F F F
189542024-02-12T01:42:43.545Z 11 220-239 166 166 166 302 302 302 F F F
189552024-02-12T01:42:43.545Z 12 240-259 167 167 167 303 303 303 F F F
189562024-02-12T01:42:43.545Z 13 260-279 171 171 171 308 308 308 F F F
189572024-02-12T01:42:43.545Z 14 280-299 170 170 170 307 307 307 F F F
189582024-02-12T01:42:43.546Z 15 300-319 170 170 170 307 307 307 F F F
189592024-02-12T01:42:43.546Z 16 320-339 170 170 170 307 307 307 F F F
189602024-02-12T01:42:43.546Z 17 340-359 160 160 160 285 285 285 F F F
189612024-02-12T01:42:43.546Z 18 360-379 166 166 166 302 302 302 F F F
189622024-02-12T01:42:43.546Z 19 380-399 153 153 153 271 271 271 F F F
189632024-02-12T01:42:43.546Z 20 400-419 163 163 163 290 290 290 F F F
189642024-02-12T01:42:43.546Z 21 420-439 165 165 165 297 297 297 F F F
189652024-02-12T01:42:43.546Z 22 440-459 170 170 170 307 307 307 F F F
189662024-02-12T01:42:43.546Z 23 460-479 169 169 169 305 305 305 F F F
189672024-02-12T01:42:43.546Z 24 480-499 171 171 171 308 308 308 F F F
189682024-02-12T01:42:43.546Z 25 500-519 168 168 168 304 304 304 F F F
189692024-02-12T01:42:43.546Z 26 520-539 164 164 164 296 296 296 F F F
189702024-02-12T01:42:43.546Z 27 540-559 166 166 166 300 300 300 F F F
189712024-02-12T01:42:43.546Z 28 560-579 169 169 169 305 305 305 F F F
189722024-02-12T01:42:43.546Z 29 580-599 171 171 171 308 308 308 F F F
189732024-02-12T01:42:43.546ZMax gen: 171, Max flush: 308
189742024-02-12T01:42:43.546Z
189752024-02-12T01:42:43.546ZBegin loop 86
189762024-02-12T01:42:43.546ZDownstairs to restart: 0
189772024-02-12T01:42:43.565Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.431644597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
189782024-02-12T01:42:43.565Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.431943993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
189792024-02-12T01:42:43.569Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.431968972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
189802024-02-12T01:42:43.569Z{"msg":"Crucible a9ef2e6a-c131-4cf8-b404-d7f6ba954f0d has session id: b82f0195-02b6-4990-ac74-cc5c204b22dd","v":0,"name":"crucible","level":30Crucible runtime is spawned
189812024-02-12T01:42:43.570Z,"time":"2024-02-12T01:42:42.43200728Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189822024-02-12T01:42:43.570Z{"msg":"Upstairs opts: Upstairs UUID: a9ef2e6a-c131-4cf8-b404-d7f6ba954f0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432097525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189832024-02-12T01:42:43.570Z{"msg":"Crucible stats registered with UUID: a9ef2e6a-c131-4cf8-b404-d7f6ba954f0d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432141693Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189842024-02-12T01:42:43.570Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432168692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
189852024-02-12T01:42:43.570Z{"msg":"a9ef2e6a-c131-4cf8-b404-d7f6ba954f0d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432248598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189862024-02-12T01:42:43.570Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432281917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189872024-02-12T01:42:43.570Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432304306Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189882024-02-12T01:42:43.570Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432341654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189892024-02-12T01:42:43.570Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432376412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189902024-02-12T01:42:43.570Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.4324196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189912024-02-12T01:42:43.570Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432440859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189922024-02-12T01:42:43.570Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432477147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189932024-02-12T01:42:43.570Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432540404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189942024-02-12T01:42:43.570Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.43262871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189952024-02-12T01:42:43.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432691247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189962024-02-12T01:42:43.570Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.432724855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189972024-02-12T01:42:43.570Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:42.432886898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189982024-02-12T01:42:43.570Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:42.432937025Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
189992024-02-12T01:42:43.570Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:42.432981433Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190002024-02-12T01:42:43.570Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433035251Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190012024-02-12T01:42:43.570Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433061079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190022024-02-12T01:42:43.570Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433086558Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190032024-02-12T01:42:43.571Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433404723Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190042024-02-12T01:42:43.571Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433456011Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190052024-02-12T01:42:43.571Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.43348041Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190062024-02-12T01:42:43.571Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433511638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190072024-02-12T01:42:43.571Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.433995965Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190082024-02-12T01:42:43.571Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434049122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190092024-02-12T01:42:43.571Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434642934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190102024-02-12T01:42:43.571Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434679692Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190112024-02-12T01:42:43.571Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434696212Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190122024-02-12T01:42:43.571Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.4347167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190132024-02-12T01:42:43.571Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434737379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190142024-02-12T01:42:43.571Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:42.434751859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190152024-02-12T01:42:44.568Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.434209226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190162024-02-12T01:42:44.568Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.43431489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190172024-02-12T01:42:44.572Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.435133821Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190182024-02-12T01:42:44.572Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.435188709Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190192024-02-12T01:42:44.572Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.435643647Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190202024-02-12T01:42:44.572Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436246118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190212024-02-12T01:42:44.572Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436300046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190222024-02-12T01:42:44.572Z{"msg":"[0]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436344513Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190232024-02-12T01:42:44.572Z{"msg":"[0]R generation[0..12]: [170, 170, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436376102Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190242024-02-12T01:42:44.573Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436392851Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190252024-02-12T01:42:44.573Z{"msg":"[1]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.43641418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190262024-02-12T01:42:44.573Z{"msg":"[1]R generation[0..12]: [170, 170, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.43642894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190272024-02-12T01:42:44.573Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436449788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190282024-02-12T01:42:44.573Z{"msg":"[2]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 304, 304, 304, 290, 291, 302]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436465068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190292024-02-12T01:42:44.573Z{"msg":"[2]R generation[0..12]: [170, 170, 165, 159, 164, 168, 168, 168, 168, 163, 164, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436490447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190302024-02-12T01:42:44.573Z{"msg":"Wait for a query_work_queue command to finish before sending IO
190312024-02-12T01:42:44.573Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436526725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190322024-02-12T01:42:44.573Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436560933Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190332024-02-12T01:42:44.573Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436577573Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190342024-02-12T01:42:44.573Z{"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436597572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190352024-02-12T01:42:44.573Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436613791Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190362024-02-12T01:42:44.573Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436649359Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190372024-02-12T01:42:44.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436674508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190382024-02-12T01:42:44.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436712336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190392024-02-12T01:42:44.573Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436747594Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190402024-02-12T01:42:44.573Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436763873Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190412024-02-12T01:42:44.573Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436794062Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190422024-02-12T01:42:44.573Z{"msg":"a9ef2e6a-c131-4cf8-b404-d7f6ba954f0d is now active with session: b82f0195-02b6-4990-ac74-cc5c204b22dd","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.43683159Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190432024-02-12T01:42:44.573Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436857009Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823,"session_id":"b82f0195-02b6-4990-ac74-cc5c204b22dd"}
190442024-02-12T01:42:44.573Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:43.436871248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
190452024-02-12T01:42:44.573ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
190462024-02-12T01:42:44.574ZLoading write count information from file "/tmp/test_repair_verify.data"
190472024-02-12T01:42:44.574ZRun Repair workload
190482024-02-12T01:42:44.577Z01/30 Write block 167 len 32768 data: 12 12 17 16 16 13 14 14
190492024-02-12T01:42:44.577Z02/30 Read block 6 len 4096
190502024-02-12T01:42:44.577Z03/30 Read block 398 len 8192
190512024-02-12T01:42:44.577Z04/30 Read block 6 len 4096
190522024-02-12T01:42:44.577Z05/30 Write block 345 len 40960 data: 9 10 9 8 10 14 16 18 14 10
190532024-02-12T01:42:44.577Z06/30 Read block 428 len 12288
190542024-02-12T01:42:44.580Z07/30 Write block 132 len 40960 data: 15 14 14 12 15 13 15 18 19 21
190552024-02-12T01:42:44.580Z08/30 Write block 233 len 8192 data: 9 11
190562024-02-12T01:42:44.580Z09/30 Read block 598 len 8192
190572024-02-12T01:42:44.580Z10/30 Read block 138 len 36864
190582024-02-12T01:42:44.583Z11/30 Read block 501 len 32768
190592024-02-12T01:42:44.583Z12/30 Flush
190602024-02-12T01:42:44.586Z13/30 Write block 126 len 36864 data: 18 17 15 14 13 16 16 15 15
190612024-02-12T01:42:44.586Z14/30 Read block 166 len 12288
190622024-02-12T01:42:44.586Z15/30 Read block 4 len 4096
190632024-02-12T01:42:44.589Z16/30 Write block 540 len 24576 data: 13 12 12 9 9 10
190642024-02-12T01:42:44.589Z17/30 Read block 246 len 20480
190652024-02-12T01:42:44.589Z18/30 Write block 319 len 8192 data: 12 12
190662024-02-12T01:42:44.595Z19/30 Write block 234 len 24576 data: 12 11 14 16 17 17
190672024-02-12T01:42:44.595Z20/30 Read block 138 len 20480
190682024-02-12T01:42:44.595Z21/30 Write block 432 len 40960 data: 8 8 9 8 8 10 10 13 12 15
190692024-02-12T01:42:44.595Z22/30 Write block 168 len 24576 data: 13 18 17 17 14 15
190702024-02-12T01:42:44.598Z23/30 Write block 168 len 20480 data: 14 19 18 18 15
190712024-02-12T01:42:44.598Z24/30 Read block 355 len 16384
190722024-02-12T01:42:44.598Z25/30 Flush
190732024-02-12T01:42:44.601Z26/30 Write block 541 len 8192 data: 13 13
190742024-02-12T01:42:44.601Z27/30 Write block 530 len 40960 data: 14 13 15 18 17 19 19 18 16 13
190752024-02-12T01:42:44.601Z28/30 Read block 326 len 40960
190762024-02-12T01:42:44.604Z29/30 Read block 566 len 40960
190772024-02-12T01:42:44.604Z30/30 Read block 569 len 28672
190782024-02-12T01:42:44.604Z----------------------------------------------------------------
190792024-02-12T01:42:44.604Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
190802024-02-12T01:42:44.604ZStates: Active Active Active
190812024-02-12T01:42:44.604ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190822024-02-12T01:42:44.604Z 1 Acked 1000 Write 8 Done Done Done false
190832024-02-12T01:42:44.604Z 2 Acked 1001 Read 1 Done Done Done false
190842024-02-12T01:42:44.604Z 3 Acked 1002 Read 2 Done Done Done false
190852024-02-12T01:42:44.604Z 4 Acked 1003 Read 1 Done Done Done false
190862024-02-12T01:42:44.605Z 5 Acked 1004 Write 10 Done Done Done false
190872024-02-12T01:42:44.605Z 6 Acked 1005 Read 3 Done Done Done false
190882024-02-12T01:42:44.605Z 7 Acked 1006 Write 10 Sent Done Done false
190892024-02-12T01:42:44.605Z 8 Acked 1007 Write 2 Sent Done Done false
190902024-02-12T01:42:44.605Z 9 Acked 1008 Read 2 Sent Done Done false
190912024-02-12T01:42:44.605Z 10 Acked 1009 Read 9 Sent Done Done false
190922024-02-12T01:42:44.605Z 11 Acked 1010 Read 8 Sent Done Done false
190932024-02-12T01:42:44.605Z 12 Acked 1011 Flush 0 Sent Done Done false
190942024-02-12T01:42:44.605Z 13 Acked 1012 Write 9 Sent Done Done false
190952024-02-12T01:42:44.605Z 14 Acked 1013 Read 3 Sent Done Done false
190962024-02-12T01:42:44.605Z 15 Acked 1014 Read 1 Sent Done Done false
190972024-02-12T01:42:44.605Z 16 Acked 1015 Write 6 Sent Done Done false
190982024-02-12T01:42:44.605Z 17 Acked 1016 Read 5 Sent Done Done false
190992024-02-12T01:42:44.605Z 18 Acked 1017 Write 2 Sent Done Done false
191002024-02-12T01:42:44.605Z 19 Acked 1018 Write 6 Sent Done Done false
191012024-02-12T01:42:44.605Z 20 Acked 1019 Read 5 Sent Done Done false
191022024-02-12T01:42:44.605Z 21 Acked 1020 Write 10 Sent Done Done false
191032024-02-12T01:42:44.605Z 22 Acked 1021 Write 6 Sent Done Done false
191042024-02-12T01:42:44.605Z 23 Acked 1022 Write 5 Sent Done Done false
191052024-02-12T01:42:44.605Z 24 Acked 1023 Read 4 Sent Done Done false
191062024-02-12T01:42:44.605Z 25 Acked 1024 Flush 0 Sent Done Done false
191072024-02-12T01:42:44.605Z 26 Acked 1025 Write 2 Sent Done Done false
191082024-02-12T01:42:44.605Z 27 Acked 1026 Write 10 Sent Done Done false
191092024-02-12T01:42:44.606Z 28 Acked 1027 Read 10 Sent Done Done false
191102024-02-12T01:42:44.606Z 29 Acked 1028 Read 10 Sent Done Done false
191112024-02-12T01:42:44.606Z 30 Acked 1029 Read 7 Sent Done Done false
191122024-02-12T01:42:44.606Z STATES DS:0 DS:1 DS:2 TOTAL
191132024-02-12T01:42:44.606Z New 0 0 0 0
191142024-02-12T01:42:44.606Z Sent 24 0 0 24
191152024-02-12T01:42:44.606Z Done 6 30 30 66
191162024-02-12T01:42:44.606Z Skipped 0 0 0 0
191172024-02-12T01:42:44.606Z Error 0 0 0 0
191182024-02-12T01:42:44.606ZLast Flush: 0 1024 1024
191192024-02-12T01:42:44.606ZDownstairs last five completed:
191202024-02-12T01:42:44.606ZUpstairs last five completed: 30 29 28 27 26
191212024-02-12T01:42:44.609Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:43.471021799Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1823}
191222024-02-12T01:42:44.609ZWrote out file "/tmp/test_repair_verify.data"
191232024-02-12T01:42:44.610Z
191242024-02-12T01:42:44.613ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
191252024-02-12T01:42:44.629ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191262024-02-12T01:42:44.629Z 0 000-019 170 170 170 306 306 306 F F F
191272024-02-12T01:42:44.629Z 1 020-039 170 170 170 306 306 306 F F F
191282024-02-12T01:42:44.629Z 2 040-059 165 165 165 297 297 297 F F F
191292024-02-12T01:42:44.629Z 3 060-079 159 159 159 281 281 281 F F F
191302024-02-12T01:42:44.629Z 4 080-099 164 164 164 292 292 292 F F F
191312024-02-12T01:42:44.629Z 5 100-119 168 168 168 304 304 304 F F F
191322024-02-12T01:42:44.629Z 6 120-139 168 172 172 304 310 310 F F F <---
191332024-02-12T01:42:44.629Z 7 140-159 168 172 172 304 309 309 F F F <---
191342024-02-12T01:42:44.629Z 8 160-179 168 172 172 304 310 310 T F F <---
191352024-02-12T01:42:44.629Z 9 180-199 163 163 163 290 290 290 F F F
191362024-02-12T01:42:44.633Z 10 200-219 164 164 164 291 291 291 F F F
191372024-02-12T01:42:44.633Z 11 220-239 166 172 172 302 310 310 F F F <---
191382024-02-12T01:42:44.633Z 12 240-259 167 167 167 303 303 303 F F F
191392024-02-12T01:42:44.633Z 13 260-279 171 171 171 308 308 308 F F F
191402024-02-12T01:42:44.633Z 14 280-299 170 170 170 307 307 307 F F F
191412024-02-12T01:42:44.633Z 15 300-319 170 172 172 307 310 310 F F F <---
191422024-02-12T01:42:44.633Z 16 320-339 170 172 172 307 310 310 F F F <---
191432024-02-12T01:42:44.633Z 17 340-359 160 172 172 285 309 309 T F F <---
191442024-02-12T01:42:44.633Z 18 360-379 166 166 166 302 302 302 F F F
191452024-02-12T01:42:44.633Z 19 380-399 153 153 153 271 271 271 F F F
191462024-02-12T01:42:44.633Z 20 400-419 163 163 163 290 290 290 F F F
191472024-02-12T01:42:44.633Z 21 420-439 165 172 172 297 310 310 F F F <---
191482024-02-12T01:42:44.633Z 22 440-459 170 172 172 307 310 310 F F F <---
191492024-02-12T01:42:44.633Z 23 460-479 169 169 169 305 305 305 F F F
191502024-02-12T01:42:44.633Z 24 480-499 171 171 171 308 308 308 F F F
191512024-02-12T01:42:44.633Z 25 500-519 168 168 168 304 304 304 F F F
191522024-02-12T01:42:44.633Z 26 520-539 164 164 164 296 296 296 F T T <---
191532024-02-12T01:42:44.633Z 27 540-559 166 172 172 300 310 310 F T T <---
191542024-02-12T01:42:44.633Z 28 560-579 169 169 169 305 305 305 F F F
191552024-02-12T01:42:44.634Z 29 580-599 171 171 171 308 308 308 F F F
191562024-02-12T01:42:44.634ZMax gen: 172, Max flush: 310
191572024-02-12T01:42:44.634ZError: Difference in extent metadata found!
191582024-02-12T01:42:44.634ZA Difference in extent metadata is expected here
191592024-02-12T01:42:44.634ZOn loop 86
191602024-02-12T01:42:44.634Z
191612024-02-12T01:42:44.637ZVerifying data now
191622024-02-12T01:42:50.196ZNo change after verify
191632024-02-12T01:42:50.196ZLoop: 86 Downstairs dump after verify (and repair):
191642024-02-12T01:42:50.205ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191652024-02-12T01:42:50.205Z 0 000-019 170 170 170 306 306 306 F F F
191662024-02-12T01:42:50.205Z 1 020-039 170 170 170 306 306 306 F F F
191672024-02-12T01:42:50.205Z 2 040-059 165 165 165 297 297 297 F F F
191682024-02-12T01:42:50.206Z 3 060-079 159 159 159 281 281 281 F F F
191692024-02-12T01:42:50.206Z 4 080-099 164 164 164 292 292 292 F F F
191702024-02-12T01:42:50.216Z 5 100-119 168 168 168 304 304 304 F F F
191712024-02-12T01:42:50.216Z 6 120-139 172 172 172 310 310 310 F F F
191722024-02-12T01:42:50.216Z 7 140-159 172 172 172 309 309 309 F F F
191732024-02-12T01:42:50.216Z 8 160-179 172 172 172 310 310 310 F F F
191742024-02-12T01:42:50.216Z 9 180-199 163 163 163 290 290 290 F F F
191752024-02-12T01:42:50.216Z 10 200-219 164 164 164 291 291 291 F F F
191762024-02-12T01:42:50.216Z 11 220-239 172 172 172 310 310 310 F F F
191772024-02-12T01:42:50.216Z 12 240-259 167 167 167 303 303 303 F F F
191782024-02-12T01:42:50.216Z 13 260-279 171 171 171 308 308 308 F F F
191792024-02-12T01:42:50.216Z 14 280-299 170 170 170 307 307 307 F F F
191802024-02-12T01:42:50.216Z 15 300-319 172 172 172 310 310 310 F F F
191812024-02-12T01:42:50.216Z 16 320-339 172 172 172 310 310 310 F F F
191822024-02-12T01:42:50.216Z 17 340-359 172 172 172 309 309 309 F F F
191832024-02-12T01:42:50.216Z 18 360-379 166 166 166 302 302 302 F F F
191842024-02-12T01:42:50.216Z 19 380-399 153 153 153 271 271 271 F F F
191852024-02-12T01:42:50.216Z 20 400-419 163 163 163 290 290 290 F F F
191862024-02-12T01:42:50.216Z 21 420-439 172 172 172 310 310 310 F F F
191872024-02-12T01:42:50.217Z 22 440-459 172 172 172 310 310 310 F F F
191882024-02-12T01:42:50.217Z 23 460-479 169 169 169 305 305 305 F F F
191892024-02-12T01:42:50.217Z 24 480-499 171 171 171 308 308 308 F F F
191902024-02-12T01:42:50.217Z 25 500-519 168 168 168 304 304 304 F F F
191912024-02-12T01:42:50.217Z 26 520-539 173 173 173 311 311 311 F F F
191922024-02-12T01:42:50.217Z 27 540-559 173 173 173 311 311 311 F F F
191932024-02-12T01:42:50.217Z 28 560-579 169 169 169 305 305 305 F F F
191942024-02-12T01:42:50.217Z 29 580-599 171 171 171 308 308 308 F F F
191952024-02-12T01:42:50.217ZMax gen: 173, Max flush: 311
191962024-02-12T01:42:50.217Z
191972024-02-12T01:42:50.217ZBegin loop 87
191982024-02-12T01:42:50.217ZDownstairs to restart: 0
191992024-02-12T01:42:50.227Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093459056Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
192002024-02-12T01:42:50.227Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093703075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
192012024-02-12T01:42:50.231Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093740913Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
192022024-02-12T01:42:50.231Z{Crucible runtime is spawned
192032024-02-12T01:42:50.231Z"msg":"Crucible 12c5bf07-9d19-4ade-b8fe-f59a7cf2753c has session id: a03b8380-98ef-4bba-aa3a-b612808ba6b4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093860867Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192042024-02-12T01:42:50.232Z{"msg":"Upstairs opts: Upstairs UUID: 12c5bf07-9d19-4ade-b8fe-f59a7cf2753c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093926324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192052024-02-12T01:42:50.232Z{"msg":"Crucible stats registered with UUID: 12c5bf07-9d19-4ade-b8fe-f59a7cf2753c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.093970752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192062024-02-12T01:42:50.232Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.0940119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
192072024-02-12T01:42:50.232Z{"msg":"12c5bf07-9d19-4ade-b8fe-f59a7cf2753c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094058458Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192082024-02-12T01:42:50.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094083877Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192092024-02-12T01:42:50.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094112915Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192102024-02-12T01:42:50.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094134064Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192112024-02-12T01:42:50.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094172253Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192122024-02-12T01:42:50.232Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094198651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192132024-02-12T01:42:50.232Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.09422342Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192142024-02-12T01:42:50.232Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094275078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192152024-02-12T01:42:50.232Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094334125Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192162024-02-12T01:42:50.232Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094385582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192172024-02-12T01:42:50.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.09443225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192182024-02-12T01:42:50.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094478218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192192024-02-12T01:42:50.232Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.094504077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192202024-02-12T01:42:50.232Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095090689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192212024-02-12T01:42:50.232Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095130597Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192222024-02-12T01:42:50.232Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095157195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192232024-02-12T01:42:50.232Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095204553Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192242024-02-12T01:42:50.232Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095770746Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192252024-02-12T01:42:50.232Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.095821454Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192262024-02-12T01:42:50.232Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096259783Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192272024-02-12T01:42:50.233Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096310151Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192282024-02-12T01:42:50.233Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096340209Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192292024-02-12T01:42:50.233Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096379327Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192302024-02-12T01:42:50.233Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096419935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192312024-02-12T01:42:50.233Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096457353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192322024-02-12T01:42:50.233Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096473063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192332024-02-12T01:42:50.233Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.096487272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192342024-02-12T01:42:50.237Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.09694944Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192352024-02-12T01:42:50.237Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.09737537Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192362024-02-12T01:42:50.237Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097438297Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192372024-02-12T01:42:50.237Z{"msg":"[0]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 310, 309, 310, 290, 291, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097485875Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192382024-02-12T01:42:50.237Z{"msg":"[0]R generation[0..12]: [170, 170, 165, 159, 164, 168, 172, 172, 172, 163, 164, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097522563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192392024-02-12T01:42:50.237Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097558811Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192402024-02-12T01:42:50.237Z{"msg":"[1]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 310, 309, 310, 290, 291, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097620848Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs",Wait for a query_work_queue command to finish before sending IO
192412024-02-12T01:42:50.237Z"":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192422024-02-12T01:42:50.237Z{"msg":"[1]R generation[0..12]: [170, 170, 165, 159, 164, 168, 172, 172, 172, 163, 164, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097665966Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192432024-02-12T01:42:50.237Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097713353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192442024-02-12T01:42:50.237Z{"msg":"[2]R flush_numbers[0..12]: [306, 306, 297, 281, 292, 304, 310, 309, 310, 290, 291, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097743672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192452024-02-12T01:42:50.237Z{"msg":"[2]R generation[0..12]: [170, 170, 165, 159, 164, 168, 172, 172, 172, 163, 164, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097778001Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192462024-02-12T01:42:50.237Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097816769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192472024-02-12T01:42:50.237Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097836218Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192482024-02-12T01:42:50.237Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097871186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192492024-02-12T01:42:50.237Z{"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097886485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192502024-02-12T01:42:50.237Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097916584Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192512024-02-12T01:42:50.238Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097952282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192522024-02-12T01:42:50.238Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.097975131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192532024-02-12T01:42:50.238Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.09799855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192542024-02-12T01:42:50.238Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098021749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192552024-02-12T01:42:50.238Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098045878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192562024-02-12T01:42:50.238Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098065677Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192572024-02-12T01:42:50.238Z{"msg":"12c5bf07-9d19-4ade-b8fe-f59a7cf2753c is now active with session: a03b8380-98ef-4bba-aa3a-b612808ba6b4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098097285Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192582024-02-12T01:42:50.238Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098110865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831,"session_id":"a03b8380-98ef-4bba-aa3a-b612808ba6b4"}
192592024-02-12T01:42:50.238Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:49.098124094Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
192602024-02-12T01:42:50.238ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
192612024-02-12T01:42:50.238ZLoading write count information from file "/tmp/test_repair_verify.data"
192622024-02-12T01:42:50.238ZRun Repair workload
192632024-02-12T01:42:50.238Z01/30 Write block 11 len 20480 data: 10 11 12 12 12
192642024-02-12T01:42:50.238Z02/30 Write block 434 len 28672 data: 10 9 9 11 11 14 13
192652024-02-12T01:42:50.238Z03/30 Write block 380 len 40960 data: 13 12 10 10 11 13 15 14 14 14
192662024-02-12T01:42:50.241Z04/30 Write block 576 len 32768 data: 13 13 11 11 11 12 12 12
192672024-02-12T01:42:50.241Z05/30 Read block 487 len 40960
192682024-02-12T01:42:50.242Z06/30 Write block 63 len 24576 data: 16 16 18 17 18 17
192692024-02-12T01:42:50.242Z07/30 Read block 597 len 4096
192702024-02-12T01:42:50.242Z08/30 Flush
192712024-02-12T01:42:50.244Z09/30 Write block 285 len 36864 data: 11 10 13 16 14 15 17 15 16
192722024-02-12T01:42:50.247Z10/30 Write block 221 len 24576 data: 12 13 14 14 13 9
192732024-02-12T01:42:50.247Z11/30 Write block 485 len 40960 data: 13 14 16 17 14 16 13 13 13 13
192742024-02-12T01:42:50.247Z12/30 Flush
192752024-02-12T01:42:50.251Z13/30 Write block 178 len 28672 data: 18 19 14 16 15 16 17
192762024-02-12T01:42:50.251Z14/30 Write block 243 len 36864 data: 16 13 11 9 10 9 8 8 11
192772024-02-12T01:42:50.254Z15/30 Read block 365 len 4096
192782024-02-12T01:42:50.254Z16/30 Write block 188 len 8192 data: 16 16
192792024-02-12T01:42:50.254Z17/30 Read block 368 len 16384
192802024-02-12T01:42:50.254Z18/30 Write block 274 len 8192 data: 18 12
192812024-02-12T01:42:50.257Z19/30 Read block 277 len 8192
192822024-02-12T01:42:50.257Z20/30 Write block 388 len 28672 data: 15 15 18 17 17 18 17
192832024-02-12T01:42:50.257Z21/30 Read block 362 len 24576
192842024-02-12T01:42:50.257Z22/30 Flush
192852024-02-12T01:42:50.260Z23/30 Write block 273 len 12288 data: 18 19 13
192862024-02-12T01:42:50.261Z24/30 Read block 112 len 4096
192872024-02-12T01:42:50.264Z25/30 Write block 376 len 12288 data: 16 15 15
192882024-02-12T01:42:50.264Z26/30 Write block 438 len 20480 data: 12 15 14 16 13
192892024-02-12T01:42:50.264Z27/30 Read block 305 len 20480
192902024-02-12T01:42:50.264Z28/30 Write block 205 len 12288 data: 14 14 14
192912024-02-12T01:42:50.264Z29/30 Read block 363 len 28672
192922024-02-12T01:42:50.267Z30/30 Read block 265 len 24576
192932024-02-12T01:42:50.267Z----------------------------------------------------------------
192942024-02-12T01:42:50.267Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
192952024-02-12T01:42:50.268ZStates: Active Active Active
192962024-02-12T01:42:50.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192972024-02-12T01:42:50.268Z 1 Acked 1000 Write 5 Done Done Done false
192982024-02-12T01:42:50.268Z 2 Acked 1001 Write 7 Sent Done Done false
192992024-02-12T01:42:50.268Z 3 Acked 1002 Write 10 Sent Done Done false
193002024-02-12T01:42:50.268Z 4 Acked 1003 Write 8 Sent Done Done false
193012024-02-12T01:42:50.268Z 5 Acked 1004 Read 10 Sent Done Done false
193022024-02-12T01:42:50.268Z 6 Acked 1005 Write 6 Sent Done Done false
193032024-02-12T01:42:50.268Z 7 Acked 1006 Read 1 Sent Done Done false
193042024-02-12T01:42:50.268Z 8 Acked 1007 Flush 0 Sent Done Done false
193052024-02-12T01:42:50.268Z 9 Acked 1008 Write 9 Sent Done Done false
193062024-02-12T01:42:50.268Z 10 Acked 1009 Write 6 Sent Done Done false
193072024-02-12T01:42:50.268Z 11 Acked 1010 Write 10 Sent Done Done false
193082024-02-12T01:42:50.268Z 12 Acked 1011 Flush 0 Sent Done Done false
193092024-02-12T01:42:50.268Z 13 Acked 1012 Write 7 Sent Done Done false
193102024-02-12T01:42:50.268Z 14 Acked 1013 Write 9 Sent Done Done false
193112024-02-12T01:42:50.268Z 15 Acked 1014 Read 1 Sent Done Done false
193122024-02-12T01:42:50.268Z 16 Acked 1015 Write 2 Sent Done Done false
193132024-02-12T01:42:50.268Z 17 Acked 1016 Read 4 Sent Done Done false
193142024-02-12T01:42:50.268Z 18 Acked 1017 Write 2 Sent Done Done false
193152024-02-12T01:42:50.268Z 19 Acked 1018 Read 2 Sent Done Done false
193162024-02-12T01:42:50.268Z 20 Acked 1019 Write 7 Sent Done Done false
193172024-02-12T01:42:50.268Z 21 Acked 1020 Read 6 Sent Done Done false
193182024-02-12T01:42:50.268Z 22 Acked 1021 Flush 0 Sent Done Done false
193192024-02-12T01:42:50.268Z 23 Acked 1022 Write 3 Sent Done Done false
193202024-02-12T01:42:50.269Z 24 Acked 1023 Read 1 Sent Done Done false
193212024-02-12T01:42:50.269Z 25 Acked 1024 Write 3 Sent Done Done false
193222024-02-12T01:42:50.269Z 26 Acked 1025 Write 5 Sent Done Done false
193232024-02-12T01:42:50.269Z 27 Acked 1026 Read 5 Sent Done Done false
193242024-02-12T01:42:50.269Z 28 Acked 1027 Write 3 Sent Done Done false
193252024-02-12T01:42:50.269Z 29 Acked 1028 Read 7 Sent Done Done false
193262024-02-12T01:42:50.269Z 30 Acked 1029 Read 6 Sent Sent Done false
193272024-02-12T01:42:50.269Z STATES DS:0 DS:1 DS:2 TOTAL
193282024-02-12T01:42:50.269Z New 0 0 0 0
193292024-02-12T01:42:50.269Z Sent 29 1 0 30
193302024-02-12T01:42:50.269Z Done 1 29 30 60
193312024-02-12T01:42:50.269Z Skipped 0 0 0 0
193322024-02-12T01:42:50.269Z Error 0 0 0 0
193332024-02-12T01:42:50.269ZLast Flush: 0 1021 1021
193342024-02-12T01:42:50.269ZDownstairs last five completed:
193352024-02-12T01:42:50.269ZUpstairs last five completed: 30 29 28 27 26
193362024-02-12T01:42:50.269Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:49.132819459Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1831}
193372024-02-12T01:42:50.269ZWrote out file "/tmp/test_repair_verify.data"
193382024-02-12T01:42:50.273Z
193392024-02-12T01:42:50.276ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193402024-02-12T01:42:50.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193412024-02-12T01:42:50.292Z 0 000-019 170 174 174 306 312 312 T F F <---
193422024-02-12T01:42:50.292Z 1 020-039 170 170 170 306 306 306 F F F
193432024-02-12T01:42:50.292Z 2 040-059 165 165 165 297 297 297 F F F
193442024-02-12T01:42:50.292Z 3 060-079 159 174 174 281 312 312 F F F <---
193452024-02-12T01:42:50.292Z 4 080-099 164 164 164 292 292 292 F F F
193462024-02-12T01:42:50.292Z 5 100-119 168 168 168 304 304 304 F F F
193472024-02-12T01:42:50.292Z 6 120-139 172 172 172 310 310 310 F F F
193482024-02-12T01:42:50.292Z 7 140-159 172 172 172 309 309 309 F F F
193492024-02-12T01:42:50.292Z 8 160-179 172 174 174 310 314 314 F F F <---
193502024-02-12T01:42:50.292Z 9 180-199 163 174 174 290 314 314 F F F <---
193512024-02-12T01:42:50.292Z 10 200-219 164 164 164 291 291 291 F T T <---
193522024-02-12T01:42:50.292Z 11 220-239 172 174 174 310 313 313 F F F <---
193532024-02-12T01:42:50.292Z 12 240-259 167 174 174 303 314 314 F F F <---
193542024-02-12T01:42:50.292Z 13 260-279 171 174 174 308 314 314 F T T <---
193552024-02-12T01:42:50.295Z 14 280-299 170 174 174 307 313 313 F F F <---
193562024-02-12T01:42:50.295Z 15 300-319 172 172 172 310 310 310 F F F
193572024-02-12T01:42:50.295Z 16 320-339 172 172 172 310 310 310 F F F
193582024-02-12T01:42:50.295Z 17 340-359 172 172 172 309 309 309 F F F
193592024-02-12T01:42:50.295Z 18 360-379 166 166 166 302 302 302 F T T <---
193602024-02-12T01:42:50.295Z 19 380-399 153 174 174 271 314 314 F F F <---
193612024-02-12T01:42:50.295Z 20 400-419 163 163 163 290 290 290 F F F
193622024-02-12T01:42:50.295Z 21 420-439 172 174 174 310 312 312 F T T <---
193632024-02-12T01:42:50.295Z 22 440-459 172 174 174 310 312 312 F T T <---
193642024-02-12T01:42:50.296Z 23 460-479 169 169 169 305 305 305 F F F
193652024-02-12T01:42:50.296Z 24 480-499 171 174 174 308 313 313 F F F <---
193662024-02-12T01:42:50.296Z 25 500-519 168 168 168 304 304 304 F F F
193672024-02-12T01:42:50.296Z 26 520-539 173 173 173 311 311 311 F F F
193682024-02-12T01:42:50.296Z 27 540-559 173 173 173 311 311 311 F F F
193692024-02-12T01:42:50.296Z 28 560-579 169 174 174 305 312 312 F F F <---
193702024-02-12T01:42:50.296Z 29 580-599 171 174 174 308 312 312 F F F <---
193712024-02-12T01:42:50.296ZMax gen: 174, Max flush: 314
193722024-02-12T01:42:50.296ZError: Difference in extent metadata found!
193732024-02-12T01:42:50.296ZA Difference in extent metadata is expected here
193742024-02-12T01:42:50.296ZOn loop 87
193752024-02-12T01:42:50.296Z
193762024-02-12T01:42:50.302ZVerifying data now
193772024-02-12T01:42:55.058ZNo change after verify
193782024-02-12T01:42:55.058ZLoop: 87 Downstairs dump after verify (and repair):
193792024-02-12T01:42:55.073ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193802024-02-12T01:42:55.073Z 0 000-019 174 174 174 312 312 312 F F F
193812024-02-12T01:42:55.073Z 1 020-039 170 170 170 306 306 306 F F F
193822024-02-12T01:42:55.073Z 2 040-059 165 165 165 297 297 297 F F F
193832024-02-12T01:42:55.073Z 3 060-079 174 174 174 312 312 312 F F F
193842024-02-12T01:42:55.073Z 4 080-099 164 164 164 292 292 292 F F F
193852024-02-12T01:42:55.074Z 5 100-119 168 168 168 304 304 304 F F F
193862024-02-12T01:42:55.074Z 6 120-139 172 172 172 310 310 310 F F F
193872024-02-12T01:42:55.074Z 7 140-159 172 172 172 309 309 309 F F F
193882024-02-12T01:42:55.077Z 8 160-179 174 174 174 314 314 314 F F F
193892024-02-12T01:42:55.077Z 9 180-199 174 174 174 314 314 314 F F F
193902024-02-12T01:42:55.077Z 10 200-219 175 175 175 315 315 315 F F F
193912024-02-12T01:42:55.077Z 11 220-239 174 174 174 313 313 313 F F F
193922024-02-12T01:42:55.077Z 12 240-259 174 174 174 314 314 314 F F F
193932024-02-12T01:42:55.077Z 13 260-279 175 175 175 315 315 315 F F F
193942024-02-12T01:42:55.077Z 14 280-299 174 174 174 313 313 313 F F F
193952024-02-12T01:42:55.077Z 15 300-319 172 172 172 310 310 310 F F F
193962024-02-12T01:42:55.077Z 16 320-339 172 172 172 310 310 310 F F F
193972024-02-12T01:42:55.077Z 17 340-359 172 172 172 309 309 309 F F F
193982024-02-12T01:42:55.077Z 18 360-379 175 175 175 315 315 315 F F F
193992024-02-12T01:42:55.077Z 19 380-399 174 174 174 314 314 314 F F F
194002024-02-12T01:42:55.077Z 20 400-419 163 163 163 290 290 290 F F F
194012024-02-12T01:42:55.077Z 21 420-439 175 175 175 315 315 315 F F F
194022024-02-12T01:42:55.077Z 22 440-459 175 175 175 315 315 315 F F F
194032024-02-12T01:42:55.077Z 23 460-479 169 169 169 305 305 305 F F F
194042024-02-12T01:42:55.077Z 24 480-499 174 174 174 313 313 313 F F F
194052024-02-12T01:42:55.077Z 25 500-519 168 168 168 304 304 304 F F F
194062024-02-12T01:42:55.078Z 26 520-539 173 173 173 311 311 311 F F F
194072024-02-12T01:42:55.078Z 27 540-559 173 173 173 311 311 311 F F F
194082024-02-12T01:42:55.078Z 28 560-579 174 174 174 312 312 312 F F F
194092024-02-12T01:42:55.078Z 29 580-599 174 174 174 312 312 312 F F F
194102024-02-12T01:42:55.078ZMax gen: 175, Max flush: 315
194112024-02-12T01:42:55.078Z
194122024-02-12T01:42:55.078ZBegin loop 88
194132024-02-12T01:42:55.078ZDownstairs to restart: 2
194142024-02-12T01:42:55.095Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961321136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
194152024-02-12T01:42:55.096Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961586203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
194162024-02-12T01:42:55.100Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961621981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
194172024-02-12T01:42:55.100Z{"msg":"Crucible 7fdbff7e-0c0f-4297-bbc0-e532101a61f0 has session id: 49154a75-60c5-41cd-a236-64909a41a6d5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.96165276Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839Crucible runtime is spawned
194182024-02-12T01:42:55.100Z,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194192024-02-12T01:42:55.100Z{"msg":"Upstairs opts: Upstairs UUID: 7fdbff7e-0c0f-4297-bbc0-e532101a61f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961710247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194202024-02-12T01:42:55.100Z{"msg":"Crucible stats registered with UUID: 7fdbff7e-0c0f-4297-bbc0-e532101a61f0","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961755885Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194212024-02-12T01:42:55.100Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961796593Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
194222024-02-12T01:42:55.100Z{"msg":"7fdbff7e-0c0f-4297-bbc0-e532101a61f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.96187029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194232024-02-12T01:42:55.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961916667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194242024-02-12T01:42:55.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961949906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194252024-02-12T01:42:55.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.961972525Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194262024-02-12T01:42:55.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962001613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194272024-02-12T01:42:55.100Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.96207206Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194282024-02-12T01:42:55.100Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962135257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194292024-02-12T01:42:55.100Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962160476Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194302024-02-12T01:42:55.101Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962197654Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194312024-02-12T01:42:55.101Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962221013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194322024-02-12T01:42:55.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962252931Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194332024-02-12T01:42:55.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962402704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194342024-02-12T01:42:55.101Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962453662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194352024-02-12T01:42:55.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962956618Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194362024-02-12T01:42:55.101Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.962998796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194372024-02-12T01:42:55.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.963014245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194382024-02-12T01:42:55.101Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.963033754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194392024-02-12T01:42:55.101Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.963575648Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194402024-02-12T01:42:55.101Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.963719961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194412024-02-12T01:42:55.101Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964023697Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194422024-02-12T01:42:55.101Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964064415Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194432024-02-12T01:42:55.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964277245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194442024-02-12T01:42:55.101Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964321813Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194452024-02-12T01:42:55.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964363461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194462024-02-12T01:42:55.101Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964396679Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194472024-02-12T01:42:55.101Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964418888Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194482024-02-12T01:42:55.101Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964433047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194492024-02-12T01:42:55.105Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.964632038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194502024-02-12T01:42:55.105Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965027219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194512024-02-12T01:42:55.105Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965107435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194522024-02-12T01:42:55.105Z{"msg":"[0]R flush_numbers[0..12]: [312, 306, 297, 312, 292, 304, 310, 309, 314, 314, 315, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965155773Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194532024-02-12T01:42:55.106Z{"msg":"[0]R generation[0..12]: [174, 170, 165, 174, 164, 168, 172, 172, 174, 174, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965172752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194542024-02-12T01:42:55.106Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965196381Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194552024-02-12T01:42:55.106Z{"msg":"[1]R flush_numbers[0..12]: [312, 306, 297, 312, 292, 304, 310, 309, 314, 314, 315, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.96521158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194562024-02-12T01:42:55.106Z{"msg":"[1]R generation[0..12]: [174, 170, 165, 174, 164, 168, 172, 172, 174, 174, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965232899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194572024-02-12T01:42:55.106Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965247379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194582024-02-12T01:42:55.106Z{"msg":"[2]R flush_numbers[0..12]: [312, 306, 297, 312, 292, 304, 310, 309, 314, 314, 315, 313]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965268667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194592024-02-12T01:42:55.106Z{"msg":"[2]R generation[0..12]: [174, 170, 165, 174, 164, 168, 172, 172, 174, 174, 175, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965283457Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194602024-02-12T01:42:55.106ZWait for a query_work_queue command to finish before sending IO
194612024-02-12T01:42:55.106Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965308956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194622024-02-12T01:42:55.106Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965329245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194632024-02-12T01:42:55.106Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965343124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194642024-02-12T01:42:55.106Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965368073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194652024-02-12T01:42:55.106Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965381742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194662024-02-12T01:42:55.106Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965407101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194672024-02-12T01:42:55.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.96542391Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194682024-02-12T01:42:55.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965438259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194692024-02-12T01:42:55.106Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965451429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194702024-02-12T01:42:55.106Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965472438Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194712024-02-12T01:42:55.106Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965485267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194722024-02-12T01:42:55.106Z{"msg":"7fdbff7e-0c0f-4297-bbc0-e532101a61f0 is now active with session: 49154a75-60c5-41cd-a236-64909a41a6d5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965498716Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194732024-02-12T01:42:55.107Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965510656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839,"session_id":"49154a75-60c5-41cd-a236-64909a41a6d5"}
194742024-02-12T01:42:55.107Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:53.965529905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
194752024-02-12T01:42:55.107ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194762024-02-12T01:42:55.107ZLoading write count information from file "/tmp/test_repair_verify.data"
194772024-02-12T01:42:55.107ZRun Repair workload
194782024-02-12T01:42:55.107Z01/30 Write block 457 len 12288 data: 11 15 12
194792024-02-12T01:42:55.107Z02/30 Read block 220 len 28672
194802024-02-12T01:42:55.107Z03/30 Read block 44 len 20480
194812024-02-12T01:42:55.107Z04/30 Write block 540 len 32768 data: 14 14 14 10 10 11 13 13
194822024-02-12T01:42:55.110Z05/30 Write block 442 len 20480 data: 14 9 11 10 10
194832024-02-12T01:42:55.110Z06/30 Flush
194842024-02-12T01:42:55.110Z07/30 Write block 362 len 28672 data: 12 14 18 18 16 22 21
194852024-02-12T01:42:55.110Z08/30 Write block 82 len 20480 data: 19 16 17 17 14
194862024-02-12T01:42:55.110Z09/30 Read block 12 len 20480
194872024-02-12T01:42:55.113Z10/30 Write block 1 len 12288 data: 3 4 5
194882024-02-12T01:42:55.113Z11/30 Write block 29 len 8192 data: 18 16
194892024-02-12T01:42:55.113Z12/30 Write block 447 len 4096 data: 11
194902024-02-12T01:42:55.113Z13/30 Write block 230 len 28672 data: 7 9 9 10 13 12 15
194912024-02-12T01:42:55.113Z14/30 Write block 37 len 8192 data: 12 16
194922024-02-12T01:42:55.116Z15/30 Write block 182 len 24576 data: 16 17 18 13 14 15
194932024-02-12T01:42:55.116Z16/30 Write block 513 len 32768 data: 11 10 13 10 10 8 9 12
194942024-02-12T01:42:55.116Z17/30 Read block 509 len 40960
194952024-02-12T01:42:55.119Z18/30 Flush
194962024-02-12T01:42:55.122Z19/30 Write block 315 len 8192 data: 14 10
194972024-02-12T01:42:55.122Z20/30 Read block 340 len 16384
194982024-02-12T01:42:55.125Z21/30 Write block 44 len 28672 data: 13 13 15 17 17 18 17
194992024-02-12T01:42:55.125Z22/30 Read block 280 len 4096
195002024-02-12T01:42:55.125Z23/30 Write block 493 len 36864 data: 14 14 16 14 14 15 14 11 12
195012024-02-12T01:42:55.128Z24/30 Write block 292 len 4096 data: 16
195022024-02-12T01:42:55.128Z25/30 Read block 35 len 16384
195032024-02-12T01:42:55.128Z26/30 Read block 401 len 12288
195042024-02-12T01:42:55.131Z27/30 Write block 153 len 40960 data: 12 12 9 9 10 11 11 14 13 14
195052024-02-12T01:42:55.131Z28/30 Read block 113 len 32768
195062024-02-12T01:42:55.131Z29/30 Write block 135 len 28672 data: 13 16 14 16 19 20 22
195072024-02-12T01:42:55.137Z30/30 Write block 384 len 36864 data: 12 14 16 15 16 16 19 18 18
195082024-02-12T01:42:55.137Z----------------------------------------------------------------
195092024-02-12T01:42:55.137Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
195102024-02-12T01:42:55.137ZStates: Active Active Active
195112024-02-12T01:42:55.137ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195122024-02-12T01:42:55.137Z 1 Acked 1000 Write 3 Done Done Done false
195132024-02-12T01:42:55.137Z 2 Acked 1001 Read 7 Done Done Done false
195142024-02-12T01:42:55.137Z 3 Acked 1002 Read 5 Done Done Done false
195152024-02-12T01:42:55.137Z 4 Acked 1003 Write 8 Done Done Sent false
195162024-02-12T01:42:55.137Z 5 Acked 1004 Write 5 Done Done Sent false
195172024-02-12T01:42:55.138Z 6 Acked 1005 Flush 0 Done Done Sent false
195182024-02-12T01:42:55.138Z 7 Acked 1006 Write 7 Done Done Sent false
195192024-02-12T01:42:55.138Z 8 Acked 1007 Write 5 Done Done Sent false
195202024-02-12T01:42:55.138Z 9 Acked 1008 Read 5 Done Done Sent false
195212024-02-12T01:42:55.138Z 10 Acked 1009 Write 3 Done Done Sent false
195222024-02-12T01:42:55.138Z 11 Acked 1010 Write 2 Done Done Sent false
195232024-02-12T01:42:55.138Z 12 Acked 1011 Write 1 Done Done Sent false
195242024-02-12T01:42:55.138Z 13 Acked 1012 Write 7 Done Done Sent false
195252024-02-12T01:42:55.138Z 14 Acked 1013 Write 2 Done Done Sent false
195262024-02-12T01:42:55.138Z 15 Acked 1014 Write 6 Done Done Sent false
195272024-02-12T01:42:55.138Z 16 Acked 1015 Write 8 Done Done Sent false
195282024-02-12T01:42:55.138Z 17 Acked 1016 Read 10 Done Done Sent false
195292024-02-12T01:42:55.138Z 18 Acked 1017 Flush 0 Done Done Sent false
195302024-02-12T01:42:55.138Z 19 Acked 1018 Write 2 Done Done Sent false
195312024-02-12T01:42:55.138Z 20 Acked 1019 Read 4 Done Done Sent false
195322024-02-12T01:42:55.138Z 21 Acked 1020 Write 7 Done Done Sent false
195332024-02-12T01:42:55.138Z 22 Acked 1021 Read 1 Done Done Sent false
195342024-02-12T01:42:55.138Z 23 Acked 1022 Write 9 Done Done Sent false
195352024-02-12T01:42:55.138Z 24 Acked 1023 Write 1 Done Done Sent false
195362024-02-12T01:42:55.138Z 25 Acked 1024 Read 4 Done Done Sent false
195372024-02-12T01:42:55.138Z 26 Acked 1025 Read 3 Done Done Sent false
195382024-02-12T01:42:55.138Z 27 Acked 1026 Write 10 Done Done Sent false
195392024-02-12T01:42:55.138Z 28 Acked 1027 Read 8 Done Done Sent false
195402024-02-12T01:42:55.139Z 29 Acked 1028 Write 7 Done Done Sent false
195412024-02-12T01:42:55.139Z 30 Acked 1029 Write 9 Sent Sent Sent false
195422024-02-12T01:42:55.139Z STATES DS:0 DS:1 DS:2 TOTAL
195432024-02-12T01:42:55.139Z New 0 0 0 0
195442024-02-12T01:42:55.139Z Sent 1 1 27 29
195452024-02-12T01:42:55.139Z Done 29 29 3 61
195462024-02-12T01:42:55.139Z Skipped 0 0 0 0
195472024-02-12T01:42:55.139Z Error 0 0 0 0
195482024-02-12T01:42:55.139ZLast Flush: 1017 1017 0
195492024-02-12T01:42:55.139ZDownstairs last five completed:
195502024-02-12T01:42:55.139ZUpstairs last five completed: 30 29 28 27 26
195512024-02-12T01:42:55.139Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:53.998088142Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1839}
195522024-02-12T01:42:55.139ZWrote out file "/tmp/test_repair_verify.data"
195532024-02-12T01:42:55.142Z
195542024-02-12T01:42:55.142ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
195552024-02-12T01:42:55.158ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195562024-02-12T01:42:55.158Z 0 000-019 176 176 174 317 317 312 F F F <---
195572024-02-12T01:42:55.158Z 1 020-039 176 176 170 317 317 306 F F F <---
195582024-02-12T01:42:55.159Z 2 040-059 165 165 165 297 297 297 T T F <---
195592024-02-12T01:42:55.159Z 3 060-079 174 174 174 312 312 312 F F F
195602024-02-12T01:42:55.159Z 4 080-099 176 176 164 317 317 292 F F F <---
195612024-02-12T01:42:55.159Z 5 100-119 168 168 168 304 304 304 F F F
195622024-02-12T01:42:55.159Z 6 120-139 172 172 172 310 310 310 T T F <---
195632024-02-12T01:42:55.159Z 7 140-159 172 172 172 309 309 309 T T F <---
195642024-02-12T01:42:55.159Z 8 160-179 174 174 174 314 314 314 T T F <---
195652024-02-12T01:42:55.159Z 9 180-199 176 176 174 317 317 314 F F F <---
195662024-02-12T01:42:55.159Z 10 200-219 175 175 175 315 315 315 F F F
195672024-02-12T01:42:55.159Z 11 220-239 176 176 174 317 317 313 F F F <---
195682024-02-12T01:42:55.159Z 12 240-259 174 174 174 314 314 314 F F F
195692024-02-12T01:42:55.162Z 13 260-279 175 175 175 315 315 315 F F F
195702024-02-12T01:42:55.162Z 14 280-299 174 174 174 313 313 313 T T F <---
195712024-02-12T01:42:55.162Z 15 300-319 172 172 172 310 310 310 T T F <---
195722024-02-12T01:42:55.162Z 16 320-339 172 172 172 310 310 310 F F F
195732024-02-12T01:42:55.163Z 17 340-359 172 172 172 309 309 309 F F F
195742024-02-12T01:42:55.163Z 18 360-379 176 176 175 317 317 315 F F F <---
195752024-02-12T01:42:55.163Z 19 380-399 174 174 174 314 314 314 T T F <---
195762024-02-12T01:42:55.163Z 20 400-419 163 163 163 290 290 290 F F F
195772024-02-12T01:42:55.163Z 21 420-439 175 175 175 315 315 315 F F F
195782024-02-12T01:42:55.163Z 22 440-459 176 176 175 317 317 315 F F T <---
195792024-02-12T01:42:55.163Z 23 460-479 169 169 169 305 305 305 F F F
195802024-02-12T01:42:55.163Z 24 480-499 174 174 174 313 313 313 T T F <---
195812024-02-12T01:42:55.163Z 25 500-519 176 176 168 317 317 304 T T F <---
195822024-02-12T01:42:55.163Z 26 520-539 176 176 173 317 317 311 F F F <---
195832024-02-12T01:42:55.163Z 27 540-559 176 176 173 316 316 311 F F F <---
195842024-02-12T01:42:55.163Z 28 560-579 174 174 174 312 312 312 F F F
195852024-02-12T01:42:55.163Z 29 580-599 174 174 174 312 312 312 F F F
195862024-02-12T01:42:55.163ZMax gen: 176, Max flush: 317
195872024-02-12T01:42:55.163ZError: Difference in extent metadata found!
195882024-02-12T01:42:55.163ZA Difference in extent metadata is expected here
195892024-02-12T01:42:55.163ZOn loop 88
195902024-02-12T01:42:55.163Z
195912024-02-12T01:42:55.166ZVerifying data now
195922024-02-12T01:43:00.009ZNo change after verify
195932024-02-12T01:43:00.009ZLoop: 88 Downstairs dump after verify (and repair):
195942024-02-12T01:43:00.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195952024-02-12T01:43:00.025Z 0 000-019 176 176 176 317 317 317 F F F
195962024-02-12T01:43:00.025Z 1 020-039 176 176 176 317 317 317 F F F
195972024-02-12T01:43:00.025Z 2 040-059 177 177 177 318 318 318 F F F
195982024-02-12T01:43:00.025Z 3 060-079 174 174 174 312 312 312 F F F
195992024-02-12T01:43:00.025Z 4 080-099 176 176 176 317 317 317 F F F
196002024-02-12T01:43:00.025Z 5 100-119 168 168 168 304 304 304 F F F
196012024-02-12T01:43:00.025Z 6 120-139 177 177 177 318 318 318 F F F
196022024-02-12T01:43:00.025Z 7 140-159 177 177 177 318 318 318 F F F
196032024-02-12T01:43:00.028Z 8 160-179 177 177 177 318 318 318 F F F
196042024-02-12T01:43:00.028Z 9 180-199 176 176 176 317 317 317 F F F
196052024-02-12T01:43:00.028Z 10 200-219 175 175 175 315 315 315 F F F
196062024-02-12T01:43:00.028Z 11 220-239 176 176 176 317 317 317 F F F
196072024-02-12T01:43:00.029Z 12 240-259 174 174 174 314 314 314 F F F
196082024-02-12T01:43:00.029Z 13 260-279 175 175 175 315 315 315 F F F
196092024-02-12T01:43:00.029Z 14 280-299 177 177 177 318 318 318 F F F
196102024-02-12T01:43:00.029Z 15 300-319 177 177 177 318 318 318 F F F
196112024-02-12T01:43:00.029Z 16 320-339 172 172 172 310 310 310 F F F
196122024-02-12T01:43:00.029Z 17 340-359 172 172 172 309 309 309 F F F
196132024-02-12T01:43:00.029Z 18 360-379 176 176 176 317 317 317 F F F
196142024-02-12T01:43:00.029Z 19 380-399 177 177 177 318 318 318 F F F
196152024-02-12T01:43:00.029Z 20 400-419 163 163 163 290 290 290 F F F
196162024-02-12T01:43:00.029Z 21 420-439 175 175 175 315 315 315 F F F
196172024-02-12T01:43:00.029Z 22 440-459 176 176 176 317 317 317 F F F
196182024-02-12T01:43:00.029Z 23 460-479 169 169 169 305 305 305 F F F
196192024-02-12T01:43:00.029Z 24 480-499 177 177 177 318 318 318 F F F
196202024-02-12T01:43:00.029Z 25 500-519 177 177 177 318 318 318 F F F
196212024-02-12T01:43:00.029Z 26 520-539 176 176 176 317 317 317 F F F
196222024-02-12T01:43:00.029Z 27 540-559 176 176 176 316 316 316 F F F
196232024-02-12T01:43:00.029Z 28 560-579 174 174 174 312 312 312 F F F
196242024-02-12T01:43:00.029Z 29 580-599 174 174 174 312 312 312 F F F
196252024-02-12T01:43:00.029ZMax gen: 177, Max flush: 318
196262024-02-12T01:43:00.029Z
196272024-02-12T01:43:00.029ZBegin loop 89
196282024-02-12T01:43:00.029ZDownstairs to restart: 2
196292024-02-12T01:43:00.047Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.911807902Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
196302024-02-12T01:43:00.047Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
196312024-02-12T01:43:00.047Z,"time":"2024-02-12T01:42:58.912085789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
196322024-02-12T01:43:00.052Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912137026Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
196332024-02-12T01:43:00.052Z{"msg":"Crucible b1024d16-5b4a-43b9-b9c7-ac4ea3d8a04f has session id: db8b0a65-6fcd-4388-b63c-b0699f4a5050","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912179554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196342024-02-12T01:43:00.052Z{"msg":"Upstairs opts: Upstairs UUID: b1024d16-5b4a-43b9-b9c7-ac4ea3d8a04f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912222582Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196352024-02-12T01:43:00.052Z{"msg":"Crucible stats registered with UUID: b1024d16-5b4a-43b9-b9c7-ac4ea3d8a04f","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.91226489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196362024-02-12T01:43:00.052Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912285019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
196372024-02-12T01:43:00.052Z{"msg":"b1024d16-5b4a-43b9-b9c7-ac4ea3d8a04f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912350516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196382024-02-12T01:43:00.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912382324Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196392024-02-12T01:43:00.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912400734Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196402024-02-12T01:43:00.052Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912433312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196412024-02-12T01:43:00.052Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912459751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196422024-02-12T01:43:00.053Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912501749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196432024-02-12T01:43:00.053Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912537167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196442024-02-12T01:43:00.053Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912575305Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196452024-02-12T01:43:00.053Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912648652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196462024-02-12T01:43:00.053Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912699049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196472024-02-12T01:43:00.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912733098Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196482024-02-12T01:43:00.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912766516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196492024-02-12T01:43:00.053Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.912813274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196502024-02-12T01:43:00.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.913507661Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196512024-02-12T01:43:00.053Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.913548409Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196522024-02-12T01:43:00.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.913574778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196532024-02-12T01:43:00.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.913617636Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196542024-02-12T01:43:00.053Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914114202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196552024-02-12T01:43:00.053Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914316192Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196562024-02-12T01:43:00.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914633637Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196572024-02-12T01:43:00.053Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914675835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196582024-02-12T01:43:00.053Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914724703Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196592024-02-12T01:43:00.053Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914768421Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196602024-02-12T01:43:00.053Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.91479346Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196612024-02-12T01:43:00.053Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914823408Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196622024-02-12T01:43:00.053Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914927363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196632024-02-12T01:43:00.053Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.914967581Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196642024-02-12T01:43:00.053Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.915429519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196652024-02-12T01:43:00.058Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.91583402Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196662024-02-12T01:43:00.058Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.915879988Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196672024-02-12T01:43:00.058Z{"msg":"[0]R flush_numbers[0..12]: [317, 317, 318, 312, 317, 304, 318, 318, 318, 317, 315, 317]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.915907456Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196682024-02-12T01:43:00.058Z{"msg":"[0]R generation[0..12]: [176, 176, 177, 174, 176, 168, 177, 177, 177, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.915943934Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196692024-02-12T01:43:00.058Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.915979883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196702024-02-12T01:43:00.058Z{"msg":"[1]R flush_numbers[0..12]: [317, 317, 318, 312, 317, 304, 318, 318, 318, 317, 315, 317]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.91603447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196712024-02-12T01:43:00.058Z{"msg":"Wait for a query_work_queue command to finish before sending IO
196722024-02-12T01:43:00.058Z[1]R generation[0..12]: [176, 176, 177, 174, 176, 168, 177, 177, 177, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916076258Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196732024-02-12T01:43:00.058Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916116656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196742024-02-12T01:43:00.058Z{"msg":"[2]R flush_numbers[0..12]: [317, 317, 318, 312, 317, 304, 318, 318, 318, 317, 315, 317]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916155615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196752024-02-12T01:43:00.058Z{"msg":"[2]R generation[0..12]: [176, 176, 177, 174, 176, 168, 177, 177, 177, 176, 175, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916193613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196762024-02-12T01:43:00.058Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916225111Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196772024-02-12T01:43:00.058Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916239631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196782024-02-12T01:43:00.058Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916260859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196792024-02-12T01:43:00.058Z{"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916274239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196802024-02-12T01:43:00.058Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916294308Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196812024-02-12T01:43:00.058Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916308107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196822024-02-12T01:43:00.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916326486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196832024-02-12T01:43:00.058Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916339906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196842024-02-12T01:43:00.059Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916359695Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196852024-02-12T01:43:00.059Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916373144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196862024-02-12T01:43:00.059Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916391793Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196872024-02-12T01:43:00.059Z{"msg":"b1024d16-5b4a-43b9-b9c7-ac4ea3d8a04f is now active with session: db8b0a65-6fcd-4388-b63c-b0699f4a5050","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916405012Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196882024-02-12T01:43:00.059Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916424652Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847,"session_id":"db8b0a65-6fcd-4388-b63c-b0699f4a5050"}
196892024-02-12T01:43:00.059Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:42:58.916437191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
196902024-02-12T01:43:00.059ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196912024-02-12T01:43:00.059ZLoading write count information from file "/tmp/test_repair_verify.data"
196922024-02-12T01:43:00.059ZRun Repair workload
196932024-02-12T01:43:00.059Z01/30 Write block 99 len 24576 data: 8 7 10 9 9 13
196942024-02-12T01:43:00.059Z02/30 Write block 418 len 16384 data: 11 10 10 11
196952024-02-12T01:43:00.059Z03/30 Flush
196962024-02-12T01:43:00.059Z04/30 Write block 178 len 36864 data: 19 20 15 17 17 18 19 14 15
196972024-02-12T01:43:00.065Z05/30 Flush
196982024-02-12T01:43:00.065Z06/30 Write block 346 len 20480 data: 11 10 9 11 15
196992024-02-12T01:43:00.065Z07/30 Read block 69 len 36864
197002024-02-12T01:43:00.065Z08/30 Write block 332 len 4096 data: 10
197012024-02-12T01:43:00.065Z09/30 Read block 536 len 28672
197022024-02-12T01:43:00.069Z10/30 Write block 65 len 40960 data: 19 18 19 18 16 15 15 10 9 9
197032024-02-12T01:43:00.069Z11/30 Write block 78 len 32768 data: 11 14 16 16 20 17 18 18
197042024-02-12T01:43:00.069Z12/30 Write block 76 len 20480 data: 11 11 12 15 17
197052024-02-12T01:43:00.069Z13/30 Flush
197062024-02-12T01:43:00.071Z14/30 Write block 417 len 24576 data: 14 12 11 11 12 10
197072024-02-12T01:43:00.071Z15/30 Write block 381 len 12288 data: 13 11 11
197082024-02-12T01:43:00.071Z16/30 Write block 120 len 16384 data: 16 15 14 16
197092024-02-12T01:43:00.072Z17/30 Flush
197102024-02-12T01:43:00.074Z18/30 Write block 514 len 36864 data: 11 14 11 11 9 10 13 12 13
197112024-02-12T01:43:00.077Z19/30 Read block 411 len 12288
197122024-02-12T01:43:00.077Z20/30 Write block 462 len 4096 data: 11
197132024-02-12T01:43:00.077Z21/30 Write block 472 len 20480 data: 13 17 18 17 14
197142024-02-12T01:43:00.077Z22/30 Read block 339 len 32768
197152024-02-12T01:43:00.081Z23/30 Write block 227 len 32768 data: 10 10 10 8 10 10 11 14
197162024-02-12T01:43:00.081Z24/30 Write block 174 len 32768 data: 15 16 18 16 20 21 16 18
197172024-02-12T01:43:00.081Z25/30 Flush
197182024-02-12T01:43:00.083Z26/30 Write block 278 len 16384 data: 15 16 15 13
197192024-02-12T01:43:00.087Z27/30 Write block 263 len 40960 data: 10 11 15 17 18 16 15 17 19 19
197202024-02-12T01:43:00.087Z28/30 Read block 512 len 28672
197212024-02-12T01:43:00.087Z29/30 Read block 575 len 20480
197222024-02-12T01:43:00.091Z30/30 Read block 529 len 16384
197232024-02-12T01:43:00.091Z----------------------------------------------------------------
197242024-02-12T01:43:00.091Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
197252024-02-12T01:43:00.091ZStates: Active Active Active
197262024-02-12T01:43:00.091ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
197272024-02-12T01:43:00.091Z 1 Acked 1000 Write 6 Done Done Sent false
197282024-02-12T01:43:00.091Z 2 Acked 1001 Write 4 Done Done Sent false
197292024-02-12T01:43:00.091Z 3 Acked 1002 Flush 0 Done Done Sent false
197302024-02-12T01:43:00.091Z 4 Acked 1003 Write 9 Done Done Sent false
197312024-02-12T01:43:00.091Z 5 Acked 1004 Flush 0 Done Done Sent false
197322024-02-12T01:43:00.091Z 6 Acked 1005 Write 5 Done Done Sent false
197332024-02-12T01:43:00.091Z 7 Acked 1006 Read 9 Done Done Sent false
197342024-02-12T01:43:00.091Z 8 Acked 1007 Write 1 Done Done Sent false
197352024-02-12T01:43:00.091Z 9 Acked 1008 Read 7 Done Done Sent false
197362024-02-12T01:43:00.091Z 10 Acked 1009 Write 10 Done Done Sent false
197372024-02-12T01:43:00.091Z 11 Acked 1010 Write 8 Done Done Sent false
197382024-02-12T01:43:00.091Z 12 Acked 1011 Write 5 Done Done Sent false
197392024-02-12T01:43:00.091Z 13 Acked 1012 Flush 0 Done Done Sent false
197402024-02-12T01:43:00.091Z 14 Acked 1013 Write 6 Done Done Sent false
197412024-02-12T01:43:00.092Z 15 Acked 1014 Write 3 Done Done Sent false
197422024-02-12T01:43:00.092Z 16 Acked 1015 Write 4 Done Done Sent false
197432024-02-12T01:43:00.092Z 17 Acked 1016 Flush 0 Done Done Sent false
197442024-02-12T01:43:00.092Z 18 Acked 1017 Write 9 Done Done Sent false
197452024-02-12T01:43:00.092Z 19 Acked 1018 Read 3 Done Done Sent false
197462024-02-12T01:43:00.092Z 20 Acked 1019 Write 1 Done Done Sent false
197472024-02-12T01:43:00.092Z 21 Acked 1020 Write 5 Done Done Sent false
197482024-02-12T01:43:00.092Z 22 Acked 1021 Read 8 Done Done Sent false
197492024-02-12T01:43:00.092Z 23 Acked 1022 Write 8 Done Done Sent false
197502024-02-12T01:43:00.092Z 24 Acked 1023 Write 8 Done Done Sent false
197512024-02-12T01:43:00.092Z 25 Acked 1024 Flush 0 Done Done Sent false
197522024-02-12T01:43:00.092Z 26 Acked 1025 Write 4 Done Done Sent false
197532024-02-12T01:43:00.092Z 27 Acked 1026 Write 10 Done Done Sent false
197542024-02-12T01:43:00.092Z 28 Acked 1027 Read 7 Done Done Sent false
197552024-02-12T01:43:00.092Z 29 Acked 1028 Read 5 Done Done Sent false
197562024-02-12T01:43:00.092Z 30 Acked 1029 Read 4 Done Sent Sent false
197572024-02-12T01:43:00.092Z STATES DS:0 DS:1 DS:2 TOTAL
197582024-02-12T01:43:00.092Z New 0 0 0 0
197592024-02-12T01:43:00.092Z Sent 0 1 30 31
197602024-02-12T01:43:00.092Z Done 30 29 0 59
197612024-02-12T01:43:00.092Z Skipped 0 0 0 0
197622024-02-12T01:43:00.092Z Error 0 0 0 0
197632024-02-12T01:43:00.092ZLast Flush: 1024 1024 0
197642024-02-12T01:43:00.093ZDownstairs last five completed:
197652024-02-12T01:43:00.093ZUpstairs last five completed: 30 29 28 27 26
197662024-02-12T01:43:00.093Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:42:58.953377789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1847}
197672024-02-12T01:43:00.093ZWrote out file "/tmp/test_repair_verify.data"
197682024-02-12T01:43:00.096Z
197692024-02-12T01:43:00.096ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197702024-02-12T01:43:00.111ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197712024-02-12T01:43:00.111Z 0 000-019 176 176 176 317 317 317 F F F
197722024-02-12T01:43:00.111Z 1 020-039 176 176 176 317 317 317 F F F
197732024-02-12T01:43:00.111Z 2 040-059 177 177 177 318 318 318 F F F
197742024-02-12T01:43:00.111Z 3 060-079 178 178 174 321 321 312 F F F <---
197752024-02-12T01:43:00.111Z 4 080-099 178 178 176 321 321 317 F F F <---
197762024-02-12T01:43:00.114Z 5 100-119 178 178 168 319 319 304 F F F <---
197772024-02-12T01:43:00.115Z 6 120-139 178 178 177 322 322 318 F F F <---
197782024-02-12T01:43:00.115Z 7 140-159 177 177 177 318 318 318 F F F
197792024-02-12T01:43:00.115Z 8 160-179 178 178 177 323 323 318 F F F <---
197802024-02-12T01:43:00.115Z 9 180-199 178 178 176 323 323 317 F F F <---
197812024-02-12T01:43:00.115Z 10 200-219 175 175 175 315 315 315 F F F
197822024-02-12T01:43:00.115Z 11 220-239 178 178 176 323 323 317 F F F <---
197832024-02-12T01:43:00.115Z 12 240-259 174 174 174 314 314 314 F F F
197842024-02-12T01:43:00.115Z 13 260-279 175 175 175 315 315 315 T T F <---
197852024-02-12T01:43:00.115Z 14 280-299 177 177 177 318 318 318 T T F <---
197862024-02-12T01:43:00.115Z 15 300-319 177 177 177 318 318 318 F F F
197872024-02-12T01:43:00.115Z 16 320-339 178 178 172 321 321 310 F F F <---
197882024-02-12T01:43:00.115Z 17 340-359 178 178 172 321 321 309 F F F <---
197892024-02-12T01:43:00.115Z 18 360-379 176 176 176 317 317 317 F F F
197902024-02-12T01:43:00.115Z 19 380-399 178 178 177 322 322 318 F F F <---
197912024-02-12T01:43:00.115Z 20 400-419 178 178 163 322 322 290 F F F <---
197922024-02-12T01:43:00.115Z 21 420-439 178 178 175 322 322 315 F F F <---
197932024-02-12T01:43:00.115Z 22 440-459 176 176 176 317 317 317 F F F
197942024-02-12T01:43:00.115Z 23 460-479 178 178 169 323 323 305 F F F <---
197952024-02-12T01:43:00.115Z 24 480-499 177 177 177 318 318 318 F F F
197962024-02-12T01:43:00.115Z 25 500-519 178 178 177 323 323 318 F F F <---
197972024-02-12T01:43:00.115Z 26 520-539 178 178 176 323 323 317 F F F <---
197982024-02-12T01:43:00.115Z 27 540-559 176 176 176 316 316 316 F F F
197992024-02-12T01:43:00.115Z 28 560-579 174 174 174 312 312 312 F F F
198002024-02-12T01:43:00.116Z 29 580-599 174 174 174 312 312 312 F F F
198012024-02-12T01:43:00.116ZMax gen: 178, Max flush: 323
198022024-02-12T01:43:00.116ZError: Difference in extent metadata found!
198032024-02-12T01:43:00.116ZA Difference in extent metadata is expected here
198042024-02-12T01:43:00.116ZOn loop 89
198052024-02-12T01:43:00.116Z
198062024-02-12T01:43:00.119ZVerifying data now
198072024-02-12T01:43:04.908ZNo change after verify
198082024-02-12T01:43:04.908ZLoop: 89 Downstairs dump after verify (and repair):
198092024-02-12T01:43:04.922ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198102024-02-12T01:43:04.922Z 0 000-019 176 176 176 317 317 317 F F F
198112024-02-12T01:43:04.922Z 1 020-039 176 176 176 317 317 317 F F F
198122024-02-12T01:43:04.922Z 2 040-059 177 177 177 318 318 318 F F F
198132024-02-12T01:43:04.923Z 3 060-079 178 178 178 321 321 321 F F F
198142024-02-12T01:43:04.923Z 4 080-099 178 178 178 321 321 321 F F F
198152024-02-12T01:43:04.927Z 5 100-119 178 178 178 319 319 319 F F F
198162024-02-12T01:43:04.927Z 6 120-139 178 178 178 322 322 322 F F F
198172024-02-12T01:43:04.927Z 7 140-159 177 177 177 318 318 318 F F F
198182024-02-12T01:43:04.927Z 8 160-179 178 178 178 323 323 323 F F F
198192024-02-12T01:43:04.927Z 9 180-199 178 178 178 323 323 323 F F F
198202024-02-12T01:43:04.927Z 10 200-219 175 175 175 315 315 315 F F F
198212024-02-12T01:43:04.927Z 11 220-239 178 178 178 323 323 323 F F F
198222024-02-12T01:43:04.927Z 12 240-259 174 174 174 314 314 314 F F F
198232024-02-12T01:43:04.927Z 13 260-279 179 179 179 324 324 324 F F F
198242024-02-12T01:43:04.927Z 14 280-299 179 179 179 324 324 324 F F F
198252024-02-12T01:43:04.927Z 15 300-319 177 177 177 318 318 318 F F F
198262024-02-12T01:43:04.928Z 16 320-339 178 178 178 321 321 321 F F F
198272024-02-12T01:43:04.928Z 17 340-359 178 178 178 321 321 321 F F F
198282024-02-12T01:43:04.928Z 18 360-379 176 176 176 317 317 317 F F F
198292024-02-12T01:43:04.928Z 19 380-399 178 178 178 322 322 322 F F F
198302024-02-12T01:43:04.928Z 20 400-419 178 178 178 322 322 322 F F F
198312024-02-12T01:43:04.928Z 21 420-439 178 178 178 322 322 322 F F F
198322024-02-12T01:43:04.928Z 22 440-459 176 176 176 317 317 317 F F F
198332024-02-12T01:43:04.928Z 23 460-479 178 178 178 323 323 323 F F F
198342024-02-12T01:43:04.928Z 24 480-499 177 177 177 318 318 318 F F F
198352024-02-12T01:43:04.928Z 25 500-519 178 178 178 323 323 323 F F F
198362024-02-12T01:43:04.928Z 26 520-539 178 178 178 323 323 323 F F F
198372024-02-12T01:43:04.928Z 27 540-559 176 176 176 316 316 316 F F F
198382024-02-12T01:43:04.928Z 28 560-579 174 174 174 312 312 312 F F F
198392024-02-12T01:43:04.928Z 29 580-599 174 174 174 312 312 312 F F F
198402024-02-12T01:43:04.928ZMax gen: 179, Max flush: 324
198412024-02-12T01:43:04.928Z
198422024-02-12T01:43:04.928ZBegin loop 90
198432024-02-12T01:43:04.928ZDownstairs to restart: 2
198442024-02-12T01:43:04.943Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808068155Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
198452024-02-12T01:43:04.950Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808347502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
198462024-02-12T01:43:04.950Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.80838178Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
198472024-02-12T01:43:04.950Z{"msg":"Crucible c0fd0782-6578-4ead-b9fa-03d0fdff297e has session id: f7fdfc59-e645-4c8e-832b-bb076ce9ea4d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808405549Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198482024-02-12T01:43:04.950ZCrucible runtime is spawned
198492024-02-12T01:43:04.950Z{"msg":"Upstairs opts: Upstairs UUID: c0fd0782-6578-4ead-b9fa-03d0fdff297e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808479796Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198502024-02-12T01:43:04.950Z{"msg":"Crucible stats registered with UUID: c0fd0782-6578-4ead-b9fa-03d0fdff297e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808516624Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198512024-02-12T01:43:04.950Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808540293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
198522024-02-12T01:43:04.950Z{"msg":"c0fd0782-6578-4ead-b9fa-03d0fdff297e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808570751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198532024-02-12T01:43:04.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.80859368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198542024-02-12T01:43:04.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808625229Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198552024-02-12T01:43:04.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808656977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198562024-02-12T01:43:04.950Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808681596Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198572024-02-12T01:43:04.950Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808720764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198582024-02-12T01:43:04.951Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808744883Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198592024-02-12T01:43:04.951Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808776461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198602024-02-12T01:43:04.951Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808823849Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198612024-02-12T01:43:04.951Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.808871177Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198622024-02-12T01:43:04.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.80901764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198632024-02-12T01:43:04.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.809080757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198642024-02-12T01:43:04.951Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.809121855Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198652024-02-12T01:43:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.809796213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198662024-02-12T01:43:04.951Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.809833801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198672024-02-12T01:43:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.80984983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198682024-02-12T01:43:04.951Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.809880399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198692024-02-12T01:43:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810316238Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198702024-02-12T01:43:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810412074Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198712024-02-12T01:43:04.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810753267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198722024-02-12T01:43:04.951Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810807575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198732024-02-12T01:43:04.951Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810844113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198742024-02-12T01:43:04.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810858792Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198752024-02-12T01:43:04.951Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810880571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198762024-02-12T01:43:04.951Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.810894671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198772024-02-12T01:43:04.951Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811012185Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198782024-02-12T01:43:04.951Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811040533Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198792024-02-12T01:43:04.951Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811499882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198802024-02-12T01:43:04.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811919282Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198812024-02-12T01:43:04.951Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.81194881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198822024-02-12T01:43:04.951Z{"msg":"[0]R flush_numbers[0..12]: [317, 317, 318, 321, 321, 319, 322, 318, 323, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811964669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198832024-02-12T01:43:04.952Z{"msg":"[0]R generation[0..12]: [176, 176, 177, 178, 178, 178, 178, 177, 178, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.811983239Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198842024-02-12T01:43:04.952Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812025357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198852024-02-12T01:43:04.952Z{"msg":"[1]R flush_numbers[0..12]: [317, 317, 318, 321, 321, 319, 322, 318, 323, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812045856Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198862024-02-12T01:43:04.952Z{"msg":"[1]R generation[0..12]: [176, 176, 177, 178, 178, 178, 178, 177, 178, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812075894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198872024-02-12T01:43:04.952ZWait for a query_work_queue command to finish before sending IO
198882024-02-12T01:43:04.952Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812125732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198892024-02-12T01:43:04.952Z{"msg":"[2]R flush_numbers[0..12]: [317, 317, 318, 321, 321, 319, 322, 318, 323, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.81216035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198902024-02-12T01:43:04.952Z{"msg":"[2]R generation[0..12]: [176, 176, 177, 178, 178, 178, 178, 177, 178, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812193889Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198912024-02-12T01:43:04.952Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812227747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198922024-02-12T01:43:04.952Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812259805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198932024-02-12T01:43:04.952Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812291474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198942024-02-12T01:43:04.952Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812323702Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198952024-02-12T01:43:04.952Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812355801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198962024-02-12T01:43:04.952Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812387669Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198972024-02-12T01:43:04.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812417478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198982024-02-12T01:43:04.952Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812450396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
198992024-02-12T01:43:04.956Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812472865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
199002024-02-12T01:43:04.956Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812502264Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
199012024-02-12T01:43:04.956Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812516683Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
199022024-02-12T01:43:04.956Z{"msg":"c0fd0782-6578-4ead-b9fa-03d0fdff297e is now active with session: f7fdfc59-e645-4c8e-832b-bb076ce9ea4d","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812536942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
199032024-02-12T01:43:04.956ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199042024-02-12T01:43:04.956Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.812567241Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855,"session_id":"f7fdfc59-e645-4c8e-832b-bb076ce9ea4d"}
199052024-02-12T01:43:04.956Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:03.81258414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
199062024-02-12T01:43:04.956ZLoading write count information from file "/tmp/test_repair_verify.data"
199072024-02-12T01:43:04.956ZRun Repair workload
199082024-02-12T01:43:04.956Z01/30 Write block 558 len 40960 data: 9 10 8 12 10 13 11 12 12 13
199092024-02-12T01:43:04.956Z02/30 Read block 204 len 4096
199102024-02-12T01:43:04.956Z03/30 Write block 112 len 24576 data: 8 13 15 15 15 15
199112024-02-12T01:43:04.956Z04/30 Write block 162 len 8192 data: 15 13
199122024-02-12T01:43:04.956Z05/30 Read block 508 len 24576
199132024-02-12T01:43:04.956Z06/30 Read block 107 len 28672
199142024-02-12T01:43:04.957Z07/30 Read block 29 len 8192
199152024-02-12T01:43:04.957Z08/30 Write block 379 len 16384 data: 16 14 14 12
199162024-02-12T01:43:04.960Z09/30 Write block 447 len 12288 data: 12 12 12
199172024-02-12T01:43:04.960Z10/30 Write block 371 len 36864 data: 16 18 16 17 19 17 16 16 17
199182024-02-12T01:43:04.960Z11/30 Write block 481 len 40960 data: 16 16 15 15 14 15 17 18 15 17
199192024-02-12T01:43:04.960Z12/30 Read block 337 len 20480
199202024-02-12T01:43:04.963Z13/30 Write block 370 len 24576 data: 19 17 19 17 18 20
199212024-02-12T01:43:04.963Z14/30 Write block 586 len 36864 data: 14 11 11 12 15 12 13 11 10
199222024-02-12T01:43:04.963Z15/30 Write block 448 len 4096 data: 13
199232024-02-12T01:43:04.963Z16/30 Read block 147 len 16384
199242024-02-12T01:43:04.969Z17/30 Write block 81 len 32768 data: 17 21 18 19 19 15 18 15
199252024-02-12T01:43:04.969Z18/30 Read block 24 len 40960
199262024-02-12T01:43:04.969Z19/30 Write block 574 len 16384 data: 19 17 14 14
199272024-02-12T01:43:04.969Z20/30 Write block 482 len 16384 data: 17 16 16 15
199282024-02-12T01:43:04.969Z21/30 Read block 579 len 40960
199292024-02-12T01:43:04.972Z22/30 Write block 313 len 40960 data: 15 13 15 11 11 12 13 13 11 13
199302024-02-12T01:43:04.972Z23/30 Read block 273 len 36864
199312024-02-12T01:43:04.972Z24/30 Read block 164 len 28672
199322024-02-12T01:43:04.972Z25/30 Write block 470 len 36864 data: 12 12 14 18 19 18 15 14 12
199332024-02-12T01:43:04.972Z26/30 Read block 439 len 16384
199342024-02-12T01:43:04.976Z27/30 Write block 158 len 40960 data: 12 12 15 14 16 14 13 14 13 13
199352024-02-12T01:43:04.976Z28/30 Read block 303 len 36864
199362024-02-12T01:43:04.976Z29/30 Read block 404 len 8192
199372024-02-12T01:43:04.976Z30/30 Write block 10 len 12288 data: 9 11 12
199382024-02-12T01:43:04.980Z----------------------------------------------------------------
199392024-02-12T01:43:04.980Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
199402024-02-12T01:43:04.980ZStates: Active Active Active
199412024-02-12T01:43:04.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199422024-02-12T01:43:04.980Z 1 Acked 1000 Write 10 Done Done Sent false
199432024-02-12T01:43:04.980Z 2 Acked 1001 Read 1 Done Done Sent false
199442024-02-12T01:43:04.980Z 3 Acked 1002 Write 6 Done Done Sent false
199452024-02-12T01:43:04.980Z 4 Acked 1003 Write 2 Done Done Sent false
199462024-02-12T01:43:04.980Z 5 Acked 1004 Read 6 Done Done Sent false
199472024-02-12T01:43:04.980Z 6 Acked 1005 Read 7 Done Done Sent false
199482024-02-12T01:43:04.980Z 7 Acked 1006 Read 2 Done Done Sent false
199492024-02-12T01:43:04.980Z 8 Acked 1007 Write 4 Done Done Sent false
199502024-02-12T01:43:04.980Z 9 Acked 1008 Write 3 Done Done Sent false
199512024-02-12T01:43:04.980Z 10 Acked 1009 Write 9 Done Done Sent false
199522024-02-12T01:43:04.980Z 11 Acked 1010 Write 10 Done Done Sent false
199532024-02-12T01:43:04.980Z 12 Acked 1011 Read 5 Done Done Sent false
199542024-02-12T01:43:04.980Z 13 Acked 1012 Write 6 Done Done Sent false
199552024-02-12T01:43:04.981Z 14 Acked 1013 Write 9 Done Done Sent false
199562024-02-12T01:43:04.981Z 15 Acked 1014 Write 1 Done Done Sent false
199572024-02-12T01:43:04.981Z 16 Acked 1015 Read 4 Done Done Sent false
199582024-02-12T01:43:04.981Z 17 Acked 1016 Write 8 Done Done Sent false
199592024-02-12T01:43:04.981Z 18 Acked 1017 Read 10 Done Done Sent false
199602024-02-12T01:43:04.981Z 19 Acked 1018 Write 4 Done Done Sent false
199612024-02-12T01:43:04.981Z 20 Acked 1019 Write 4 Done Done Sent false
199622024-02-12T01:43:04.981Z 21 Acked 1020 Read 10 Done Done Sent false
199632024-02-12T01:43:04.981Z 22 Acked 1021 Write 10 Done Done Sent false
199642024-02-12T01:43:04.981Z 23 Acked 1022 Read 9 Done Done Sent false
199652024-02-12T01:43:04.981Z 24 Acked 1023 Read 7 Done Done Sent false
199662024-02-12T01:43:04.981Z 25 Acked 1024 Write 9 Done Done Sent false
199672024-02-12T01:43:04.981Z 26 Acked 1025 Read 4 Done Done Sent false
199682024-02-12T01:43:04.981Z 27 Acked 1026 Write 10 Done Done Sent false
199692024-02-12T01:43:04.981Z 28 Acked 1027 Read 9 Done Done Sent false
199702024-02-12T01:43:04.981Z 29 Acked 1028 Read 2 Done Done Sent false
199712024-02-12T01:43:04.981Z 30 Acked 1029 Write 3 Sent Sent Sent false
199722024-02-12T01:43:04.981Z STATES DS:0 DS:1 DS:2 TOTAL
199732024-02-12T01:43:04.981Z New 0 0 0 0
199742024-02-12T01:43:04.981Z Sent 1 1 30 32
199752024-02-12T01:43:04.981Z Done 29 29 0 58
199762024-02-12T01:43:04.981Z Skipped 0 0 0 0
199772024-02-12T01:43:04.981Z Error 0 0 0 0
199782024-02-12T01:43:04.981ZLast Flush: 0 0 0
199792024-02-12T01:43:04.981ZDownstairs last five completed:
199802024-02-12T01:43:04.982ZUpstairs last five completed: 30 29 28 27 26
199812024-02-12T01:43:04.982Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:03.84108336Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1855}
199822024-02-12T01:43:04.982ZWrote out file "/tmp/test_repair_verify.data"
199832024-02-12T01:43:04.985Z
199842024-02-12T01:43:04.985ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199852024-02-12T01:43:05.005ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199862024-02-12T01:43:05.005Z 0 000-019 176 176 176 317 317 317 T T F <---
199872024-02-12T01:43:05.005Z 1 020-039 176 176 176 317 317 317 F F F
199882024-02-12T01:43:05.005Z 2 040-059 177 177 177 318 318 318 F F F
199892024-02-12T01:43:05.005Z 3 060-079 178 178 178 321 321 321 F F F
199902024-02-12T01:43:05.005Z 4 080-099 178 178 178 321 321 321 T T F <---
199912024-02-12T01:43:05.005Z 5 100-119 178 178 178 319 319 319 T T F <---
199922024-02-12T01:43:05.005Z 6 120-139 178 178 178 322 322 322 F F F
199932024-02-12T01:43:05.005Z 7 140-159 177 177 177 318 318 318 T T F <---
199942024-02-12T01:43:05.005Z 8 160-179 178 178 178 323 323 323 T T F <---
199952024-02-12T01:43:05.008Z 9 180-199 178 178 178 323 323 323 F F F
199962024-02-12T01:43:05.009Z 10 200-219 175 175 175 315 315 315 F F F
199972024-02-12T01:43:05.009Z 11 220-239 178 178 178 323 323 323 F F F
199982024-02-12T01:43:05.009Z 12 240-259 174 174 174 314 314 314 F F F
199992024-02-12T01:43:05.009Z 13 260-279 179 179 179 324 324 324 F F F
200002024-02-12T01:43:05.009Z 14 280-299 179 179 179 324 324 324 F F F
200012024-02-12T01:43:05.009Z 15 300-319 177 177 177 318 318 318 T T F <---
200022024-02-12T01:43:05.009Z 16 320-339 178 178 178 321 321 321 T T F <---
200032024-02-12T01:43:05.009Z 17 340-359 178 178 178 321 321 321 F F F
200042024-02-12T01:43:05.009Z 18 360-379 176 176 176 317 317 317 T T F <---
200052024-02-12T01:43:05.009Z 19 380-399 178 178 178 322 322 322 T T F <---
200062024-02-12T01:43:05.009Z 20 400-419 178 178 178 322 322 322 F F F
200072024-02-12T01:43:05.009Z 21 420-439 178 178 178 322 322 322 F F F
200082024-02-12T01:43:05.009Z 22 440-459 176 176 176 317 317 317 T T F <---
200092024-02-12T01:43:05.009Z 23 460-479 178 178 178 323 323 323 T T F <---
200102024-02-12T01:43:05.009Z 24 480-499 177 177 177 318 318 318 T T F <---
200112024-02-12T01:43:05.009Z 25 500-519 178 178 178 323 323 323 F F F
200122024-02-12T01:43:05.009Z 26 520-539 178 178 178 323 323 323 F F F
200132024-02-12T01:43:05.009Z 27 540-559 176 176 176 316 316 316 T T F <---
200142024-02-12T01:43:05.009Z 28 560-579 174 174 174 312 312 312 T T F <---
200152024-02-12T01:43:05.009Z 29 580-599 174 174 174 312 312 312 T T F <---
200162024-02-12T01:43:05.009ZMax gen: 179, Max flush: 324
200172024-02-12T01:43:05.009ZError: Difference in extent metadata found!
200182024-02-12T01:43:05.009ZA Difference in extent metadata is expected here
200192024-02-12T01:43:05.010ZOn loop 90
200202024-02-12T01:43:05.010Z
200212024-02-12T01:43:05.013ZVerifying data now
200222024-02-12T01:43:09.765ZNo change after verify
200232024-02-12T01:43:09.765ZLoop: 90 Downstairs dump after verify (and repair):
200242024-02-12T01:43:09.781ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200252024-02-12T01:43:09.781Z 0 000-019 180 180 180 325 325 325 F F F
200262024-02-12T01:43:09.781Z 1 020-039 176 176 176 317 317 317 F F F
200272024-02-12T01:43:09.781Z 2 040-059 177 177 177 318 318 318 F F F
200282024-02-12T01:43:09.781Z 3 060-079 178 178 178 321 321 321 F F F
200292024-02-12T01:43:09.781Z 4 080-099 180 180 180 325 325 325 F F F
200302024-02-12T01:43:09.781Z 5 100-119 180 180 180 325 325 325 F F F
200312024-02-12T01:43:09.781Z 6 120-139 178 178 178 322 322 322 F F F
200322024-02-12T01:43:09.781Z 7 140-159 180 180 180 325 325 325 F F F
200332024-02-12T01:43:09.781Z 8 160-179 180 180 180 325 325 325 F F F
200342024-02-12T01:43:09.781Z 9 180-199 178 178 178 323 323 323 F F F
200352024-02-12T01:43:09.786Z 10 200-219 175 175 175 315 315 315 F F F
200362024-02-12T01:43:09.786Z 11 220-239 178 178 178 323 323 323 F F F
200372024-02-12T01:43:09.786Z 12 240-259 174 174 174 314 314 314 F F F
200382024-02-12T01:43:09.786Z 13 260-279 179 179 179 324 324 324 F F F
200392024-02-12T01:43:09.786Z 14 280-299 179 179 179 324 324 324 F F F
200402024-02-12T01:43:09.786Z 15 300-319 180 180 180 325 325 325 F F F
200412024-02-12T01:43:09.786Z 16 320-339 180 180 180 325 325 325 F F F
200422024-02-12T01:43:09.786Z 17 340-359 178 178 178 321 321 321 F F F
200432024-02-12T01:43:09.786Z 18 360-379 180 180 180 325 325 325 F F F
200442024-02-12T01:43:09.786Z 19 380-399 180 180 180 325 325 325 F F F
200452024-02-12T01:43:09.786Z 20 400-419 178 178 178 322 322 322 F F F
200462024-02-12T01:43:09.787Z 21 420-439 178 178 178 322 322 322 F F F
200472024-02-12T01:43:09.787Z 22 440-459 180 180 180 325 325 325 F F F
200482024-02-12T01:43:09.787Z 23 460-479 180 180 180 325 325 325 F F F
200492024-02-12T01:43:09.787Z 24 480-499 180 180 180 325 325 325 F F F
200502024-02-12T01:43:09.787Z 25 500-519 178 178 178 323 323 323 F F F
200512024-02-12T01:43:09.787Z 26 520-539 178 178 178 323 323 323 F F F
200522024-02-12T01:43:09.787Z 27 540-559 180 180 180 325 325 325 F F F
200532024-02-12T01:43:09.787Z 28 560-579 180 180 180 325 325 325 F F F
200542024-02-12T01:43:09.787Z 29 580-599 180 180 180 325 325 325 F F F
200552024-02-12T01:43:09.787ZMax gen: 180, Max flush: 325
200562024-02-12T01:43:09.787Z
200572024-02-12T01:43:09.787ZBegin loop 91
200582024-02-12T01:43:09.787ZDownstairs to restart: 1
200592024-02-12T01:43:09.804Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.667965724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
200602024-02-12T01:43:09.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.66825487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
200612024-02-12T01:43:09.810ZCrucible runtime is spawned
200622024-02-12T01:43:09.810Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668395793Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
200632024-02-12T01:43:09.810Z{"msg":"Crucible 7b23ac7a-1067-4c21-8944-2116cf1b13ab has session id: 214020d1-cbfa-4f60-9e69-250e20ecbfd4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.66844925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200642024-02-12T01:43:09.810Z{"msg":"Upstairs opts: Upstairs UUID: 7b23ac7a-1067-4c21-8944-2116cf1b13ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668505278Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200652024-02-12T01:43:09.810Z{"msg":"Crucible stats registered with UUID: 7b23ac7a-1067-4c21-8944-2116cf1b13ab","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668550236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200662024-02-12T01:43:09.810Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668590604Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
200672024-02-12T01:43:09.810Z{"msg":"7b23ac7a-1067-4c21-8944-2116cf1b13ab active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668628772Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200682024-02-12T01:43:09.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.66866935Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200692024-02-12T01:43:09.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668714198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200702024-02-12T01:43:09.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668753396Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200712024-02-12T01:43:09.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668779035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200722024-02-12T01:43:09.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668827183Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200732024-02-12T01:43:09.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668900869Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200742024-02-12T01:43:09.811Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.668976615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200752024-02-12T01:43:09.811Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669030453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200762024-02-12T01:43:09.811Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.66907836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200772024-02-12T01:43:09.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669106119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200782024-02-12T01:43:09.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669138468Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200792024-02-12T01:43:09.811Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669157627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200802024-02-12T01:43:09.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669651233Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200812024-02-12T01:43:09.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669699751Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200822024-02-12T01:43:09.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669730859Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200832024-02-12T01:43:09.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.669795266Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200842024-02-12T01:43:09.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670258574Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200852024-02-12T01:43:09.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670392748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200862024-02-12T01:43:09.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670514882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200872024-02-12T01:43:09.812Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.67056349Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200882024-02-12T01:43:09.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670709653Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200892024-02-12T01:43:09.812Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670751631Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200902024-02-12T01:43:09.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.670795518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200912024-02-12T01:43:09.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671000579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200922024-02-12T01:43:09.812Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671036427Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200932024-02-12T01:43:09.812Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671051666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200942024-02-12T01:43:09.812Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671066516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200952024-02-12T01:43:09.812Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671457257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200962024-02-12T01:43:09.812Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671498575Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200972024-02-12T01:43:09.812Z{"msg":"[0]R flush_numbers[0..12]: [325, 317, 318, 321, 325, 325, 322, 325, 325, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671514554Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200982024-02-12T01:43:09.813Z{"msg":"[0]R generation[0..12]: [180, 176, 177, 178, 180, 180, 178, 180, 180, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671529514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
200992024-02-12T01:43:09.813Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671550942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201002024-02-12T01:43:09.813Z{"msg":"[1]R flush_numbers[0..12]: [325, 317, 318, 321, 325, 325, 322, 325, 325, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671573461Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201012024-02-12T01:43:09.813Z{"msg":"[1]R generation[0..12]: [180, 176, 177, 178, 180, 180, 178, 180, 180, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.67159485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201022024-02-12T01:43:09.813Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.67160972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201032024-02-12T01:43:09.813Z{"msg":"[2]R flush_numbers[0..12]: [325, 317, 318, 321, 325, 325, 322, 325, 325, 323, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671635429Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201042024-02-12T01:43:09.813Z{"msg":"[2]R generation[0..12]: [180, 176, 177, 178, 180, 180, 178, 180, 180, 178, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671649778Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201052024-02-12T01:43:09.813Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
201062024-02-12T01:43:09.813Zcrucible","level":30,"time":"2024-02-12T01:43:08.671672987Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201072024-02-12T01:43:09.813Z{"msg":"Max found gen is 181","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671688826Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201082024-02-12T01:43:09.813Z{"msg":"Generation requested: 182 >= found:181","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671709175Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201092024-02-12T01:43:09.813Z{"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671722244Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201102024-02-12T01:43:09.813Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671747113Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201112024-02-12T01:43:09.814Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671760572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201122024-02-12T01:43:09.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671773372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201132024-02-12T01:43:09.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671792871Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201142024-02-12T01:43:09.814Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.67180617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201152024-02-12T01:43:09.814Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.67181983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201162024-02-12T01:43:09.814Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671831979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201172024-02-12T01:43:09.814Z{"msg":"7b23ac7a-1067-4c21-8944-2116cf1b13ab is now active with session: 214020d1-cbfa-4f60-9e69-250e20ecbfd4","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671858118Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201182024-02-12T01:43:09.814Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671877477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863,"session_id":"214020d1-cbfa-4f60-9e69-250e20ecbfd4"}
201192024-02-12T01:43:09.814Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:08.671898866Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
201202024-02-12T01:43:09.818ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201212024-02-12T01:43:09.818ZLoading write count information from file "/tmp/test_repair_verify.data"
201222024-02-12T01:43:09.818ZRun Repair workload
201232024-02-12T01:43:09.818Z01/30 Write block 108 len 4096 data: 15
201242024-02-12T01:43:09.818Z02/30 Write block 294 len 20480 data: 13 14 12 13 11
201252024-02-12T01:43:09.819Z03/30 Write block 548 len 8192 data: 13 15
201262024-02-12T01:43:09.819Z04/30 Read block 347 len 8192
201272024-02-12T01:43:09.819Z05/30 Read block 468 len 4096
201282024-02-12T01:43:09.819Z06/30 Write block 590 len 40960 data: 16 13 14 12 11 11 11 9 6 4
201292024-02-12T01:43:09.819Z07/30 Read block 396 len 12288
201302024-02-12T01:43:09.819Z08/30 Read block 320 len 12288
201312024-02-12T01:43:09.819Z09/30 Read block 421 len 32768
201322024-02-12T01:43:09.819Z10/30 Read block 140 len 16384
201332024-02-12T01:43:09.819Z11/30 Write block 123 len 28672 data: 17 17 16 19 18 16 15
201342024-02-12T01:43:09.819Z12/30 Flush
201352024-02-12T01:43:09.823Z13/30 Write block 115 len 32768 data: 16 16 16 17 15 17 16 15
201362024-02-12T01:43:09.823Z14/30 Read block 465 len 40960
201372024-02-12T01:43:09.828Z15/30 Read block 510 len 20480
201382024-02-12T01:43:09.828Z16/30 Write block 514 len 12288 data: 12 15 12
201392024-02-12T01:43:09.828Z17/30 Read block 217 len 16384
201402024-02-12T01:43:09.832Z18/30 Write block 500 len 36864 data: 12 13 13 11 11 12 12 13 10
201412024-02-12T01:43:09.832Z19/30 Write block 0 len 40960 data: 3 4 5 6 7 10 11 9 9 10
201422024-02-12T01:43:09.832Z20/30 Read block 38 len 32768
201432024-02-12T01:43:09.833Z21/30 Read block 529 len 8192
201442024-02-12T01:43:09.833Z22/30 Write block 453 len 32768 data: 7 9 10 10 12 16 13 10
201452024-02-12T01:43:09.833Z23/30 Read block 41 len 28672
201462024-02-12T01:43:09.836Z24/30 Write block 526 len 36864 data: 9 8 9 10 15 14 16 19 18
201472024-02-12T01:43:09.836Z25/30 Read block 363 len 32768
201482024-02-12T01:43:09.836Z26/30 Flush
201492024-02-12T01:43:09.842Z27/30 Write block 22 len 20480 data: 12 13 12 15 14
201502024-02-12T01:43:09.842Z28/30 Write block 181 len 36864 data: 19 18 19 20 15 16 16 17 17
201512024-02-12T01:43:09.846Z29/30 Write block 170 len 32768 data: 19 19 16 16 16 17 19 17
201522024-02-12T01:43:09.846Z30/30 Read block 122 len 24576
201532024-02-12T01:43:09.846Z----------------------------------------------------------------
201542024-02-12T01:43:09.846Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
201552024-02-12T01:43:09.846ZStates: Active Active Active
201562024-02-12T01:43:09.846ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
201572024-02-12T01:43:09.846Z 1 Acked 1000 Write 1 Done Done Done false
201582024-02-12T01:43:09.846Z 2 Acked 1001 Write 5 Done Done Done false
201592024-02-12T01:43:09.846Z 3 Acked 1002 Write 2 Done Sent Done false
201602024-02-12T01:43:09.847Z 4 Acked 1003 Read 2 Done Sent Done false
201612024-02-12T01:43:09.847Z 5 Acked 1004 Read 1 Done Sent Done false
201622024-02-12T01:43:09.847Z 6 Acked 1005 Write 10 Done Sent Done false
201632024-02-12T01:43:09.847Z 7 Acked 1006 Read 3 Done Sent Done false
201642024-02-12T01:43:09.847Z 8 Acked 1007 Read 3 Done Sent Done false
201652024-02-12T01:43:09.847Z 9 Acked 1008 Read 8 Done Sent Done false
201662024-02-12T01:43:09.847Z 10 Acked 1009 Read 4 Done Sent Done false
201672024-02-12T01:43:09.847Z 11 Acked 1010 Write 7 Done Sent Done false
201682024-02-12T01:43:09.847Z 12 Acked 1011 Flush 0 Done Sent Done false
201692024-02-12T01:43:09.847Z 13 Acked 1012 Write 8 Done Sent Done false
201702024-02-12T01:43:09.847Z 14 Acked 1013 Read 10 Done Sent Done false
201712024-02-12T01:43:09.847Z 15 Acked 1014 Read 5 Done Sent Done false
201722024-02-12T01:43:09.847Z 16 Acked 1015 Write 3 Done Sent Done false
201732024-02-12T01:43:09.848Z 17 Acked 1016 Read 4 Done Sent Done false
201742024-02-12T01:43:09.848Z 18 Acked 1017 Write 9 Done Sent Done false
201752024-02-12T01:43:09.848Z 19 Acked 1018 Write 10 Done Sent Done false
201762024-02-12T01:43:09.848Z 20 Acked 1019 Read 8 Done Sent Done false
201772024-02-12T01:43:09.848Z 21 Acked 1020 Read 2 Done Sent Done false
201782024-02-12T01:43:09.848Z 22 Acked 1021 Write 8 Done Sent Done false
201792024-02-12T01:43:09.848Z 23 Acked 1022 Read 7 Done Sent Done false
201802024-02-12T01:43:09.848Z 24 Acked 1023 Write 9 Done Sent Done false
201812024-02-12T01:43:09.848Z 25 Acked 1024 Read 8 Done Sent Done false
201822024-02-12T01:43:09.848Z 26 Acked 1025 Flush 0 Done Sent Done false
201832024-02-12T01:43:09.848Z 27 Acked 1026 Write 5 Done Sent Done false
201842024-02-12T01:43:09.848Z 28 Acked 1027 Write 9 Done Sent Done false
201852024-02-12T01:43:09.848Z 29 Acked 1028 Write 8 Done Sent Sent false
201862024-02-12T01:43:09.849Z 30 Acked 1029 Read 6 Done Sent Sent false
201872024-02-12T01:43:09.849Z STATES DS:0 DS:1 DS:2 TOTAL
201882024-02-12T01:43:09.849Z New 0 0 0 0
201892024-02-12T01:43:09.849Z Sent 0 28 2 30
201902024-02-12T01:43:09.849Z Done 30 2 28 60
201912024-02-12T01:43:09.849Z Skipped 0 0 0 0
201922024-02-12T01:43:09.849Z Error 0 0 0 0
201932024-02-12T01:43:09.849ZLast Flush: 1025 0 1025
201942024-02-12T01:43:09.849ZDownstairs last five completed:
201952024-02-12T01:43:09.849ZUpstairs last five completed: 30 29 28 27 26
201962024-02-12T01:43:09.849Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:08.706064896Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1863}
201972024-02-12T01:43:09.849ZWrote out file "/tmp/test_repair_verify.data"
201982024-02-12T01:43:09.853Z
201992024-02-12T01:43:09.853ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202002024-02-12T01:43:09.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202012024-02-12T01:43:09.867Z 0 000-019 182 180 182 327 325 327 F F F <---
202022024-02-12T01:43:09.867Z 1 020-039 176 176 176 317 317 317 T F T <---
202032024-02-12T01:43:09.867Z 2 040-059 177 177 177 318 318 318 F F F
202042024-02-12T01:43:09.867Z 3 060-079 178 178 178 321 321 321 F F F
202052024-02-12T01:43:09.867Z 4 080-099 180 180 180 325 325 325 F F F
202062024-02-12T01:43:09.870Z 5 100-119 182 180 182 327 325 327 F T F <---
202072024-02-12T01:43:09.870Z 6 120-139 182 178 182 327 322 327 F F F <---
202082024-02-12T01:43:09.870Z 7 140-159 180 180 180 325 325 325 F F F
202092024-02-12T01:43:09.871Z 8 160-179 180 180 180 325 325 325 T F T <---
202102024-02-12T01:43:09.871Z 9 180-199 178 178 178 323 323 323 T F T <---
202112024-02-12T01:43:09.871Z 10 200-219 175 175 175 315 315 315 F F F
202122024-02-12T01:43:09.871Z 11 220-239 178 178 178 323 323 323 F F F
202132024-02-12T01:43:09.871Z 12 240-259 174 174 174 314 314 314 F F F
202142024-02-12T01:43:09.871Z 13 260-279 179 179 179 324 324 324 F F F
202152024-02-12T01:43:09.871Z 14 280-299 182 179 182 326 324 326 F T F <---
202162024-02-12T01:43:09.871Z 15 300-319 180 180 180 325 325 325 F F F
202172024-02-12T01:43:09.871Z 16 320-339 180 180 180 325 325 325 F F F
202182024-02-12T01:43:09.871Z 17 340-359 178 178 178 321 321 321 F F F
202192024-02-12T01:43:09.871Z 18 360-379 180 180 180 325 325 325 F F F
202202024-02-12T01:43:09.871Z 19 380-399 180 180 180 325 325 325 F F F
202212024-02-12T01:43:09.871Z 20 400-419 178 178 178 322 322 322 F F F
202222024-02-12T01:43:09.871Z 21 420-439 178 178 178 322 322 322 F F F
202232024-02-12T01:43:09.871Z 22 440-459 182 180 182 327 325 327 F F F <---
202242024-02-12T01:43:09.871Z 23 460-479 182 180 182 327 325 327 F F F <---
202252024-02-12T01:43:09.871Z 24 480-499 180 180 180 325 325 325 F F F
202262024-02-12T01:43:09.871Z 25 500-519 182 178 182 327 323 327 F F F <---
202272024-02-12T01:43:09.871Z 26 520-539 182 178 182 327 323 327 F F F <---
202282024-02-12T01:43:09.871Z 27 540-559 182 180 182 326 325 326 F F F <---
202292024-02-12T01:43:09.871Z 28 560-579 180 180 180 325 325 325 F F F
202302024-02-12T01:43:09.871Z 29 580-599 182 180 182 326 325 326 F F F <---
202312024-02-12T01:43:09.871ZMax gen: 182, Max flush: 327
202322024-02-12T01:43:09.872ZError: Difference in extent metadata found!
202332024-02-12T01:43:09.872ZA Difference in extent metadata is expected here
202342024-02-12T01:43:09.872ZOn loop 91
202352024-02-12T01:43:09.872Z
202362024-02-12T01:43:09.875ZVerifying data now
202372024-02-12T01:43:15.521ZNo change after verify
202382024-02-12T01:43:15.521ZLoop: 91 Downstairs dump after verify (and repair):
202392024-02-12T01:43:15.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202402024-02-12T01:43:15.537Z 0 000-019 182 182 182 327 327 327 F F F
202412024-02-12T01:43:15.537Z 1 020-039 183 183 183 328 328 328 F F F
202422024-02-12T01:43:15.537Z 2 040-059 177 177 177 318 318 318 F F F
202432024-02-12T01:43:15.537Z 3 060-079 178 178 178 321 321 321 F F F
202442024-02-12T01:43:15.537Z 4 080-099 180 180 180 325 325 325 F F F
202452024-02-12T01:43:15.537Z 5 100-119 182 182 182 327 327 327 F F F
202462024-02-12T01:43:15.537Z 6 120-139 182 182 182 327 327 327 F F F
202472024-02-12T01:43:15.537Z 7 140-159 180 180 180 325 325 325 F F F
202482024-02-12T01:43:15.537Z 8 160-179 183 183 183 328 328 328 F F F
202492024-02-12T01:43:15.537Z 9 180-199 183 183 183 328 328 328 F F F
202502024-02-12T01:43:15.537Z 10 200-219 175 175 175 315 315 315 F F F
202512024-02-12T01:43:15.541Z 11 220-239 178 178 178 323 323 323 F F F
202522024-02-12T01:43:15.541Z 12 240-259 174 174 174 314 314 314 F F F
202532024-02-12T01:43:15.541Z 13 260-279 179 179 179 324 324 324 F F F
202542024-02-12T01:43:15.541Z 14 280-299 182 182 182 326 326 326 F F F
202552024-02-12T01:43:15.541Z 15 300-319 180 180 180 325 325 325 F F F
202562024-02-12T01:43:15.541Z 16 320-339 180 180 180 325 325 325 F F F
202572024-02-12T01:43:15.541Z 17 340-359 178 178 178 321 321 321 F F F
202582024-02-12T01:43:15.541Z 18 360-379 180 180 180 325 325 325 F F F
202592024-02-12T01:43:15.541Z 19 380-399 180 180 180 325 325 325 F F F
202602024-02-12T01:43:15.541Z 20 400-419 178 178 178 322 322 322 F F F
202612024-02-12T01:43:15.541Z 21 420-439 178 178 178 322 322 322 F F F
202622024-02-12T01:43:15.542Z 22 440-459 182 182 182 327 327 327 F F F
202632024-02-12T01:43:15.542Z 23 460-479 182 182 182 327 327 327 F F F
202642024-02-12T01:43:15.542Z 24 480-499 180 180 180 325 325 325 F F F
202652024-02-12T01:43:15.542Z 25 500-519 182 182 182 327 327 327 F F F
202662024-02-12T01:43:15.542Z 26 520-539 182 182 182 327 327 327 F F F
202672024-02-12T01:43:15.542Z 27 540-559 182 182 182 326 326 326 F F F
202682024-02-12T01:43:15.542Z 28 560-579 180 180 180 325 325 325 F F F
202692024-02-12T01:43:15.542Z 29 580-599 182 182 182 326 326 326 F F F
202702024-02-12T01:43:15.542ZMax gen: 183, Max flush: 328
202712024-02-12T01:43:15.542Z
202722024-02-12T01:43:15.542ZBegin loop 92
202732024-02-12T01:43:15.542ZDownstairs to restart: 0
202742024-02-12T01:43:15.560Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423204057Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
202752024-02-12T01:43:15.560Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423442786Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
202762024-02-12T01:43:15.560Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423469824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
202772024-02-12T01:43:15.565Z{"msg":"Crucible 3421f537-ae0c-44f6-8cdf-2047155f435c has session id: f031d130-e771-4f78-925a-4cca779ddab1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423501923Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871Crucible runtime is spawned
202782024-02-12T01:43:15.565Z,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202792024-02-12T01:43:15.565Z{"msg":"Upstairs opts: Upstairs UUID: 3421f537-ae0c-44f6-8cdf-2047155f435c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423600808Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202802024-02-12T01:43:15.565Z{"msg":"Crucible stats registered with UUID: 3421f537-ae0c-44f6-8cdf-2047155f435c","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423639216Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202812024-02-12T01:43:15.565Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423674835Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
202822024-02-12T01:43:15.565Z{"msg":"3421f537-ae0c-44f6-8cdf-2047155f435c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423755191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202832024-02-12T01:43:15.565Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423791319Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202842024-02-12T01:43:15.565Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423809038Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202852024-02-12T01:43:15.565Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423830487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202862024-02-12T01:43:15.565Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423867245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202872024-02-12T01:43:15.565Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423905824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202882024-02-12T01:43:15.565Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.423948421Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202892024-02-12T01:43:15.565Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.42398568Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202902024-02-12T01:43:15.565Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424025528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202912024-02-12T01:43:15.565Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424060226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202922024-02-12T01:43:15.565Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424087455Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202932024-02-12T01:43:15.565Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424121943Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202942024-02-12T01:43:15.565Z{"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:14.424152932Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202952024-02-12T01:43:15.565Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:14.42418551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202962024-02-12T01:43:15.565Z{"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:14.424223939Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202972024-02-12T01:43:15.566Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424245667Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202982024-02-12T01:43:15.566Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424266976Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
202992024-02-12T01:43:15.566Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424290265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203002024-02-12T01:43:15.566Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424663267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203012024-02-12T01:43:15.566Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424702366Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203022024-02-12T01:43:15.566Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424728684Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203032024-02-12T01:43:15.566Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.424752953Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203042024-02-12T01:43:15.566Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.42523685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203052024-02-12T01:43:15.566Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425401132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203062024-02-12T01:43:15.566Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425681749Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203072024-02-12T01:43:15.566Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425706858Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203082024-02-12T01:43:15.566Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425722487Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203092024-02-12T01:43:15.566Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425850271Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203102024-02-12T01:43:15.566Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425896049Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203112024-02-12T01:43:15.566Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:14.425914958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203122024-02-12T01:43:16.562Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.425591924Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203132024-02-12T01:43:16.567Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.425741507Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203142024-02-12T01:43:16.567Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.42651691Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203152024-02-12T01:43:16.567Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.426547019Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203162024-02-12T01:43:16.567Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427037005Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203172024-02-12T01:43:16.567Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427528672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203182024-02-12T01:43:16.567Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.42756236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203192024-02-12T01:43:16.567Z{"msg":"[0]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 327, 327, 325, 328, 328, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427581389Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203202024-02-12T01:43:16.567Z{"msg":"[0]R generation[0..12]: [182, 183, 177, 178, 180, 182, 182, 180, 183, 183, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427616317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203212024-02-12T01:43:16.568Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427643726Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203222024-02-12T01:43:16.568Z{"msg":"[1]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 327, 327, 325, 328, 328, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427676765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203232024-02-12T01:43:16.568Z{"msg":"[1]R generation[0..12]: [182, 183, 177, 178, 180, 182, 182, 180, 183, 183, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427697033Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203242024-02-12T01:43:16.568Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427723562Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203252024-02-12T01:43:16.568Z{"msg":"[2]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 327, 327, 325, 328, 328, 315, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427741482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203262024-02-12T01:43:16.568ZWait for a query_work_queue command to finish before sending IO
203272024-02-12T01:43:16.568Z{"msg":"[2]R generation[0..12]: [182, 183, 177, 178, 180, 182, 182, 180, 183, 183, 175, 178]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.42777579Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203282024-02-12T01:43:16.568Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427795169Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203292024-02-12T01:43:16.568Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427821268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203302024-02-12T01:43:16.568Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427839227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203312024-02-12T01:43:16.568Z{"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427863516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203322024-02-12T01:43:16.568Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427880315Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203332024-02-12T01:43:16.568Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427905234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203342024-02-12T01:43:16.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427920983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203352024-02-12T01:43:16.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427946152Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203362024-02-12T01:43:16.568Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.427962991Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203372024-02-12T01:43:16.568Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.42798789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203382024-02-12T01:43:16.568Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.428003809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203392024-02-12T01:43:16.568Z{"msg":"3421f537-ae0c-44f6-8cdf-2047155f435c is now active with session: f031d130-e771-4f78-925a-4cca779ddab1","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.428020718Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203402024-02-12T01:43:16.568Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.428044137Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871,"session_id":"f031d130-e771-4f78-925a-4cca779ddab1"}
203412024-02-12T01:43:16.568Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:15.428059886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
203422024-02-12T01:43:16.568ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203432024-02-12T01:43:16.568ZLoading write count information from file "/tmp/test_repair_verify.data"
203442024-02-12T01:43:16.568ZRun Repair workload
203452024-02-12T01:43:16.568Z01/30 Write block 509 len 4096 data: 9
203462024-02-12T01:43:16.572Z02/30 Write block 107 len 12288 data: 15 16 14
203472024-02-12T01:43:16.572Z03/30 Read block 244 len 28672
203482024-02-12T01:43:16.572Z04/30 Read block 571 len 32768
203492024-02-12T01:43:16.572Z05/30 Read block 355 len 4096
203502024-02-12T01:43:16.572Z06/30 Write block 383 len 36864 data: 12 13 15 17 16 17 17 20 19
203512024-02-12T01:43:16.572Z07/30 Write block 352 len 8192 data: 19 15
203522024-02-12T01:43:16.572Z08/30 Write block 264 len 8192 data: 12 16
203532024-02-12T01:43:16.572Z09/30 Read block 10 len 36864
203542024-02-12T01:43:16.575Z10/30 Write block 125 len 28672 data: 17 20 19 17 16 14 17
203552024-02-12T01:43:16.575Z11/30 Flush
203562024-02-12T01:43:16.578Z12/30 Write block 225 len 8192 data: 14 10
203572024-02-12T01:43:16.578Z13/30 Write block 357 len 8192 data: 8 7
203582024-02-12T01:43:16.578Z14/30 Flush
203592024-02-12T01:43:16.581Z15/30 Write block 316 len 40960 data: 12 12 13 14 14 12 14 11 12 11
203602024-02-12T01:43:16.584Z16/30 Write block 192 len 20480 data: 14 13 11 10 12
203612024-02-12T01:43:16.584Z17/30 Read block 500 len 36864
203622024-02-12T01:43:16.584Z18/30 Read block 558 len 36864
203632024-02-12T01:43:16.587Z19/30 Read block 120 len 32768
203642024-02-12T01:43:16.587Z20/30 Write block 543 len 20480 data: 11 11 12 14 14
203652024-02-12T01:43:16.587Z21/30 Write block 545 len 4096 data: 13
203662024-02-12T01:43:16.587Z22/30 Read block 187 len 24576
203672024-02-12T01:43:16.590Z23/30 Flush
203682024-02-12T01:43:16.593Z24/30 Write block 459 len 40960 data: 14 11 11 12 12 11 9 11 12 14
203692024-02-12T01:43:16.593Z25/30 Read block 10 len 4096
203702024-02-12T01:43:16.593Z26/30 Write block 192 len 28672 data: 15 14 12 11 13 13 12
203712024-02-12T01:43:16.593Z27/30 Write block 182 len 24576 data: 19 20 21 16 17 17
203722024-02-12T01:43:16.597Z28/30 Write block 500 len 24576 data: 13 14 14 12 12 13
203732024-02-12T01:43:16.597Z29/30 Read block 202 len 28672
203742024-02-12T01:43:16.597Z30/30 Write block 342 len 16384 data: 9 12 10 10
203752024-02-12T01:43:16.597Z----------------------------------------------------------------
203762024-02-12T01:43:16.597Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
203772024-02-12T01:43:16.597ZStates: Active Active Active
203782024-02-12T01:43:16.597ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203792024-02-12T01:43:16.597Z 1 Acked 1000 Write 1 Sent Done Done false
203802024-02-12T01:43:16.597Z 2 Acked 1001 Write 3 Sent Done Done false
203812024-02-12T01:43:16.597Z 3 Acked 1002 Read 7 Sent Done Done false
203822024-02-12T01:43:16.597Z 4 Acked 1003 Read 8 Sent Done Done false
203832024-02-12T01:43:16.597Z 5 Acked 1004 Read 1 Sent Done Done false
203842024-02-12T01:43:16.597Z 6 Acked 1005 Write 9 Sent Done Done false
203852024-02-12T01:43:16.597Z 7 Acked 1006 Write 2 Sent Done Done false
203862024-02-12T01:43:16.597Z 8 Acked 1007 Write 2 Sent Done Done false
203872024-02-12T01:43:16.597Z 9 Acked 1008 Read 9 Sent Done Done false
203882024-02-12T01:43:16.598Z 10 Acked 1009 Write 7 Sent Done Done false
203892024-02-12T01:43:16.598Z 11 Acked 1010 Flush 0 Sent Done Done false
203902024-02-12T01:43:16.598Z 12 Acked 1011 Write 2 Sent Done Done false
203912024-02-12T01:43:16.598Z 13 Acked 1012 Write 2 Sent Done Done false
203922024-02-12T01:43:16.598Z 14 Acked 1013 Flush 0 Sent Done Done false
203932024-02-12T01:43:16.598Z 15 Acked 1014 Write 10 Sent Done Done false
203942024-02-12T01:43:16.598Z 16 Acked 1015 Write 5 Sent Done Done false
203952024-02-12T01:43:16.598Z 17 Acked 1016 Read 9 Sent Done Done false
203962024-02-12T01:43:16.598Z 18 Acked 1017 Read 9 Sent Done Done false
203972024-02-12T01:43:16.598Z 19 Acked 1018 Read 8 Sent Done Done false
203982024-02-12T01:43:16.598Z 20 Acked 1019 Write 5 Sent Done Done false
203992024-02-12T01:43:16.598Z 21 Acked 1020 Write 1 Sent Done Done false
204002024-02-12T01:43:16.598Z 22 Acked 1021 Read 6 Sent Done Done false
204012024-02-12T01:43:16.598Z 23 Acked 1022 Flush 0 Sent Done Done false
204022024-02-12T01:43:16.598Z 24 Acked 1023 Write 10 Sent Done Done false
204032024-02-12T01:43:16.598Z 25 Acked 1024 Read 1 Sent Done Done false
204042024-02-12T01:43:16.598Z 26 Acked 1025 Write 7 Sent Done Done false
204052024-02-12T01:43:16.598Z 27 Acked 1026 Write 6 Sent Done Done false
204062024-02-12T01:43:16.598Z 28 Acked 1027 Write 6 Sent Done Done false
204072024-02-12T01:43:16.598Z 29 Acked 1028 Read 7 Sent Done Done false
204082024-02-12T01:43:16.598Z 30 Acked 1029 Write 4 Sent Sent Sent false
204092024-02-12T01:43:16.598Z STATES DS:0 DS:1 DS:2 TOTAL
204102024-02-12T01:43:16.598Z New 0 0 0 0
204112024-02-12T01:43:16.599Z Sent 30 1 1 32
204122024-02-12T01:43:16.599Z Done 0 29 29 58
204132024-02-12T01:43:16.599Z Skipped 0 0 0 0
204142024-02-12T01:43:16.599Z Error 0 0 0 0
204152024-02-12T01:43:16.599ZLast Flush: 0 1022 1022
204162024-02-12T01:43:16.599ZDownstairs last five completed:
204172024-02-12T01:43:16.599ZUpstairs last five completed: 30 29 28 27 26
204182024-02-12T01:43:16.602Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:15.460152115Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1871}
204192024-02-12T01:43:16.602ZWrote out file "/tmp/test_repair_verify.data"
204202024-02-12T01:43:16.602Z
204212024-02-12T01:43:16.608ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204222024-02-12T01:43:16.622ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204232024-02-12T01:43:16.622Z 0 000-019 182 182 182 327 327 327 F F F
204242024-02-12T01:43:16.622Z 1 020-039 183 183 183 328 328 328 F F F
204252024-02-12T01:43:16.622Z 2 040-059 177 177 177 318 318 318 F F F
204262024-02-12T01:43:16.622Z 3 060-079 178 178 178 321 321 321 F F F
204272024-02-12T01:43:16.622Z 4 080-099 180 180 180 325 325 325 F F F
204282024-02-12T01:43:16.622Z 5 100-119 182 184 184 327 329 329 F F F <---
204292024-02-12T01:43:16.622Z 6 120-139 182 184 184 327 329 329 F F F <---
204302024-02-12T01:43:16.622Z 7 140-159 180 180 180 325 325 325 F F F
204312024-02-12T01:43:16.622Z 8 160-179 183 183 183 328 328 328 F F F
204322024-02-12T01:43:16.626Z 9 180-199 183 184 184 328 331 331 F T T <---
204332024-02-12T01:43:16.626Z 10 200-219 175 175 175 315 315 315 F F F
204342024-02-12T01:43:16.626Z 11 220-239 178 184 184 323 330 330 F F F <---
204352024-02-12T01:43:16.626Z 12 240-259 174 174 174 314 314 314 F F F
204362024-02-12T01:43:16.626Z 13 260-279 179 184 184 324 329 329 F F F <---
204372024-02-12T01:43:16.626Z 14 280-299 182 182 182 326 326 326 F F F
204382024-02-12T01:43:16.626Z 15 300-319 180 184 184 325 331 331 F F F <---
204392024-02-12T01:43:16.626Z 16 320-339 180 184 184 325 331 331 F F F <---
204402024-02-12T01:43:16.626Z 17 340-359 178 184 184 321 330 330 F T T <---
204412024-02-12T01:43:16.626Z 18 360-379 180 180 180 325 325 325 F F F
204422024-02-12T01:43:16.626Z 19 380-399 180 184 184 325 329 329 F F F <---
204432024-02-12T01:43:16.626Z 20 400-419 178 178 178 322 322 322 F F F
204442024-02-12T01:43:16.626Z 21 420-439 178 178 178 322 322 322 F F F
204452024-02-12T01:43:16.626Z 22 440-459 182 182 182 327 327 327 F T T <---
204462024-02-12T01:43:16.626Z 23 460-479 182 182 182 327 327 327 F T T <---
204472024-02-12T01:43:16.626Z 24 480-499 180 180 180 325 325 325 F F F
204482024-02-12T01:43:16.626Z 25 500-519 182 184 184 327 329 329 F T T <---
204492024-02-12T01:43:16.627Z 26 520-539 182 182 182 327 327 327 F F F
204502024-02-12T01:43:16.627Z 27 540-559 182 184 184 326 331 331 F F F <---
204512024-02-12T01:43:16.627Z 28 560-579 180 180 180 325 325 325 F F F
204522024-02-12T01:43:16.627Z 29 580-599 182 182 182 326 326 326 F F F
204532024-02-12T01:43:16.627ZMax gen: 184, Max flush: 331
204542024-02-12T01:43:16.627ZError: Difference in extent metadata found!
204552024-02-12T01:43:16.627ZA Difference in extent metadata is expected here
204562024-02-12T01:43:16.627ZOn loop 92
204572024-02-12T01:43:16.627Z
204582024-02-12T01:43:16.630ZVerifying data now
204592024-02-12T01:43:21.267ZNo change after verify
204602024-02-12T01:43:21.267ZLoop: 92 Downstairs dump after verify (and repair):
204612024-02-12T01:43:21.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204622024-02-12T01:43:21.282Z 0 000-019 182 182 182 327 327 327 F F F
204632024-02-12T01:43:21.282Z 1 020-039 183 183 183 328 328 328 F F F
204642024-02-12T01:43:21.282Z 2 040-059 177 177 177 318 318 318 F F F
204652024-02-12T01:43:21.282Z 3 060-079 178 178 178 321 321 321 F F F
204662024-02-12T01:43:21.282Z 4 080-099 180 180 180 325 325 325 F F F
204672024-02-12T01:43:21.282Z 5 100-119 184 184 184 329 329 329 F F F
204682024-02-12T01:43:21.282Z 6 120-139 184 184 184 329 329 329 F F F
204692024-02-12T01:43:21.282Z 7 140-159 180 180 180 325 325 325 F F F
204702024-02-12T01:43:21.282Z 8 160-179 183 183 183 328 328 328 F F F
204712024-02-12T01:43:21.285Z 9 180-199 185 185 185 332 332 332 F F F
204722024-02-12T01:43:21.286Z 10 200-219 175 175 175 315 315 315 F F F
204732024-02-12T01:43:21.286Z 11 220-239 184 184 184 330 330 330 F F F
204742024-02-12T01:43:21.286Z 12 240-259 174 174 174 314 314 314 F F F
204752024-02-12T01:43:21.286Z 13 260-279 184 184 184 329 329 329 F F F
204762024-02-12T01:43:21.286Z 14 280-299 182 182 182 326 326 326 F F F
204772024-02-12T01:43:21.286Z 15 300-319 184 184 184 331 331 331 F F F
204782024-02-12T01:43:21.286Z 16 320-339 184 184 184 331 331 331 F F F
204792024-02-12T01:43:21.286Z 17 340-359 185 185 185 332 332 332 F F F
204802024-02-12T01:43:21.286Z 18 360-379 180 180 180 325 325 325 F F F
204812024-02-12T01:43:21.286Z 19 380-399 184 184 184 329 329 329 F F F
204822024-02-12T01:43:21.286Z 20 400-419 178 178 178 322 322 322 F F F
204832024-02-12T01:43:21.286Z 21 420-439 178 178 178 322 322 322 F F F
204842024-02-12T01:43:21.286Z 22 440-459 185 185 185 332 332 332 F F F
204852024-02-12T01:43:21.286Z 23 460-479 185 185 185 332 332 332 F F F
204862024-02-12T01:43:21.286Z 24 480-499 180 180 180 325 325 325 F F F
204872024-02-12T01:43:21.286Z 25 500-519 185 185 185 332 332 332 F F F
204882024-02-12T01:43:21.286Z 26 520-539 182 182 182 327 327 327 F F F
204892024-02-12T01:43:21.286Z 27 540-559 184 184 184 331 331 331 F F F
204902024-02-12T01:43:21.286Z 28 560-579 180 180 180 325 325 325 F F F
204912024-02-12T01:43:21.286Z 29 580-599 182 182 182 326 326 326 F F F
204922024-02-12T01:43:21.286ZMax gen: 185, Max flush: 332
204932024-02-12T01:43:21.286Z
204942024-02-12T01:43:21.286ZBegin loop 93
204952024-02-12T01:43:21.287ZDownstairs to restart: 0
204962024-02-12T01:43:21.304Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.166791536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
204972024-02-12T01:43:21.304Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
204982024-02-12T01:43:21.304Z,"time":"2024-02-12T01:43:20.167047154Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
204992024-02-12T01:43:21.309Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167096272Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
205002024-02-12T01:43:21.309Z{"msg":"Crucible 1e23fcff-e0ff-4485-8c5f-f71110dbb1b8 has session id: c060021f-93fd-4e94-a954-6fc3afc893e3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.16713614Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205012024-02-12T01:43:21.309Z{"msg":"Upstairs opts: Upstairs UUID: 1e23fcff-e0ff-4485-8c5f-f71110dbb1b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167180248Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205022024-02-12T01:43:21.309Z{"msg":"Crucible stats registered with UUID: 1e23fcff-e0ff-4485-8c5f-f71110dbb1b8","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167223226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205032024-02-12T01:43:21.309Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167244985Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
205042024-02-12T01:43:21.309Z{"msg":"1e23fcff-e0ff-4485-8c5f-f71110dbb1b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167287823Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205052024-02-12T01:43:21.309Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167308131Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205062024-02-12T01:43:21.309Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.16733857Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205072024-02-12T01:43:21.309Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167354129Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205082024-02-12T01:43:21.309Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167385788Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205092024-02-12T01:43:21.309Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167402287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205102024-02-12T01:43:21.309Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167433865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205112024-02-12T01:43:21.309Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167453275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205122024-02-12T01:43:21.309Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167483473Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205132024-02-12T01:43:21.309Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167528071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205142024-02-12T01:43:21.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.1675529Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205152024-02-12T01:43:21.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167578398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205162024-02-12T01:43:21.309Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.167735141Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205172024-02-12T01:43:21.309Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168219638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205182024-02-12T01:43:21.309Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168292015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205192024-02-12T01:43:21.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168342122Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205202024-02-12T01:43:21.310Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168376891Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205212024-02-12T01:43:21.310Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168793601Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205222024-02-12T01:43:21.310Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.168918035Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205232024-02-12T01:43:21.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169350104Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205242024-02-12T01:43:21.310Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169397052Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205252024-02-12T01:43:21.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169420561Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205262024-02-12T01:43:21.310Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.16943846Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205272024-02-12T01:43:21.310Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169457029Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205282024-02-12T01:43:21.310Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169471748Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205292024-02-12T01:43:21.310Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169809112Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205302024-02-12T01:43:21.310Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.169842681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205312024-02-12T01:43:21.310Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170315068Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205322024-02-12T01:43:21.314Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170777156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205332024-02-12T01:43:21.314Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170814234Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205342024-02-12T01:43:21.314Z{"msg":"[0]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 329, 329, 325, 328, 332, 315, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170851742Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205352024-02-12T01:43:21.314Z{"msg":"[0]R generation[0..12]: [182, 183, 177, 178, 180, 184, 184, 180, 183, 185, 175, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170875761Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205362024-02-12T01:43:21.314Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.17089086Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205372024-02-12T01:43:21.314Z{"msg":"[1]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 329, 329, 325, 328, 332, 315, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.17091375Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205382024-02-12T01:43:21.314Z{"msg":"[1]R generation[0..12]: [182, 183, 177, 178, 180, 184, 184, 180, 183, 185, 175, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170929259Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205392024-02-12T01:43:21.314Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170951478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205402024-02-12T01:43:21.315Z{"msg":"[2]R flush_numbers[0..12]: [327, 328, 318, 321, 325, 329, 329, 325, 328, 332, 315, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170966027Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205412024-02-12T01:43:21.315Z{"msg":"Wait for a query_work_queue command to finish before sending IO
205422024-02-12T01:43:21.315Z[2]R generation[0..12]: [182, 183, 177, 178, 180, 184, 184, 180, 183, 185, 175, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.170988886Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205432024-02-12T01:43:21.315Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171004075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205442024-02-12T01:43:21.315Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171025414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205452024-02-12T01:43:21.315Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171040133Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205462024-02-12T01:43:21.315Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171061752Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205472024-02-12T01:43:21.315Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171076612Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205482024-02-12T01:43:21.315Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171090961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205492024-02-12T01:43:21.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.17110372Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205502024-02-12T01:43:21.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171130249Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205512024-02-12T01:43:21.315Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171144528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205522024-02-12T01:43:21.315Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171158378Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205532024-02-12T01:43:21.315Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171172127Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205542024-02-12T01:43:21.315Z{"msg":"1e23fcff-e0ff-4485-8c5f-f71110dbb1b8 is now active with session: c060021f-93fd-4e94-a954-6fc3afc893e3","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171185186Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205552024-02-12T01:43:21.315Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171205136Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879,"session_id":"c060021f-93fd-4e94-a954-6fc3afc893e3"}
205562024-02-12T01:43:21.315Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:20.171217585Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
205572024-02-12T01:43:21.315ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205582024-02-12T01:43:21.315ZLoading write count information from file "/tmp/test_repair_verify.data"
205592024-02-12T01:43:21.315ZRun Repair workload
205602024-02-12T01:43:21.315Z01/30 Write block 462 len 36864 data: 13 13 12 10 12 13 15 13 13
205612024-02-12T01:43:21.315Z02/30 Read block 106 len 36864
205622024-02-12T01:43:21.315Z03/30 Read block 83 len 4096
205632024-02-12T01:43:21.316Z04/30 Read block 269 len 32768
205642024-02-12T01:43:21.319Z05/30 Write block 268 len 36864 data: 17 16 18 20 20 19 20 14 10
205652024-02-12T01:43:21.319Z06/30 Read block 490 len 8192
205662024-02-12T01:43:21.319Z07/30 Read block 140 len 12288
205672024-02-12T01:43:21.319Z08/30 Read block 197 len 12288
205682024-02-12T01:43:21.319Z09/30 Read block 382 len 20480
205692024-02-12T01:43:21.319Z10/30 Read block 86 len 4096
205702024-02-12T01:43:21.319Z11/30 Flush
205712024-02-12T01:43:21.322Z12/30 Write block 240 len 24576 data: 14 12 14 17 14 12
205722024-02-12T01:43:21.322Z13/30 Read block 426 len 8192
205732024-02-12T01:43:21.325Z14/30 Read block 89 len 12288
205742024-02-12T01:43:21.325Z15/30 Write block 526 len 16384 data: 10 9 10 11
205752024-02-12T01:43:21.325Z16/30 Write block 203 len 12288 data: 18 15 15
205762024-02-12T01:43:21.325Z17/30 Write block 26 len 24576 data: 15 14 16 19 17 16
205772024-02-12T01:43:21.328Z18/30 Read block 501 len 12288
205782024-02-12T01:43:21.328Z19/30 Read block 336 len 40960
205792024-02-12T01:43:21.328Z20/30 Flush
205802024-02-12T01:43:21.331Z21/30 Write block 299 len 16384 data: 10 10 9 8
205812024-02-12T01:43:21.331Z22/30 Read block 76 len 4096
205822024-02-12T01:43:21.334Z23/30 Write block 2 len 36864 data: 6 7 8 11 12 10 10 11 10
205832024-02-12T01:43:21.334Z24/30 Read block 171 len 32768
205842024-02-12T01:43:21.337Z25/30 Read block 12 len 40960
205852024-02-12T01:43:21.337Z26/30 Write block 580 len 4096 data: 12
205862024-02-12T01:43:21.337Z27/30 Read block 515 len 32768
205872024-02-12T01:43:21.340Z28/30 Write block 554 len 40960 data: 12 11 10 9 10 11 9 13 11 14
205882024-02-12T01:43:21.340Z29/30 Write block 392 len 36864 data: 19 19 18 13 12 12 13 11 14
205892024-02-12T01:43:21.340Z30/30 Write block 304 len 16384 data: 10 14 15 17
205902024-02-12T01:43:21.340Z----------------------------------------------------------------
205912024-02-12T01:43:21.340Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
205922024-02-12T01:43:21.340ZStates: Active Active Active
205932024-02-12T01:43:21.341ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
205942024-02-12T01:43:21.341Z 1 Acked 1000 Write 9 Sent Done Done false
205952024-02-12T01:43:21.341Z 2 Acked 1001 Read 9 Sent Done Done false
205962024-02-12T01:43:21.341Z 3 Acked 1002 Read 1 Sent Done Done false
205972024-02-12T01:43:21.341Z 4 Acked 1003 Read 8 Sent Done Done false
205982024-02-12T01:43:21.341Z 5 Acked 1004 Write 9 Sent Done Done false
205992024-02-12T01:43:21.341Z 6 Acked 1005 Read 2 Sent Done Done false
206002024-02-12T01:43:21.341Z 7 Acked 1006 Read 3 Sent Done Done false
206012024-02-12T01:43:21.341Z 8 Acked 1007 Read 3 Sent Done Done false
206022024-02-12T01:43:21.341Z 9 Acked 1008 Read 5 Sent Done Done false
206032024-02-12T01:43:21.341Z 10 Acked 1009 Read 1 Sent Done Done false
206042024-02-12T01:43:21.341Z 11 Acked 1010 Flush 0 Sent Done Done false
206052024-02-12T01:43:21.341Z 12 Acked 1011 Write 6 Sent Done Done false
206062024-02-12T01:43:21.341Z 13 Acked 1012 Read 2 Sent Done Done false
206072024-02-12T01:43:21.341Z 14 Acked 1013 Read 3 Sent Done Done false
206082024-02-12T01:43:21.341Z 15 Acked 1014 Write 4 Sent Done Done false
206092024-02-12T01:43:21.341Z 16 Acked 1015 Write 3 Sent Done Done false
206102024-02-12T01:43:21.341Z 17 Acked 1016 Write 6 Sent Done Done false
206112024-02-12T01:43:21.341Z 18 Acked 1017 Read 3 Sent Done Done false
206122024-02-12T01:43:21.341Z 19 Acked 1018 Read 10 Sent Done Done false
206132024-02-12T01:43:21.341Z 20 Acked 1019 Flush 0 Sent Done Done false
206142024-02-12T01:43:21.341Z 21 Acked 1020 Write 4 Sent Done Done false
206152024-02-12T01:43:21.341Z 22 Acked 1021 Read 1 Sent Done Done false
206162024-02-12T01:43:21.341Z 23 Acked 1022 Write 9 Sent Done Done false
206172024-02-12T01:43:21.341Z 24 Acked 1023 Read 8 Sent Done Done false
206182024-02-12T01:43:21.342Z 25 Acked 1024 Read 10 Sent Done Done false
206192024-02-12T01:43:21.342Z 26 Acked 1025 Write 1 Sent Done Done false
206202024-02-12T01:43:21.342Z 27 Acked 1026 Read 8 Sent Done Done false
206212024-02-12T01:43:21.342Z 28 Acked 1027 Write 10 Sent Done Done false
206222024-02-12T01:43:21.342Z 29 Acked 1028 Write 9 Sent Sent Sent false
206232024-02-12T01:43:21.342Z 30 Acked 1029 Write 4 Sent Sent Sent false
206242024-02-12T01:43:21.342Z STATES DS:0 DS:1 DS:2 TOTAL
206252024-02-12T01:43:21.342Z New 0 0 0 0
206262024-02-12T01:43:21.342Z Sent 30 2 2 34
206272024-02-12T01:43:21.342Z Done 0 28 28 56
206282024-02-12T01:43:21.342Z Skipped 0 0 0 0
206292024-02-12T01:43:21.342Z Error 0 0 0 0
206302024-02-12T01:43:21.342ZLast Flush: 0 1019 1019
206312024-02-12T01:43:21.342ZDownstairs last five completed:
206322024-02-12T01:43:21.342ZUpstairs last five completed: 30 29 28 27 26
206332024-02-12T01:43:21.342Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:20.202518182Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1879}
206342024-02-12T01:43:21.342ZWrote out file "/tmp/test_repair_verify.data"
206352024-02-12T01:43:21.345Z
206362024-02-12T01:43:21.348ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206372024-02-12T01:43:22.369ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206382024-02-12T01:43:22.369Z 0 000-019 182 182 182 327 327 327 F T T <---
206392024-02-12T01:43:22.369Z 1 020-039 183 186 186 328 334 334 F F F <---
206402024-02-12T01:43:22.369Z 2 040-059 177 177 177 318 318 318 F F F
206412024-02-12T01:43:22.369Z 3 060-079 178 178 178 321 321 321 F F F
206422024-02-12T01:43:22.369Z 4 080-099 180 180 180 325 325 325 F F F
206432024-02-12T01:43:22.369Z 5 100-119 184 184 184 329 329 329 F F F
206442024-02-12T01:43:22.369Z 6 120-139 184 184 184 329 329 329 F F F
206452024-02-12T01:43:22.369Z 7 140-159 180 180 180 325 325 325 F F F
206462024-02-12T01:43:22.369Z 8 160-179 183 183 183 328 328 328 F F F
206472024-02-12T01:43:22.369Z 9 180-199 185 185 185 332 332 332 F F F
206482024-02-12T01:43:22.369Z 10 200-219 175 186 186 315 334 334 F F F <---
206492024-02-12T01:43:22.369Z 11 220-239 184 184 184 330 330 330 F F F
206502024-02-12T01:43:22.370Z 12 240-259 174 186 186 314 334 334 F F F <---
206512024-02-12T01:43:22.373Z 13 260-279 184 186 186 329 333 333 F F F <---
206522024-02-12T01:43:22.373Z 14 280-299 182 182 182 326 326 326 F T T <---
206532024-02-12T01:43:22.373Z 15 300-319 184 184 184 331 331 331 F T T <---
206542024-02-12T01:43:22.373Z 16 320-339 184 184 184 331 331 331 F F F
206552024-02-12T01:43:22.373Z 17 340-359 185 185 185 332 332 332 F F F
206562024-02-12T01:43:22.373Z 18 360-379 180 180 180 325 325 325 F F F
206572024-02-12T01:43:22.373Z 19 380-399 184 184 184 329 329 329 F T T <---
206582024-02-12T01:43:22.373Z 20 400-419 178 178 178 322 322 322 F T T <---
206592024-02-12T01:43:22.373Z 21 420-439 178 178 178 322 322 322 F F F
206602024-02-12T01:43:22.373Z 22 440-459 185 185 185 332 332 332 F F F
206612024-02-12T01:43:22.373Z 23 460-479 185 186 186 332 333 333 F F F <---
206622024-02-12T01:43:22.374Z 24 480-499 180 180 180 325 325 325 F F F
206632024-02-12T01:43:22.374Z 25 500-519 185 185 185 332 332 332 F F F
206642024-02-12T01:43:22.374Z 26 520-539 182 186 186 327 334 334 F F F <---
206652024-02-12T01:43:22.374Z 27 540-559 184 184 184 331 331 331 F T T <---
206662024-02-12T01:43:22.374Z 28 560-579 180 180 180 325 325 325 F T T <---
206672024-02-12T01:43:22.374Z 29 580-599 182 182 182 326 326 326 F T T <---
206682024-02-12T01:43:22.374ZMax gen: 186, Max flush: 334
206692024-02-12T01:43:22.374ZError: Difference in extent metadata found!
206702024-02-12T01:43:22.374ZA Difference in extent metadata is expected here
206712024-02-12T01:43:22.374ZOn loop 93
206722024-02-12T01:43:22.374Z
206732024-02-12T01:43:22.374ZVerifying data now
206742024-02-12T01:43:26.062ZNo change after verify
206752024-02-12T01:43:26.062ZLoop: 93 Downstairs dump after verify (and repair):
206762024-02-12T01:43:26.078ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206772024-02-12T01:43:26.078Z 0 000-019 187 187 187 335 335 335 F F F
206782024-02-12T01:43:26.078Z 1 020-039 186 186 186 334 334 334 F F F
206792024-02-12T01:43:26.078Z 2 040-059 177 177 177 318 318 318 F F F
206802024-02-12T01:43:26.078Z 3 060-079 178 178 178 321 321 321 F F F
206812024-02-12T01:43:26.078Z 4 080-099 180 180 180 325 325 325 F F F
206822024-02-12T01:43:26.078Z 5 100-119 184 184 184 329 329 329 F F F
206832024-02-12T01:43:26.078Z 6 120-139 184 184 184 329 329 329 F F F
206842024-02-12T01:43:26.078Z 7 140-159 180 180 180 325 325 325 F F F
206852024-02-12T01:43:26.078Z 8 160-179 183 183 183 328 328 328 F F F
206862024-02-12T01:43:26.078Z 9 180-199 185 185 185 332 332 332 F F F
206872024-02-12T01:43:26.078Z 10 200-219 186 186 186 334 334 334 F F F
206882024-02-12T01:43:26.078Z 11 220-239 184 184 184 330 330 330 F F F
206892024-02-12T01:43:26.078Z 12 240-259 186 186 186 334 334 334 F F F
206902024-02-12T01:43:26.082Z 13 260-279 186 186 186 333 333 333 F F F
206912024-02-12T01:43:26.082Z 14 280-299 187 187 187 335 335 335 F F F
206922024-02-12T01:43:26.082Z 15 300-319 187 187 187 335 335 335 F F F
206932024-02-12T01:43:26.082Z 16 320-339 184 184 184 331 331 331 F F F
206942024-02-12T01:43:26.082Z 17 340-359 185 185 185 332 332 332 F F F
206952024-02-12T01:43:26.082Z 18 360-379 180 180 180 325 325 325 F F F
206962024-02-12T01:43:26.082Z 19 380-399 187 187 187 335 335 335 F F F
206972024-02-12T01:43:26.082Z 20 400-419 187 187 187 335 335 335 F F F
206982024-02-12T01:43:26.082Z 21 420-439 178 178 178 322 322 322 F F F
206992024-02-12T01:43:26.082Z 22 440-459 185 185 185 332 332 332 F F F
207002024-02-12T01:43:26.082Z 23 460-479 186 186 186 333 333 333 F F F
207012024-02-12T01:43:26.082Z 24 480-499 180 180 180 325 325 325 F F F
207022024-02-12T01:43:26.082Z 25 500-519 185 185 185 332 332 332 F F F
207032024-02-12T01:43:26.082Z 26 520-539 186 186 186 334 334 334 F F F
207042024-02-12T01:43:26.082Z 27 540-559 187 187 187 335 335 335 F F F
207052024-02-12T01:43:26.082Z 28 560-579 187 187 187 335 335 335 F F F
207062024-02-12T01:43:26.082Z 29 580-599 187 187 187 335 335 335 F F F
207072024-02-12T01:43:26.082ZMax gen: 187, Max flush: 335
207082024-02-12T01:43:26.082Z
207092024-02-12T01:43:26.082ZBegin loop 94
207102024-02-12T01:43:26.082ZDownstairs to restart: 2
207112024-02-12T01:43:26.102Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.962920606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
207122024-02-12T01:43:26.102Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
207132024-02-12T01:43:26.102Z,"time":"2024-02-12T01:43:24.963214482Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
207142024-02-12T01:43:26.114Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963241381Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
207152024-02-12T01:43:26.114Z{"msg":"Crucible 58f8082f-dd4f-4ba5-abea-0d1cdf677296 has session id: 5df108fb-dd1e-4760-8f3d-1bc01ad50814","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.96326518Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207162024-02-12T01:43:26.114Z{"msg":"Upstairs opts: Upstairs UUID: 58f8082f-dd4f-4ba5-abea-0d1cdf677296, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963303898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207172024-02-12T01:43:26.114Z{"msg":"Crucible stats registered with UUID: 58f8082f-dd4f-4ba5-abea-0d1cdf677296","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963331047Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207182024-02-12T01:43:26.114Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963353706Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
207192024-02-12T01:43:26.114Z{"msg":"58f8082f-dd4f-4ba5-abea-0d1cdf677296 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963411983Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207202024-02-12T01:43:26.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.96347343Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207212024-02-12T01:43:26.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963501929Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207222024-02-12T01:43:26.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963527757Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207232024-02-12T01:43:26.114Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963574325Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207242024-02-12T01:43:26.114Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963611363Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207252024-02-12T01:43:26.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963653571Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207262024-02-12T01:43:26.115Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.96369163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207272024-02-12T01:43:26.115Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963728008Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207282024-02-12T01:43:26.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963766386Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207292024-02-12T01:43:26.115Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:24.963877741Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207302024-02-12T01:43:26.115Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:24.963930878Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207312024-02-12T01:43:26.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.963977836Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207322024-02-12T01:43:26.115Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:24.964022724Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207332024-02-12T01:43:26.115Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964063162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207342024-02-12T01:43:26.115Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964087101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207352024-02-12T01:43:26.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964119499Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207362024-02-12T01:43:26.115Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964144508Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207372024-02-12T01:43:26.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.96451852Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207382024-02-12T01:43:26.115Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964577747Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207392024-02-12T01:43:26.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964605006Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207402024-02-12T01:43:26.115Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.964642824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207412024-02-12T01:43:26.115Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965070744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207422024-02-12T01:43:26.115Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965198398Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207432024-02-12T01:43:26.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965609898Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207442024-02-12T01:43:26.115Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965656146Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207452024-02-12T01:43:26.115Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965673355Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207462024-02-12T01:43:26.115Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965698704Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207472024-02-12T01:43:26.115Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965713863Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207482024-02-12T01:43:26.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:24.965727633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207492024-02-12T01:43:27.104Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.965484865Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207502024-02-12T01:43:27.104Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.965664656Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207512024-02-12T01:43:27.109Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.966687917Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207522024-02-12T01:43:27.109Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.966733135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207532024-02-12T01:43:27.109Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967182884Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207542024-02-12T01:43:27.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967760226Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207552024-02-12T01:43:27.109Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967802884Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207562024-02-12T01:43:27.109Z{"msg":"[0]R flush_numbers[0..12]: [335, 334, 318, 321, 325, 329, 329, 325, 328, 332, 334, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967822613Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207572024-02-12T01:43:27.109Z{"msg":"[0]R generation[0..12]: [187, 186, 177, 178, 180, 184, 184, 180, 183, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967850442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207582024-02-12T01:43:27.109Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967868981Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207592024-02-12T01:43:27.109Z{"msg":"[1]R flush_numbers[0..12]: [335, 334, 318, 321, 325, 329, 329, 325, 328, 332, 334, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.96789513Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207602024-02-12T01:43:27.109Z{"msg":"[1]R generation[0..12]: [187, 186, 177, 178, 180, 184, 184, 180, 183, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967913279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207612024-02-12T01:43:27.109Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
207622024-02-12T01:43:27.109Z,"time":"2024-02-12T01:43:25.967939298Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207632024-02-12T01:43:27.109Z{"msg":"[2]R flush_numbers[0..12]: [335, 334, 318, 321, 325, 329, 329, 325, 328, 332, 334, 330]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967958617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207642024-02-12T01:43:27.109Z{"msg":"[2]R generation[0..12]: [187, 186, 177, 178, 180, 184, 184, 180, 183, 185, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.967984076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207652024-02-12T01:43:27.109Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968007824Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207662024-02-12T01:43:27.109Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968042333Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207672024-02-12T01:43:27.109Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968060712Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207682024-02-12T01:43:27.109Z{"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968080531Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207692024-02-12T01:43:27.109Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.96809385Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207702024-02-12T01:43:27.109Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968114459Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207712024-02-12T01:43:27.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968127059Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207722024-02-12T01:43:27.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968150497Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207732024-02-12T01:43:27.109Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968166887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207742024-02-12T01:43:27.110Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968181076Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207752024-02-12T01:43:27.110Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968202595Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207762024-02-12T01:43:27.110Z{"msg":"58f8082f-dd4f-4ba5-abea-0d1cdf677296 is now active with session: 5df108fb-dd1e-4760-8f3d-1bc01ad50814","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968216045Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207772024-02-12T01:43:27.110Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968234453Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887,"session_id":"5df108fb-dd1e-4760-8f3d-1bc01ad50814"}
207782024-02-12T01:43:27.110Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:25.968247293Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
207792024-02-12T01:43:27.110ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
207802024-02-12T01:43:27.110ZLoading write count information from file "/tmp/test_repair_verify.data"
207812024-02-12T01:43:27.110ZRun Repair workload
207822024-02-12T01:43:27.110Z01/30 Write block 222 len 24576 data: 14 15 15 15 11 11
207832024-02-12T01:43:27.113Z02/30 Read block 530 len 8192
207842024-02-12T01:43:27.113Z03/30 Read block 343 len 28672
207852024-02-12T01:43:27.113Z04/30 Read block 290 len 36864
207862024-02-12T01:43:27.113Z05/30 Write block 445 len 28672 data: 11 11 13 14 13 11 8
207872024-02-12T01:43:27.113Z06/30 Write block 324 len 24576 data: 13 12 13 12 11 10
207882024-02-12T01:43:27.113Z07/30 Read block 490 len 24576
207892024-02-12T01:43:27.117Z08/30 Write block 314 len 4096 data: 14
207902024-02-12T01:43:27.117Z09/30 Read block 556 len 20480
207912024-02-12T01:43:27.117Z10/30 Read block 25 len 28672
207922024-02-12T01:43:27.117Z11/30 Read block 285 len 32768
207932024-02-12T01:43:27.120Z12/30 Write block 214 len 28672 data: 11 11 15 11 14 14 12
207942024-02-12T01:43:27.120Z13/30 Read block 255 len 20480
207952024-02-12T01:43:27.120Z14/30 Read block 103 len 12288
207962024-02-12T01:43:27.123Z15/30 Write block 318 len 28672 data: 14 15 15 13 15 12 14
207972024-02-12T01:43:27.123Z16/30 Read block 532 len 36864
207982024-02-12T01:43:27.123Z17/30 Read block 298 len 40960
207992024-02-12T01:43:27.130Z18/30 Write block 72 len 28672 data: 11 10 10 11 12 12 13
208002024-02-12T01:43:27.130Z19/30 Read block 498 len 4096
208012024-02-12T01:43:27.130Z20/30 Write block 421 len 36864 data: 13 11 9 12 9 10 8 7 9
208022024-02-12T01:43:27.133Z21/30 Write block 64 len 4096 data: 17
208032024-02-12T01:43:27.133Z22/30 Read block 228 len 8192
208042024-02-12T01:43:27.133Z23/30 Write block 356 len 24576 data: 8 9 8 9 11 13
208052024-02-12T01:43:27.133Z24/30 Write block 315 len 28672 data: 16 13 13 15 16 16 14
208062024-02-12T01:43:27.133Z25/30 Write block 337 len 8192 data: 10 9
208072024-02-12T01:43:27.133Z26/30 Write block 480 len 24576 data: 17 17 18 17 17 16
208082024-02-12T01:43:27.133Z27/30 Write block 385 len 28672 data: 16 18 17 18 18 21 20
208092024-02-12T01:43:27.133Z28/30 Read block 470 len 20480
208102024-02-12T01:43:27.134Z29/30 Write block 187 len 32768 data: 18 18 18 14 13 16 15 13
208112024-02-12T01:43:27.134Z30/30 Read block 186 len 28672
208122024-02-12T01:43:27.137Z----------------------------------------------------------------
208132024-02-12T01:43:27.137Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
208142024-02-12T01:43:27.137ZStates: Active Active Active
208152024-02-12T01:43:27.137ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208162024-02-12T01:43:27.137Z 1 Acked 1000 Write 6 Done Done Sent false
208172024-02-12T01:43:27.138Z 2 Acked 1001 Read 2 Done Done Sent false
208182024-02-12T01:43:27.138Z 3 Acked 1002 Read 7 Done Done Sent false
208192024-02-12T01:43:27.138Z 4 Acked 1003 Read 9 Done Done Sent false
208202024-02-12T01:43:27.138Z 5 Acked 1004 Write 7 Done Done Sent false
208212024-02-12T01:43:27.138Z 6 Acked 1005 Write 6 Done Done Sent false
208222024-02-12T01:43:27.138Z 7 Acked 1006 Read 6 Done Done Sent false
208232024-02-12T01:43:27.138Z 8 Acked 1007 Write 1 Done Done Sent false
208242024-02-12T01:43:27.138Z 9 Acked 1008 Read 5 Done Done Sent false
208252024-02-12T01:43:27.138Z 10 Acked 1009 Read 7 Done Done Sent false
208262024-02-12T01:43:27.138Z 11 Acked 1010 Read 8 Done Done Sent false
208272024-02-12T01:43:27.138Z 12 Acked 1011 Write 7 Done Done Sent false
208282024-02-12T01:43:27.138Z 13 Acked 1012 Read 5 Done Done Sent false
208292024-02-12T01:43:27.138Z 14 Acked 1013 Read 3 Done Done Sent false
208302024-02-12T01:43:27.138Z 15 Acked 1014 Write 7 Done Done Sent false
208312024-02-12T01:43:27.138Z 16 Acked 1015 Read 9 Done Done Sent false
208322024-02-12T01:43:27.138Z 17 Acked 1016 Read 10 Done Done Sent false
208332024-02-12T01:43:27.138Z 18 Acked 1017 Write 7 Done Done Sent false
208342024-02-12T01:43:27.138Z 19 Acked 1018 Read 1 Done Done Sent false
208352024-02-12T01:43:27.138Z 20 Acked 1019 Write 9 Done Done Sent false
208362024-02-12T01:43:27.138Z 21 Acked 1020 Write 1 Done Done Sent false
208372024-02-12T01:43:27.138Z 22 Acked 1021 Read 2 Done Done Sent false
208382024-02-12T01:43:27.138Z 23 Acked 1022 Write 6 Done Done Sent false
208392024-02-12T01:43:27.138Z 24 Acked 1023 Write 7 Done Done Sent false
208402024-02-12T01:43:27.138Z 25 Acked 1024 Write 2 Done Done Sent false
208412024-02-12T01:43:27.139Z 26 Acked 1025 Write 6 Done Done Sent false
208422024-02-12T01:43:27.139Z 27 Acked 1026 Write 7 Done Done Sent false
208432024-02-12T01:43:27.139Z 28 Acked 1027 Read 5 Done Done Sent false
208442024-02-12T01:43:27.139Z 29 Acked 1028 Write 8 Done Done Sent false
208452024-02-12T01:43:27.139Z 30 Acked 1029 Read 7 Done Sent Sent false
208462024-02-12T01:43:27.139Z STATES DS:0 DS:1 DS:2 TOTAL
208472024-02-12T01:43:27.139Z New 0 0 0 0
208482024-02-12T01:43:27.139Z Sent 0 1 30 31
208492024-02-12T01:43:27.139Z Done 30 29 0 59
208502024-02-12T01:43:27.139Z Skipped 0 0 0 0
208512024-02-12T01:43:27.139Z Error 0 0 0 0
208522024-02-12T01:43:27.139ZLast Flush: 0 0 0
208532024-02-12T01:43:27.139ZDownstairs last five completed:
208542024-02-12T01:43:27.139ZUpstairs last five completed: 30 29 28 27 26
208552024-02-12T01:43:27.139Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:25.996899676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1887}
208562024-02-12T01:43:27.139ZWrote out file "/tmp/test_repair_verify.data"
208572024-02-12T01:43:27.142Z
208582024-02-12T01:43:27.145ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
208592024-02-12T01:43:27.165ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208602024-02-12T01:43:27.165Z 0 000-019 187 187 187 335 335 335 F F F
208612024-02-12T01:43:27.165Z 1 020-039 186 186 186 334 334 334 F F F
208622024-02-12T01:43:27.165Z 2 040-059 177 177 177 318 318 318 F F F
208632024-02-12T01:43:27.165Z 3 060-079 178 178 178 321 321 321 T T F <---
208642024-02-12T01:43:27.165Z 4 080-099 180 180 180 325 325 325 F F F
208652024-02-12T01:43:27.165Z 5 100-119 184 184 184 329 329 329 F F F
208662024-02-12T01:43:27.165Z 6 120-139 184 184 184 329 329 329 F F F
208672024-02-12T01:43:27.165Z 7 140-159 180 180 180 325 325 325 F F F
208682024-02-12T01:43:27.165Z 8 160-179 183 183 183 328 328 328 F F F
208692024-02-12T01:43:27.165Z 9 180-199 185 185 185 332 332 332 T T F <---
208702024-02-12T01:43:27.168Z 10 200-219 186 186 186 334 334 334 T T F <---
208712024-02-12T01:43:27.169Z 11 220-239 184 184 184 330 330 330 T T F <---
208722024-02-12T01:43:27.169Z 12 240-259 186 186 186 334 334 334 F F F
208732024-02-12T01:43:27.169Z 13 260-279 186 186 186 333 333 333 F F F
208742024-02-12T01:43:27.169Z 14 280-299 187 187 187 335 335 335 F F F
208752024-02-12T01:43:27.169Z 15 300-319 187 187 187 335 335 335 T T F <---
208762024-02-12T01:43:27.169Z 16 320-339 184 184 184 331 331 331 T T F <---
208772024-02-12T01:43:27.169Z 17 340-359 185 185 185 332 332 332 T T F <---
208782024-02-12T01:43:27.169Z 18 360-379 180 180 180 325 325 325 T T F <---
208792024-02-12T01:43:27.169Z 19 380-399 187 187 187 335 335 335 T T F <---
208802024-02-12T01:43:27.169Z 20 400-419 187 187 187 335 335 335 F F F
208812024-02-12T01:43:27.169Z 21 420-439 178 178 178 322 322 322 T T F <---
208822024-02-12T01:43:27.169Z 22 440-459 185 185 185 332 332 332 T T F <---
208832024-02-12T01:43:27.169Z 23 460-479 186 186 186 333 333 333 F F F
208842024-02-12T01:43:27.169Z 24 480-499 180 180 180 325 325 325 T T F <---
208852024-02-12T01:43:27.169Z 25 500-519 185 185 185 332 332 332 F F F
208862024-02-12T01:43:27.169Z 26 520-539 186 186 186 334 334 334 F F F
208872024-02-12T01:43:27.169Z 27 540-559 187 187 187 335 335 335 F F F
208882024-02-12T01:43:27.169Z 28 560-579 187 187 187 335 335 335 F F F
208892024-02-12T01:43:27.169Z 29 580-599 187 187 187 335 335 335 F F F
208902024-02-12T01:43:27.169ZMax gen: 187, Max flush: 335
208912024-02-12T01:43:27.169ZError: Difference in extent metadata found!
208922024-02-12T01:43:27.169ZA Difference in extent metadata is expected here
208932024-02-12T01:43:27.169ZOn loop 94
208942024-02-12T01:43:27.169Z
208952024-02-12T01:43:27.170ZVerifying data now
208962024-02-12T01:43:31.797ZNo change after verify
208972024-02-12T01:43:31.797ZLoop: 94 Downstairs dump after verify (and repair):
208982024-02-12T01:43:31.812ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
208992024-02-12T01:43:31.812Z 0 000-019 187 187 187 335 335 335 F F F
209002024-02-12T01:43:31.812Z 1 020-039 186 186 186 334 334 334 F F F
209012024-02-12T01:43:31.812Z 2 040-059 177 177 177 318 318 318 F F F
209022024-02-12T01:43:31.813Z 3 060-079 188 188 188 336 336 336 F F F
209032024-02-12T01:43:31.813Z 4 080-099 180 180 180 325 325 325 F F F
209042024-02-12T01:43:31.813Z 5 100-119 184 184 184 329 329 329 F F F
209052024-02-12T01:43:31.813Z 6 120-139 184 184 184 329 329 329 F F F
209062024-02-12T01:43:31.813Z 7 140-159 180 180 180 325 325 325 F F F
209072024-02-12T01:43:31.816Z 8 160-179 183 183 183 328 328 328 F F F
209082024-02-12T01:43:31.816Z 9 180-199 188 188 188 336 336 336 F F F
209092024-02-12T01:43:31.816Z 10 200-219 188 188 188 336 336 336 F F F
209102024-02-12T01:43:31.816Z 11 220-239 188 188 188 336 336 336 F F F
209112024-02-12T01:43:31.816Z 12 240-259 186 186 186 334 334 334 F F F
209122024-02-12T01:43:31.816Z 13 260-279 186 186 186 333 333 333 F F F
209132024-02-12T01:43:31.816Z 14 280-299 187 187 187 335 335 335 F F F
209142024-02-12T01:43:31.816Z 15 300-319 188 188 188 336 336 336 F F F
209152024-02-12T01:43:31.816Z 16 320-339 188 188 188 336 336 336 F F F
209162024-02-12T01:43:31.816Z 17 340-359 188 188 188 336 336 336 F F F
209172024-02-12T01:43:31.816Z 18 360-379 188 188 188 336 336 336 F F F
209182024-02-12T01:43:31.816Z 19 380-399 188 188 188 336 336 336 F F F
209192024-02-12T01:43:31.816Z 20 400-419 187 187 187 335 335 335 F F F
209202024-02-12T01:43:31.816Z 21 420-439 188 188 188 336 336 336 F F F
209212024-02-12T01:43:31.816Z 22 440-459 188 188 188 336 336 336 F F F
209222024-02-12T01:43:31.816Z 23 460-479 186 186 186 333 333 333 F F F
209232024-02-12T01:43:31.816Z 24 480-499 188 188 188 336 336 336 F F F
209242024-02-12T01:43:31.817Z 25 500-519 185 185 185 332 332 332 F F F
209252024-02-12T01:43:31.817Z 26 520-539 186 186 186 334 334 334 F F F
209262024-02-12T01:43:31.817Z 27 540-559 187 187 187 335 335 335 F F F
209272024-02-12T01:43:31.817Z 28 560-579 187 187 187 335 335 335 F F F
209282024-02-12T01:43:31.817Z 29 580-599 187 187 187 335 335 335 F F F
209292024-02-12T01:43:31.817ZMax gen: 188, Max flush: 336
209302024-02-12T01:43:31.820Z
209312024-02-12T01:43:31.820ZBegin loop 95
209322024-02-12T01:43:31.820ZDownstairs to restart: 2
209332024-02-12T01:43:31.838Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.6997077Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
209342024-02-12T01:43:31.838Z{"msg":"Crucible Version: BuildInfo {\nCrucible runtime is spawned
209352024-02-12T01:43:31.843Z version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700020805Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
209362024-02-12T01:43:31.843Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700068562Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
209372024-02-12T01:43:31.843Z{"msg":"Crucible d38568be-0f69-4707-bcab-cb6a04b30d09 has session id: c651ed4c-ce94-4ec4-ad32-2011e29ea38e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.7001109Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209382024-02-12T01:43:31.843Z{"msg":"Upstairs opts: Upstairs UUID: d38568be-0f69-4707-bcab-cb6a04b30d09, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700166048Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209392024-02-12T01:43:31.843Z{"msg":"Crucible stats registered with UUID: d38568be-0f69-4707-bcab-cb6a04b30d09","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700210016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209402024-02-12T01:43:31.843Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700268153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
209412024-02-12T01:43:31.843Z{"msg":"d38568be-0f69-4707-bcab-cb6a04b30d09 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700309081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209422024-02-12T01:43:31.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700350839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209432024-02-12T01:43:31.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700397017Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209442024-02-12T01:43:31.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700439725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209452024-02-12T01:43:31.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700471713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209462024-02-12T01:43:31.843Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700502042Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209472024-02-12T01:43:31.843Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.70054394Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209482024-02-12T01:43:31.844Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700585678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209492024-02-12T01:43:31.844Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700666164Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209502024-02-12T01:43:31.844Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700729801Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209512024-02-12T01:43:31.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700772699Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209522024-02-12T01:43:31.844Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700812447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209532024-02-12T01:43:31.844Z{"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:30.700850505Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209542024-02-12T01:43:31.844Z{"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:30.700897633Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209552024-02-12T01:43:31.844Z{"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:30.700929572Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209562024-02-12T01:43:31.844Z{"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.70096397Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209572024-02-12T01:43:31.844Z{"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.700990198Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209582024-02-12T01:43:31.844Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.701103563Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209592024-02-12T01:43:31.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.701544292Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209602024-02-12T01:43:31.844Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.701572281Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209612024-02-12T01:43:31.844Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.701587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209622024-02-12T01:43:31.844Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.701608639Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209632024-02-12T01:43:31.844Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702107075Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209642024-02-12T01:43:31.844Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702230119Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209652024-02-12T01:43:31.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702604132Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209662024-02-12T01:43:31.844Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702653489Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209672024-02-12T01:43:31.844Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702681678Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209682024-02-12T01:43:31.844Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702716906Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209692024-02-12T01:43:31.844Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702732406Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209702024-02-12T01:43:31.844Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:30.702753485Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209712024-02-12T01:43:32.841Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.702690399Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209722024-02-12T01:43:32.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.70287818Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209732024-02-12T01:43:32.846Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.703733769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209742024-02-12T01:43:32.846Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.703776447Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209752024-02-12T01:43:32.846Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.704275193Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209762024-02-12T01:43:32.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.704849606Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209772024-02-12T01:43:32.846Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.704906203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209782024-02-12T01:43:32.846Z{"msg":"[0]R flush_numbers[0..12]: [335, 334, 318, 336, 325, 329, 329, 325, 328, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.704946191Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209792024-02-12T01:43:32.846Z{"msg":"[0]R generation[0..12]: [187, 186, 177, 188, 180, 184, 184, 180, 183, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.70497379Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209802024-02-12T01:43:32.846Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.704993279Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209812024-02-12T01:43:32.846Z{"msg":"[1]R flush_numbers[0..12]: [335, 334, 318, 336, 325, 329, 329, 325, 328, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705031967Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209822024-02-12T01:43:32.846Z{"msg":"[1]R generation[0..12]: [187, 186, 177, 188, 180, 184, 184, 180, 183, 188, 188, 188]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
209832024-02-12T01:43:32.846Z,"time":"2024-02-12T01:43:31.705059016Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209842024-02-12T01:43:32.847Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705085834Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209852024-02-12T01:43:32.847Z{"msg":"[2]R flush_numbers[0..12]: [335, 334, 318, 336, 325, 329, 329, 325, 328, 336, 336, 336]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705122153Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209862024-02-12T01:43:32.847Z{"msg":"[2]R generation[0..12]: [187, 186, 177, 188, 180, 184, 184, 180, 183, 188, 188, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705140622Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209872024-02-12T01:43:32.847Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705166121Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209882024-02-12T01:43:32.847Z{"msg":"Max found gen is 189","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.70518284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209892024-02-12T01:43:32.847Z{"msg":"Generation requested: 190 >= found:189","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705208079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209902024-02-12T01:43:32.847Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705232817Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209912024-02-12T01:43:32.847Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705258716Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209922024-02-12T01:43:32.847Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705275665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209932024-02-12T01:43:32.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705298754Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209942024-02-12T01:43:32.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705315423Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209952024-02-12T01:43:32.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705340162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209962024-02-12T01:43:32.847Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705356632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209972024-02-12T01:43:32.847Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
209982024-02-12T01:43:32.847Z,"time":"2024-02-12T01:43:31.70537973Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
209992024-02-12T01:43:32.847Z{"msg":"d38568be-0f69-4707-bcab-cb6a04b30d09 is now active with session: c651ed4c-ce94-4ec4-ad32-2011e29ea38e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.70539617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
210002024-02-12T01:43:32.847Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705411689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895,"session_id":"c651ed4c-ce94-4ec4-ad32-2011e29ea38e"}
210012024-02-12T01:43:32.847Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:31.705428158Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
210022024-02-12T01:43:32.847ZLoading write count information from file "/tmp/test_repair_verify.data"
210032024-02-12T01:43:32.847ZRun Repair workload
210042024-02-12T01:43:32.847Z01/30 Write block 20 len 24576 data: 9 11 13 14 13 16
210052024-02-12T01:43:32.851Z02/30 Read block 19 len 16384
210062024-02-12T01:43:32.851Z03/30 Flush
210072024-02-12T01:43:32.851Z04/30 Write block 549 len 16384 data: 16 19 18 14
210082024-02-12T01:43:32.851Z05/30 Flush
210092024-02-12T01:43:32.853Z06/30 Write block 549 len 28672 data: 17 20 19 15 13 13 12
210102024-02-12T01:43:32.854Z07/30 Read block 16 len 36864
210112024-02-12T01:43:32.856Z08/30 Flush
210122024-02-12T01:43:32.856Z09/30 Write block 223 len 40960 data: 16 16 16 12 12 11 11 9 11 11
210132024-02-12T01:43:32.859Z10/30 Read block 487 len 40960
210142024-02-12T01:43:32.860Z11/30 Write block 576 len 24576 data: 15 15 12 12 13 13
210152024-02-12T01:43:32.860Z12/30 Read block 486 len 20480
210162024-02-12T01:43:32.865Z13/30 Read block 285 len 12288
210172024-02-12T01:43:32.865Z14/30 Read block 501 len 24576
210182024-02-12T01:43:32.865Z15/30 Read block 27 len 36864
210192024-02-12T01:43:32.868Z16/30 Write block 246 len 24576 data: 10 11 10 9 9 12
210202024-02-12T01:43:32.868Z17/30 Read block 351 len 4096
210212024-02-12T01:43:32.868Z18/30 Write block 234 len 12288 data: 15 13 16
210222024-02-12T01:43:32.869Z19/30 Read block 430 len 40960
210232024-02-12T01:43:32.869Z20/30 Write block 200 len 32768 data: 15 14 17 19 16 16 15 15
210242024-02-12T01:43:32.869Z21/30 Flush
210252024-02-12T01:43:32.871Z22/30 Write block 422 len 20480 data: 12 10 13 10 11
210262024-02-12T01:43:32.871Z23/30 Write block 164 len 24576 data: 14 15 14 14 15 20
210272024-02-12T01:43:32.874Z24/30 Write block 445 len 28672 data: 12 12 14 15 14 12 9
210282024-02-12T01:43:32.874Z25/30 Read block 232 len 40960
210292024-02-12T01:43:32.874Z26/30 Read block 380 len 36864
210302024-02-12T01:43:32.877Z27/30 Read block 267 len 20480
210312024-02-12T01:43:32.877Z28/30 Write block 587 len 4096 data: 12
210322024-02-12T01:43:32.877Z29/30 Read block 249 len 36864
210332024-02-12T01:43:32.877Z30/30 Write block 448 len 16384 data: 16 15 13 10
210342024-02-12T01:43:32.877Z----------------------------------------------------------------
210352024-02-12T01:43:32.878Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:25
210362024-02-12T01:43:32.878ZStates: Active Active Active
210372024-02-12T01:43:32.878ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
210382024-02-12T01:43:32.878Z 6 Acked 1005 Write 7 Done Done Sent false
210392024-02-12T01:43:32.878Z 7 Acked 1006 Read 9 Done Done Sent false
210402024-02-12T01:43:32.878Z 8 Acked 1007 Flush 0 Done Done Sent false
210412024-02-12T01:43:32.878Z 9 Acked 1008 Write 10 Done Done Sent false
210422024-02-12T01:43:32.878Z 10 Acked 1009 Read 10 Done Done Sent false
210432024-02-12T01:43:32.878Z 11 Acked 1010 Write 6 Done Done Sent false
210442024-02-12T01:43:32.878Z 12 Acked 1011 Read 5 Done Done Sent false
210452024-02-12T01:43:32.878Z 13 Acked 1012 Read 3 Done Done Sent false
210462024-02-12T01:43:32.878Z 14 Acked 1013 Read 6 Done Done Sent false
210472024-02-12T01:43:32.878Z 15 Acked 1014 Read 9 Done Done Sent false
210482024-02-12T01:43:32.878Z 16 Acked 1015 Write 6 Done Done Sent false
210492024-02-12T01:43:32.878Z 17 Acked 1016 Read 1 Done Done Sent false
210502024-02-12T01:43:32.878Z 18 Acked 1017 Write 3 Done Done Sent false
210512024-02-12T01:43:32.882Z 19 Acked 1018 Read 10 Done Done Sent false
210522024-02-12T01:43:32.882Z 20 Acked 1019 Write 8 Done Done Sent false
210532024-02-12T01:43:32.882Z 21 Acked 1020 Flush 0 Done Done Sent false
210542024-02-12T01:43:32.882Z 22 Acked 1021 Write 5 Done Done Sent false
210552024-02-12T01:43:32.882Z 23 Acked 1022 Write 6 Done Done Sent false
210562024-02-12T01:43:32.882Z 24 Acked 1023 Write 7 Done Done Sent false
210572024-02-12T01:43:32.882Z 25 Acked 1024 Read 10 Done Done Sent false
210582024-02-12T01:43:32.882Z 26 Acked 1025 Read 9 Done Done Sent false
210592024-02-12T01:43:32.882Z 27 Acked 1026 Read 5 Done Done Sent false
210602024-02-12T01:43:32.882Z 28 Acked 1027 Write 1 Done Done Sent false
210612024-02-12T01:43:32.882Z 29 Acked 1028 Read 9 Done Done Sent false
210622024-02-12T01:43:32.882Z 30 Acked 1029 Write 4 Sent Sent Sent false
210632024-02-12T01:43:32.883Z STATES DS:0 DS:1 DS:2 TOTAL
210642024-02-12T01:43:32.883Z New 0 0 0 0
210652024-02-12T01:43:32.883Z Sent 1 1 25 27
210662024-02-12T01:43:32.883Z Done 24 24 0 48
210672024-02-12T01:43:32.883Z Skipped 0 0 0 0
210682024-02-12T01:43:32.883Z Error 0 0 0 0
210692024-02-12T01:43:32.883ZLast Flush: 1020 1020 1004
210702024-02-12T01:43:32.883ZDownstairs last five completed: 1004 1003 1002 1001 1000
210712024-02-12T01:43:32.883ZUpstairs last five completed: 30 29 28 27 26
210722024-02-12T01:43:32.883Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:31.739532551Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1895}
210732024-02-12T01:43:32.883ZWrote out file "/tmp/test_repair_verify.data"
210742024-02-12T01:43:32.886Z
210752024-02-12T01:43:32.886ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
210762024-02-12T01:43:32.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210772024-02-12T01:43:32.902Z 0 000-019 187 187 187 335 335 335 F F F
210782024-02-12T01:43:32.903Z 1 020-039 190 190 190 337 337 337 F F F
210792024-02-12T01:43:32.903Z 2 040-059 177 177 177 318 318 318 F F F
210802024-02-12T01:43:32.903Z 3 060-079 188 188 188 336 336 336 F F F
210812024-02-12T01:43:32.903Z 4 080-099 180 180 180 325 325 325 F F F
210822024-02-12T01:43:32.903Z 5 100-119 184 184 184 329 329 329 F F F
210832024-02-12T01:43:32.903Z 6 120-139 184 184 184 329 329 329 F F F
210842024-02-12T01:43:32.903Z 7 140-159 180 180 180 325 325 325 F F F
210852024-02-12T01:43:32.903Z 8 160-179 183 183 183 328 328 328 T T F <---
210862024-02-12T01:43:32.903Z 9 180-199 188 188 188 336 336 336 F F F
210872024-02-12T01:43:32.903Z 10 200-219 190 190 188 340 340 336 F F F <---
210882024-02-12T01:43:32.906Z 11 220-239 190 190 188 340 340 336 F F F <---
210892024-02-12T01:43:32.906Z 12 240-259 190 190 186 340 340 334 F F F <---
210902024-02-12T01:43:32.906Z 13 260-279 186 186 186 333 333 333 F F F
210912024-02-12T01:43:32.906Z 14 280-299 187 187 187 335 335 335 F F F
210922024-02-12T01:43:32.906Z 15 300-319 188 188 188 336 336 336 F F F
210932024-02-12T01:43:32.906Z 16 320-339 188 188 188 336 336 336 F F F
210942024-02-12T01:43:32.906Z 17 340-359 188 188 188 336 336 336 F F F
210952024-02-12T01:43:32.906Z 18 360-379 188 188 188 336 336 336 F F F
210962024-02-12T01:43:32.907Z 19 380-399 188 188 188 336 336 336 F F F
210972024-02-12T01:43:32.907Z 20 400-419 187 187 187 335 335 335 F F F
210982024-02-12T01:43:32.907Z 21 420-439 188 188 188 336 336 336 T T F <---
210992024-02-12T01:43:32.907Z 22 440-459 188 188 188 336 336 336 T T F <---
211002024-02-12T01:43:32.907Z 23 460-479 186 186 186 333 333 333 F F F
211012024-02-12T01:43:32.907Z 24 480-499 188 188 188 336 336 336 F F F
211022024-02-12T01:43:32.907Z 25 500-519 185 185 185 332 332 332 F F F
211032024-02-12T01:43:32.907Z 26 520-539 186 186 186 334 334 334 F F F
211042024-02-12T01:43:32.907Z 27 540-559 190 190 190 339 339 338 F F F <---
211052024-02-12T01:43:32.907Z 28 560-579 190 190 187 340 340 335 F F F <---
211062024-02-12T01:43:32.907Z 29 580-599 190 190 187 340 340 335 T T F <---
211072024-02-12T01:43:32.907ZMax gen: 190, Max flush: 340
211082024-02-12T01:43:32.907ZError: Difference in extent metadata found!
211092024-02-12T01:43:32.907ZA Difference in extent metadata is expected here
211102024-02-12T01:43:32.907ZOn loop 95
211112024-02-12T01:43:32.907Z
211122024-02-12T01:43:32.910ZVerifying data now
211132024-02-12T01:43:37.398ZNo change after verify
211142024-02-12T01:43:37.398ZLoop: 95 Downstairs dump after verify (and repair):
211152024-02-12T01:43:37.413ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211162024-02-12T01:43:37.413Z 0 000-019 187 187 187 335 335 335 F F F
211172024-02-12T01:43:37.413Z 1 020-039 190 190 190 337 337 337 F F F
211182024-02-12T01:43:37.413Z 2 040-059 177 177 177 318 318 318 F F F
211192024-02-12T01:43:37.413Z 3 060-079 188 188 188 336 336 336 F F F
211202024-02-12T01:43:37.413Z 4 080-099 180 180 180 325 325 325 F F F
211212024-02-12T01:43:37.413Z 5 100-119 184 184 184 329 329 329 F F F
211222024-02-12T01:43:37.413Z 6 120-139 184 184 184 329 329 329 F F F
211232024-02-12T01:43:37.413Z 7 140-159 180 180 180 325 325 325 F F F
211242024-02-12T01:43:37.413Z 8 160-179 191 191 191 341 341 341 F F F
211252024-02-12T01:43:37.417Z 9 180-199 188 188 188 336 336 336 F F F
211262024-02-12T01:43:37.417Z 10 200-219 190 190 190 340 340 340 F F F
211272024-02-12T01:43:37.417Z 11 220-239 190 190 190 340 340 340 F F F
211282024-02-12T01:43:37.417Z 12 240-259 190 190 190 340 340 340 F F F
211292024-02-12T01:43:37.417Z 13 260-279 186 186 186 333 333 333 F F F
211302024-02-12T01:43:37.418Z 14 280-299 187 187 187 335 335 335 F F F
211312024-02-12T01:43:37.418Z 15 300-319 188 188 188 336 336 336 F F F
211322024-02-12T01:43:37.418Z 16 320-339 188 188 188 336 336 336 F F F
211332024-02-12T01:43:37.418Z 17 340-359 188 188 188 336 336 336 F F F
211342024-02-12T01:43:37.418Z 18 360-379 188 188 188 336 336 336 F F F
211352024-02-12T01:43:37.418Z 19 380-399 188 188 188 336 336 336 F F F
211362024-02-12T01:43:37.418Z 20 400-419 187 187 187 335 335 335 F F F
211372024-02-12T01:43:37.418Z 21 420-439 191 191 191 341 341 341 F F F
211382024-02-12T01:43:37.418Z 22 440-459 191 191 191 341 341 341 F F F
211392024-02-12T01:43:37.418Z 23 460-479 186 186 186 333 333 333 F F F
211402024-02-12T01:43:37.418Z 24 480-499 188 188 188 336 336 336 F F F
211412024-02-12T01:43:37.418Z 25 500-519 185 185 185 332 332 332 F F F
211422024-02-12T01:43:37.418Z 26 520-539 186 186 186 334 334 334 F F F
211432024-02-12T01:43:37.419Z 27 540-559 190 190 190 339 339 339 F F F
211442024-02-12T01:43:37.419Z 28 560-579 190 190 190 340 340 340 F F F
211452024-02-12T01:43:37.419Z 29 580-599 191 191 191 341 341 341 F F F
211462024-02-12T01:43:37.419ZMax gen: 191, Max flush: 341
211472024-02-12T01:43:37.419Z
211482024-02-12T01:43:37.419ZBegin loop 96
211492024-02-12T01:43:37.419ZDownstairs to restart: 2
211502024-02-12T01:43:37.433Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.293928685Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
211512024-02-12T01:43:37.433Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294166744Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
211522024-02-12T01:43:37.437Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294202312Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
211532024-02-12T01:43:37.437ZCrucible runtime is spawned
211542024-02-12T01:43:37.437Z{"msg":"Crucible 76b4cbd5-5a41-416f-820e-ba168a93c39e has session id: 6d186778-e365-4ad9-b217-39bd3861efef","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29425356Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211552024-02-12T01:43:37.437Z{"msg":"Upstairs opts: Upstairs UUID: 76b4cbd5-5a41-416f-820e-ba168a93c39e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294287928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211562024-02-12T01:43:37.437Z{"msg":"Crucible stats registered with UUID: 76b4cbd5-5a41-416f-820e-ba168a93c39e","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294309147Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211572024-02-12T01:43:37.437Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294333156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
211582024-02-12T01:43:37.437Z{"msg":"76b4cbd5-5a41-416f-820e-ba168a93c39e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29446268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211592024-02-12T01:43:37.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294490979Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211602024-02-12T01:43:37.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294540626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211612024-02-12T01:43:37.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294584534Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211622024-02-12T01:43:37.438Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294615203Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211632024-02-12T01:43:37.438Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294651771Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211642024-02-12T01:43:37.438Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294711418Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211652024-02-12T01:43:37.438Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294759676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211662024-02-12T01:43:37.438Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294808474Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211672024-02-12T01:43:37.438Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294848242Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211682024-02-12T01:43:37.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29487628Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211692024-02-12T01:43:37.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294913638Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211702024-02-12T01:43:37.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.294965326Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211712024-02-12T01:43:37.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.295563268Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211722024-02-12T01:43:37.438Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.295612245Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211732024-02-12T01:43:37.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.295679202Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211742024-02-12T01:43:37.438Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29572516Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211752024-02-12T01:43:37.438Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296190188Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211762024-02-12T01:43:37.438Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296330881Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211772024-02-12T01:43:37.438Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296691124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211782024-02-12T01:43:37.438Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296788689Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211792024-02-12T01:43:37.439Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296820448Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211802024-02-12T01:43:37.439Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296855666Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211812024-02-12T01:43:37.439Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296872135Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211822024-02-12T01:43:37.439Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296887044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211832024-02-12T01:43:37.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296899984Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211842024-02-12T01:43:37.439Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.296914073Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211852024-02-12T01:43:37.443Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.297464137Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211862024-02-12T01:43:37.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.297886587Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211872024-02-12T01:43:37.443Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.297919925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211882024-02-12T01:43:37.443Z{"msg":"[0]R flush_numbers[0..12]: [335, 337, 318, 336, 325, 329, 329, 325, 341, 336, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.297946414Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211892024-02-12T01:43:37.443Z{"msg":"[0]R generation[0..12]: [187, 190, 177, 188, 180, 184, 184, 180, 191, 188, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.297975632Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211902024-02-12T01:43:37.443Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298001641Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211912024-02-12T01:43:37.443Z{"msg":"[1]R flush_numbers[0..12]: [335, 337, 318, 336, 325, 329, 329, 325, 341, 336, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29802674Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211922024-02-12T01:43:37.443Z{"msg":"[1]R generation[0..12]: [187, 190, 177, 188, 180, 184, 184, 180, 191, 188, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298041449Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211932024-02-12T01:43:37.443Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298056219Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211942024-02-12T01:43:37.443Z{"msg":"[2]R flush_numbers[0..12]: [335, 337, 318, 336, 325, 329, 329, 325, 341, 336, 340, 340]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298070368Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211952024-02-12T01:43:37.443ZWait for a query_work_queue command to finish before sending IO
211962024-02-12T01:43:37.444Z{"msg":"[2]R generation[0..12]: [187, 190, 177, 188, 180, 184, 184, 180, 191, 188, 190, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298092887Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211972024-02-12T01:43:37.444Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298107816Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211982024-02-12T01:43:37.444Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298127565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
211992024-02-12T01:43:37.444Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298141195Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212002024-02-12T01:43:37.444Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298160794Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212012024-02-12T01:43:37.444Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298174753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212022024-02-12T01:43:37.444Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298194872Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212032024-02-12T01:43:37.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298208201Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212042024-02-12T01:43:37.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.29824331Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212052024-02-12T01:43:37.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298258199Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212062024-02-12T01:43:37.444Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298278598Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212072024-02-12T01:43:37.444Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298291328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212082024-02-12T01:43:37.444Z{"msg":"76b4cbd5-5a41-416f-820e-ba168a93c39e is now active with session: 6d186778-e365-4ad9-b217-39bd3861efef","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298310876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212092024-02-12T01:43:37.444Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298332755Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903,"session_id":"6d186778-e365-4ad9-b217-39bd3861efef"}
212102024-02-12T01:43:37.444Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:36.298355304Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
212112024-02-12T01:43:37.444ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212122024-02-12T01:43:37.444ZLoading write count information from file "/tmp/test_repair_verify.data"
212132024-02-12T01:43:37.444ZRun Repair workload
212142024-02-12T01:43:37.444Z01/30 Write block 533 len 12288 data: 20 19 20
212152024-02-12T01:43:37.444Z02/30 Write block 586 len 24576 data: 15 13 12 13 17 14
212162024-02-12T01:43:37.444Z03/30 Flush
212172024-02-12T01:43:37.448Z04/30 Write block 364 len 8192 data: 19 19
212182024-02-12T01:43:37.448Z05/30 Write block 392 len 12288 data: 20 20 19
212192024-02-12T01:43:37.448Z06/30 Read block 74 len 36864
212202024-02-12T01:43:37.448Z07/30 Read block 127 len 20480
212212024-02-12T01:43:37.448Z08/30 Write block 190 len 28672 data: 15 14 17 16 14 12 14
212222024-02-12T01:43:37.448Z09/30 Read block 409 len 20480
212232024-02-12T01:43:37.451Z10/30 Write block 162 len 36864 data: 17 15 15 16 15 15 16 21 20
212242024-02-12T01:43:37.451Z11/30 Read block 533 len 20480
212252024-02-12T01:43:37.451Z12/30 Write block 542 len 8192 data: 15 12
212262024-02-12T01:43:37.454Z13/30 Write block 71 len 28672 data: 16 12 11 11 12 13 13
212272024-02-12T01:43:37.454Z14/30 Read block 534 len 16384
212282024-02-12T01:43:37.454Z15/30 Read block 170 len 36864
212292024-02-12T01:43:37.455Z16/30 Read block 77 len 20480
212302024-02-12T01:43:37.458Z17/30 Flush
212312024-02-12T01:43:37.458Z18/30 Write block 79 len 36864 data: 16 18 18 22 19 20 20 16 19
212322024-02-12T01:43:37.461Z19/30 Read block 325 len 4096
212332024-02-12T01:43:37.461Z20/30 Write block 140 len 28672 data: 21 23 22 20 20 19 13
212342024-02-12T01:43:37.462Z21/30 Read block 532 len 28672
212352024-02-12T01:43:37.462Z22/30 Read block 201 len 32768
212362024-02-12T01:43:37.465Z23/30 Write block 542 len 4096 data: 16
212372024-02-12T01:43:37.465Z24/30 Write block 487 len 28672 data: 18 19 16 18 14 14 15
212382024-02-12T01:43:37.465Z25/30 Write block 392 len 36864 data: 21 21 20 14 13 13 14 12 15
212392024-02-12T01:43:37.465Z26/30 Write block 343 len 20480 data: 13 11 11 12 11
212402024-02-12T01:43:37.465Z27/30 Write block 271 len 16384 data: 21 21 20 21
212412024-02-12T01:43:37.469Z28/30 Write block 575 len 20480 data: 18 16 16 13 13
212422024-02-12T01:43:37.469Z29/30 Write block 78 len 8192 data: 14 17
212432024-02-12T01:43:37.470Z30/30 Write block 239 len 24576 data: 18 15 13 15 18 15
212442024-02-12T01:43:37.470Z----------------------------------------------------------------
212452024-02-12T01:43:37.470Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:27
212462024-02-12T01:43:37.470ZStates: Active Active Active
212472024-02-12T01:43:37.470ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
212482024-02-12T01:43:37.470Z 4 Acked 1003 Write 2 Done Done Done false
212492024-02-12T01:43:37.470Z 5 Acked 1004 Write 3 Done Done Done false
212502024-02-12T01:43:37.470Z 6 Acked 1005 Read 9 Done Done Done false
212512024-02-12T01:43:37.470Z 7 Acked 1006 Read 5 Done Done Done false
212522024-02-12T01:43:37.470Z 8 Acked 1007 Write 7 Done Done Done false
212532024-02-12T01:43:37.470Z 9 Acked 1008 Read 5 Done Done Done false
212542024-02-12T01:43:37.470Z 10 Acked 1009 Write 9 Done Done Done false
212552024-02-12T01:43:37.471Z 11 Acked 1010 Read 5 Done Done Done false
212562024-02-12T01:43:37.471Z 12 Acked 1011 Write 2 Done Done Sent false
212572024-02-12T01:43:37.471Z 13 Acked 1012 Write 7 Done Done Sent false
212582024-02-12T01:43:37.471Z 14 Acked 1013 Read 4 Done Done Sent false
212592024-02-12T01:43:37.471Z 15 Acked 1014 Read 9 Done Done Sent false
212602024-02-12T01:43:37.471Z 16 Acked 1015 Read 5 Done Done Sent false
212612024-02-12T01:43:37.471Z 17 Acked 1016 Flush 0 Done Done Sent false
212622024-02-12T01:43:37.471Z 18 Acked 1017 Write 9 Done Done Sent false
212632024-02-12T01:43:37.471Z 19 Acked 1018 Read 1 Done Done Sent false
212642024-02-12T01:43:37.471Z 20 Acked 1019 Write 7 Done Done Sent false
212652024-02-12T01:43:37.471Z 21 Acked 1020 Read 7 Done Done Sent false
212662024-02-12T01:43:37.471Z 22 Acked 1021 Read 8 Done Done Sent false
212672024-02-12T01:43:37.471Z 23 Acked 1022 Write 1 Done Done Sent false
212682024-02-12T01:43:37.472Z 24 Acked 1023 Write 7 Done Done Sent false
212692024-02-12T01:43:37.472Z 25 Acked 1024 Write 9 Done Done Sent false
212702024-02-12T01:43:37.472Z 26 Acked 1025 Write 5 Done Done Sent false
212712024-02-12T01:43:37.472Z 27 Acked 1026 Write 4 Done Done Sent false
212722024-02-12T01:43:37.472Z 28 Acked 1027 Write 5 Done Done Sent false
212732024-02-12T01:43:37.472Z 29 Acked 1028 Write 2 Sent Sent Sent false
212742024-02-12T01:43:37.472Z 30 Acked 1029 Write 6 Sent Sent Sent false
212752024-02-12T01:43:37.472Z STATES DS:0 DS:1 DS:2 TOTAL
212762024-02-12T01:43:37.472Z New 0 0 0 0
212772024-02-12T01:43:37.472Z Sent 2 2 19 23
212782024-02-12T01:43:37.472Z Done 25 25 8 58
212792024-02-12T01:43:37.472Z Skipped 0 0 0 0
212802024-02-12T01:43:37.472Z Error 0 0 0 0
212812024-02-12T01:43:37.473ZLast Flush: 1016 1016 1002
212822024-02-12T01:43:37.473ZDownstairs last five completed: 1002 1001 1000
212832024-02-12T01:43:37.473ZUpstairs last five completed: 30 29 28 27 26
212842024-02-12T01:43:37.473Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:36.328988623Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1903}
212852024-02-12T01:43:37.473ZWrote out file "/tmp/test_repair_verify.data"
212862024-02-12T01:43:37.476Z
212872024-02-12T01:43:37.476ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
212882024-02-12T01:43:37.506ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212892024-02-12T01:43:37.506Z 0 000-019 187 187 187 335 335 335 F F F
212902024-02-12T01:43:37.506Z 1 020-039 190 190 190 337 337 337 F F F
212912024-02-12T01:43:37.506Z 2 040-059 177 177 177 318 318 318 F F F
212922024-02-12T01:43:37.506Z 3 060-079 192 192 188 343 343 336 T T F <---
212932024-02-12T01:43:37.506Z 4 080-099 180 180 180 325 325 325 T T F <---
212942024-02-12T01:43:37.506Z 5 100-119 184 184 184 329 329 329 F F F
212952024-02-12T01:43:37.506Z 6 120-139 184 184 184 329 329 329 F F F
212962024-02-12T01:43:37.507Z 7 140-159 180 180 180 325 325 325 T T F <---
212972024-02-12T01:43:37.510Z 8 160-179 192 192 191 343 343 341 F F T <---
212982024-02-12T01:43:37.510Z 9 180-199 192 192 188 343 343 336 F F T <---
212992024-02-12T01:43:37.511Z 10 200-219 190 190 190 340 340 340 F F F
213002024-02-12T01:43:37.511Z 11 220-239 190 190 190 340 340 340 T T F <---
213012024-02-12T01:43:37.511Z 12 240-259 190 190 190 340 340 340 T T F <---
213022024-02-12T01:43:37.511Z 13 260-279 186 186 186 333 333 333 T T F <---
213032024-02-12T01:43:37.511Z 14 280-299 187 187 187 335 335 335 F F F
213042024-02-12T01:43:37.511Z 15 300-319 188 188 188 336 336 336 F F F
213052024-02-12T01:43:37.511Z 16 320-339 188 188 188 336 336 336 F F F
213062024-02-12T01:43:37.511Z 17 340-359 188 188 188 336 336 336 T T F <---
213072024-02-12T01:43:37.511Z 18 360-379 192 192 188 343 343 336 F F T <---
213082024-02-12T01:43:37.511Z 19 380-399 192 192 188 343 343 336 T T T <---
213092024-02-12T01:43:37.511Z 20 400-419 187 187 187 335 335 335 T T F <---
213102024-02-12T01:43:37.511Z 21 420-439 191 191 191 341 341 341 F F F
213112024-02-12T01:43:37.511Z 22 440-459 191 191 191 341 341 341 F F F
213122024-02-12T01:43:37.512Z 23 460-479 186 186 186 333 333 333 F F F
213132024-02-12T01:43:37.512Z 24 480-499 188 188 188 336 336 336 T T F <---
213142024-02-12T01:43:37.512Z 25 500-519 185 185 185 332 332 332 F F F
213152024-02-12T01:43:37.512Z 26 520-539 192 192 192 342 342 342 F F F
213162024-02-12T01:43:37.512Z 27 540-559 192 192 190 343 343 339 T T F <---
213172024-02-12T01:43:37.512Z 28 560-579 190 190 190 340 340 340 T T F <---
213182024-02-12T01:43:37.512Z 29 580-599 192 192 192 342 342 342 F F F
213192024-02-12T01:43:37.512ZMax gen: 192, Max flush: 343
213202024-02-12T01:43:37.512ZError: Difference in extent metadata found!
213212024-02-12T01:43:37.512ZA Difference in extent metadata is expected here
213222024-02-12T01:43:37.512ZOn loop 96
213232024-02-12T01:43:37.512Z
213242024-02-12T01:43:37.512ZVerifying data now
213252024-02-12T01:43:43.248ZNo change after verify
213262024-02-12T01:43:43.248ZLoop: 96 Downstairs dump after verify (and repair):
213272024-02-12T01:43:43.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213282024-02-12T01:43:43.265Z 0 000-019 187 187 187 335 335 335 F F F
213292024-02-12T01:43:43.265Z 1 020-039 190 190 190 337 337 337 F F F
213302024-02-12T01:43:43.265Z 2 040-059 177 177 177 318 318 318 F F F
213312024-02-12T01:43:43.265Z 3 060-079 193 193 193 344 344 344 F F F
213322024-02-12T01:43:43.265Z 4 080-099 193 193 193 344 344 344 F F F
213332024-02-12T01:43:43.266Z 5 100-119 184 184 184 329 329 329 F F F
213342024-02-12T01:43:43.266Z 6 120-139 184 184 184 329 329 329 F F F
213352024-02-12T01:43:43.269Z 7 140-159 193 193 193 344 344 344 F F F
213362024-02-12T01:43:43.269Z 8 160-179 192 192 192 343 343 343 F F F
213372024-02-12T01:43:43.269Z 9 180-199 192 192 192 343 343 343 F F F
213382024-02-12T01:43:43.269Z 10 200-219 190 190 190 340 340 340 F F F
213392024-02-12T01:43:43.269Z 11 220-239 193 193 193 344 344 344 F F F
213402024-02-12T01:43:43.269Z 12 240-259 193 193 193 344 344 344 F F F
213412024-02-12T01:43:43.269Z 13 260-279 193 193 193 344 344 344 F F F
213422024-02-12T01:43:43.269Z 14 280-299 187 187 187 335 335 335 F F F
213432024-02-12T01:43:43.269Z 15 300-319 188 188 188 336 336 336 F F F
213442024-02-12T01:43:43.269Z 16 320-339 188 188 188 336 336 336 F F F
213452024-02-12T01:43:43.269Z 17 340-359 193 193 193 344 344 344 F F F
213462024-02-12T01:43:43.269Z 18 360-379 192 192 192 343 343 343 F F F
213472024-02-12T01:43:43.269Z 19 380-399 193 193 193 344 344 344 F F F
213482024-02-12T01:43:43.269Z 20 400-419 193 193 193 344 344 344 F F F
213492024-02-12T01:43:43.269Z 21 420-439 191 191 191 341 341 341 F F F
213502024-02-12T01:43:43.269Z 22 440-459 191 191 191 341 341 341 F F F
213512024-02-12T01:43:43.269Z 23 460-479 186 186 186 333 333 333 F F F
213522024-02-12T01:43:43.270Z 24 480-499 193 193 193 344 344 344 F F F
213532024-02-12T01:43:43.270Z 25 500-519 185 185 185 332 332 332 F F F
213542024-02-12T01:43:43.270Z 26 520-539 192 192 192 342 342 342 F F F
213552024-02-12T01:43:43.270Z 27 540-559 193 193 193 344 344 344 F F F
213562024-02-12T01:43:43.270Z 28 560-579 193 193 193 344 344 344 F F F
213572024-02-12T01:43:43.270Z 29 580-599 192 192 192 342 342 342 F F F
213582024-02-12T01:43:43.270ZMax gen: 193, Max flush: 344
213592024-02-12T01:43:43.270Z
213602024-02-12T01:43:43.270ZBegin loop 97
213612024-02-12T01:43:43.270ZDownstairs to restart: 0
213622024-02-12T01:43:43.283Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144146627Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
213632024-02-12T01:43:43.284Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144404225Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
213642024-02-12T01:43:43.284Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144423114Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
213652024-02-12T01:43:43.288Z{"msg":"Crucible 5e4a2bf5-c9bd-4b7b-8afa-f75ead77f959 has session id: f28b7eb8-2ca1-4564-b9f6-320a5e711205","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144456353Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213662024-02-12T01:43:43.288Z{"msg":"Upstairs opts: Upstairs UUID: 5e4a2bf5-c9bd-4b7b-8afa-f75ead77f959, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144496071Z"Crucible runtime is spawned
213672024-02-12T01:43:43.288Z,"hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213682024-02-12T01:43:43.288Z{"msg":"Crucible stats registered with UUID: 5e4a2bf5-c9bd-4b7b-8afa-f75ead77f959","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144539738Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213692024-02-12T01:43:43.288Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144576927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
213702024-02-12T01:43:43.288Z{"msg":"5e4a2bf5-c9bd-4b7b-8afa-f75ead77f959 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144620765Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213712024-02-12T01:43:43.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144659213Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213722024-02-12T01:43:43.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144686171Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213732024-02-12T01:43:43.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.14472544Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213742024-02-12T01:43:43.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144760138Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213752024-02-12T01:43:43.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144806486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213762024-02-12T01:43:43.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144840224Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213772024-02-12T01:43:43.289Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144882722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213782024-02-12T01:43:43.289Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.14493435Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213792024-02-12T01:43:43.289Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.144982708Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213802024-02-12T01:43:43.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145104552Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213812024-02-12T01:43:43.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145264764Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213822024-02-12T01:43:43.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145298412Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213832024-02-12T01:43:43.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145776719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213842024-02-12T01:43:43.289Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145823167Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213852024-02-12T01:43:43.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145869365Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213862024-02-12T01:43:43.289Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.145923923Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213872024-02-12T01:43:43.289Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.146427688Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213882024-02-12T01:43:43.289Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.146482466Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213892024-02-12T01:43:43.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147111236Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213902024-02-12T01:43:43.289Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147147964Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213912024-02-12T01:43:43.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147164603Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213922024-02-12T01:43:43.289Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147184722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213932024-02-12T01:43:43.289Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147199642Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213942024-02-12T01:43:43.289Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147223961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213952024-02-12T01:43:43.289Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.14724486Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213962024-02-12T01:43:43.289Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147282078Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213972024-02-12T01:43:43.293Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.147713617Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213982024-02-12T01:43:43.293Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148140287Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
213992024-02-12T01:43:43.293Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148182615Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214002024-02-12T01:43:43.293Z{"msg":"[0]R flush_numbers[0..12]: [335, 337, 318, 344, 344, 329, 329, 344, 343, 343, 340, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148200164Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214012024-02-12T01:43:43.293Z{"msg":"[0]R generation[0..12]: [187, 190, 177, 193, 193, 184, 184, 193, 192, 192, 190, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148223843Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214022024-02-12T01:43:43.294Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148240162Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214032024-02-12T01:43:43.294Z{"msg":"[1]R flush_numbers[0..12]: [335, 337, 318, 344, 344, 329, 329, 344, 343, 343, 340, 344]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148262611Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214042024-02-12T01:43:43.294Z{"msg":"[1]R generation[0..12]: [187, 190, 177, 193, 193, 184, 184, 193, 192, 192, 190, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.14827809Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214052024-02-12T01:43:43.294Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148300179Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214062024-02-12T01:43:43.294Z{"msg":"[2]R flush_numbers[0..12]: [335, 337, 318, 344, 344, 329, 329, 344, 343, 343, 340, 344]Wait for a query_work_queue command to finish before sending IO
214072024-02-12T01:43:43.294Z","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148319148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214082024-02-12T01:43:43.294Z{"msg":"[2]R generation[0..12]: [187, 190, 177, 193, 193, 184, 184, 193, 192, 192, 190, 193]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148340357Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214092024-02-12T01:43:43.294Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148355387Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214102024-02-12T01:43:43.294Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148380905Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214112024-02-12T01:43:43.294Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148395514Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214122024-02-12T01:43:43.294Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148409144Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214132024-02-12T01:43:43.294Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148422063Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214142024-02-12T01:43:43.294Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148445952Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214152024-02-12T01:43:43.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148462222Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214162024-02-12T01:43:43.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148476411Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214172024-02-12T01:43:43.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.14849101Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214182024-02-12T01:43:43.294Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148504229Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214192024-02-12T01:43:43.294Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148516839Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214202024-02-12T01:43:43.294Z{"msg":"5e4a2bf5-c9bd-4b7b-8afa-f75ead77f959 is now active with session: f28b7eb8-2ca1-4564-b9f6-320a5e711205","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148529608Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214212024-02-12T01:43:43.294Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148587665Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911,"session_id":"f28b7eb8-2ca1-4564-b9f6-320a5e711205"}
214222024-02-12T01:43:43.294Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:42.148612464Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
214232024-02-12T01:43:43.294ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
214242024-02-12T01:43:43.294ZLoading write count information from file "/tmp/test_repair_verify.data"
214252024-02-12T01:43:43.295ZRun Repair workload
214262024-02-12T01:43:43.295Z01/30 Write block 29 len 12288 data: 20 18 17
214272024-02-12T01:43:43.295Z02/30 Read block 252 len 16384
214282024-02-12T01:43:43.295Z03/30 Write block 566 len 16384 data: 13 14 15 17
214292024-02-12T01:43:43.295Z04/30 Write block 583 len 16384 data: 13 14 14 16
214302024-02-12T01:43:43.295Z05/30 Read block 516 len 8192
214312024-02-12T01:43:43.298Z06/30 Read block 561 len 12288
214322024-02-12T01:43:43.298Z07/30 Read block 463 len 12288
214332024-02-12T01:43:43.299Z08/30 Read block 42 len 36864
214342024-02-12T01:43:43.299Z09/30 Read block 33 len 16384
214352024-02-12T01:43:43.299Z10/30 Read block 261 len 36864
214362024-02-12T01:43:43.299Z11/30 Write block 34 len 4096 data: 18
214372024-02-12T01:43:43.299Z12/30 Read block 495 len 32768
214382024-02-12T01:43:43.302Z13/30 Write block 74 len 20480 data: 12 13 14 14 15
214392024-02-12T01:43:43.302Z14/30 Write block 426 len 20480 data: 12 9 8 10 10
214402024-02-12T01:43:43.302Z15/30 Read block 219 len 28672
214412024-02-12T01:43:43.302Z16/30 Write block 145 len 32768 data: 20 14 14 14 17 18 17 15
214422024-02-12T01:43:43.305Z17/30 Read block 227 len 28672
214432024-02-12T01:43:43.305Z18/30 Read block 591 len 16384
214442024-02-12T01:43:43.305Z19/30 Flush
214452024-02-12T01:43:43.308Z20/30 Flush
214462024-02-12T01:43:43.311Z21/30 Write block 355 len 12288 data: 10 9 10
214472024-02-12T01:43:43.311Z22/30 Read block 149 len 28672
214482024-02-12T01:43:43.311Z23/30 Write block 341 len 36864 data: 8 10 14 12 12 13 12 10 12
214492024-02-12T01:43:43.311Z24/30 Write block 327 len 4096 data: 13
214502024-02-12T01:43:43.311Z25/30 Read block 550 len 8192
214512024-02-12T01:43:43.314Z26/30 Write block 114 len 16384 data: 16 17 17 17
214522024-02-12T01:43:43.314Z27/30 Read block 194 len 32768
214532024-02-12T01:43:43.314Z28/30 Write block 235 len 20480 data: 14 17 17 18 19
214542024-02-12T01:43:43.318Z29/30 Write block 43 len 20480 data: 14 14 14 16 18
214552024-02-12T01:43:43.318Z30/30 Write block 168 len 40960 data: 17 22 21 20 17 17 17 18 20 18
214562024-02-12T01:43:43.318Z----------------------------------------------------------------
214572024-02-12T01:43:43.318Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
214582024-02-12T01:43:43.318ZStates: Active Active Active
214592024-02-12T01:43:43.318ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
214602024-02-12T01:43:43.318Z 1 Acked 1000 Write 3 Done Done Done false
214612024-02-12T01:43:43.318Z 2 Acked 1001 Read 4 Sent Done Done false
214622024-02-12T01:43:43.318Z 3 Acked 1002 Write 4 Sent Done Done false
214632024-02-12T01:43:43.318Z 4 Acked 1003 Write 4 Sent Done Done false
214642024-02-12T01:43:43.319Z 5 Acked 1004 Read 2 Sent Done Done false
214652024-02-12T01:43:43.319Z 6 Acked 1005 Read 3 Sent Done Done false
214662024-02-12T01:43:43.319Z 7 Acked 1006 Read 3 Sent Done Done false
214672024-02-12T01:43:43.319Z 8 Acked 1007 Read 9 Sent Done Done false
214682024-02-12T01:43:43.319Z 9 Acked 1008 Read 4 Sent Done Done false
214692024-02-12T01:43:43.319Z 10 Acked 1009 Read 9 Sent Done Done false
214702024-02-12T01:43:43.319Z 11 Acked 1010 Write 1 Sent Done Done false
214712024-02-12T01:43:43.319Z 12 Acked 1011 Read 8 Sent Done Done false
214722024-02-12T01:43:43.319Z 13 Acked 1012 Write 5 Sent Done Done false
214732024-02-12T01:43:43.319Z 14 Acked 1013 Write 5 Sent Done Done false
214742024-02-12T01:43:43.319Z 15 Acked 1014 Read 7 Sent Done Done false
214752024-02-12T01:43:43.319Z 16 Acked 1015 Write 8 Sent Done Done false
214762024-02-12T01:43:43.319Z 17 Acked 1016 Read 7 Sent Done Done false
214772024-02-12T01:43:43.319Z 18 Acked 1017 Read 4 Sent Done Done false
214782024-02-12T01:43:43.319Z 19 Acked 1018 Flush 0 Sent Done Done false
214792024-02-12T01:43:43.319Z 20 Acked 1019 Flush 0 Sent Done Done false
214802024-02-12T01:43:43.319Z 21 Acked 1020 Write 3 Sent Done Done false
214812024-02-12T01:43:43.319Z 22 Acked 1021 Read 7 Sent Done Done false
214822024-02-12T01:43:43.319Z 23 Acked 1022 Write 9 Sent Done Done false
214832024-02-12T01:43:43.319Z 24 Acked 1023 Write 1 Sent Done Done false
214842024-02-12T01:43:43.319Z 25 Acked 1024 Read 2 Sent Done Done false
214852024-02-12T01:43:43.319Z 26 Acked 1025 Write 4 Sent Done Done false
214862024-02-12T01:43:43.319Z 27 Acked 1026 Read 8 Sent Done Done false
214872024-02-12T01:43:43.319Z 28 Acked 1027 Write 5 Sent Done Done false
214882024-02-12T01:43:43.320Z 29 Acked 1028 Write 5 Sent Done Done false
214892024-02-12T01:43:43.320Z 30 Acked 1029 Write 10 Sent Sent Sent false
214902024-02-12T01:43:43.320Z STATES DS:0 DS:1 DS:2 TOTAL
214912024-02-12T01:43:43.320Z New 0 0 0 0
214922024-02-12T01:43:43.320Z Sent 29 1 1 31
214932024-02-12T01:43:43.320Z Done 1 29 29 59
214942024-02-12T01:43:43.320Z Skipped 0 0 0 0
214952024-02-12T01:43:43.320Z Error 0 0 0 0
214962024-02-12T01:43:43.320ZLast Flush: 0 1019 1019
214972024-02-12T01:43:43.320ZDownstairs last five completed:
214982024-02-12T01:43:43.320ZUpstairs last five completed: 30 29 28 27 26
214992024-02-12T01:43:43.320Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:42.177089696Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1911}
215002024-02-12T01:43:43.320ZWrote out file "/tmp/test_repair_verify.data"
215012024-02-12T01:43:43.323Z
215022024-02-12T01:43:43.326ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
215032024-02-12T01:43:43.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215042024-02-12T01:43:43.341Z 0 000-019 187 187 187 335 335 335 F F F
215052024-02-12T01:43:43.341Z 1 020-039 190 194 194 337 345 345 T F F <---
215062024-02-12T01:43:43.341Z 2 040-059 177 177 177 318 318 318 F T T <---
215072024-02-12T01:43:43.341Z 3 060-079 193 194 194 344 345 345 F F F <---
215082024-02-12T01:43:43.341Z 4 080-099 193 193 193 344 344 344 F F F
215092024-02-12T01:43:43.341Z 5 100-119 184 184 184 329 329 329 F T T <---
215102024-02-12T01:43:43.341Z 6 120-139 184 184 184 329 329 329 F F F
215112024-02-12T01:43:43.341Z 7 140-159 193 194 194 344 345 345 F F F <---
215122024-02-12T01:43:43.347Z 8 160-179 192 192 192 343 343 343 F T T <---
215132024-02-12T01:43:43.347Z 9 180-199 192 192 192 343 343 343 F F F
215142024-02-12T01:43:43.347Z 10 200-219 190 190 190 340 340 340 F F F
215152024-02-12T01:43:43.347Z 11 220-239 193 193 193 344 344 344 F T T <---
215162024-02-12T01:43:43.347Z 12 240-259 193 193 193 344 344 344 F F F
215172024-02-12T01:43:43.347Z 13 260-279 193 193 193 344 344 344 F F F
215182024-02-12T01:43:43.347Z 14 280-299 187 187 187 335 335 335 F F F
215192024-02-12T01:43:43.347Z 15 300-319 188 188 188 336 336 336 F F F
215202024-02-12T01:43:43.347Z 16 320-339 188 188 188 336 336 336 F T T <---
215212024-02-12T01:43:43.347Z 17 340-359 193 193 193 344 344 344 F T T <---
215222024-02-12T01:43:43.347Z 18 360-379 192 192 192 343 343 343 F F F
215232024-02-12T01:43:43.347Z 19 380-399 193 193 193 344 344 344 F F F
215242024-02-12T01:43:43.348Z 20 400-419 193 193 193 344 344 344 F F F
215252024-02-12T01:43:43.348Z 21 420-439 191 194 194 341 345 345 F F F <---
215262024-02-12T01:43:43.348Z 22 440-459 191 191 191 341 341 341 F F F
215272024-02-12T01:43:43.348Z 23 460-479 186 186 186 333 333 333 F F F
215282024-02-12T01:43:43.348Z 24 480-499 193 193 193 344 344 344 F F F
215292024-02-12T01:43:43.348Z 25 500-519 185 185 185 332 332 332 F F F
215302024-02-12T01:43:43.348Z 26 520-539 192 192 192 342 342 342 F F F
215312024-02-12T01:43:43.348Z 27 540-559 193 193 193 344 344 344 F F F
215322024-02-12T01:43:43.348Z 28 560-579 193 194 194 344 345 345 F F F <---
215332024-02-12T01:43:43.348Z 29 580-599 192 194 194 342 345 345 F F F <---
215342024-02-12T01:43:43.348ZMax gen: 194, Max flush: 345
215352024-02-12T01:43:43.348ZError: Difference in extent metadata found!
215362024-02-12T01:43:43.348ZA Difference in extent metadata is expected here
215372024-02-12T01:43:43.348ZOn loop 97
215382024-02-12T01:43:43.348Z
215392024-02-12T01:43:43.351ZVerifying data now
215402024-02-12T01:43:47.959ZNo change after verify
215412024-02-12T01:43:47.959ZLoop: 97 Downstairs dump after verify (and repair):
215422024-02-12T01:43:47.973ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215432024-02-12T01:43:47.973Z 0 000-019 187 187 187 335 335 335 F F F
215442024-02-12T01:43:47.974Z 1 020-039 194 194 194 345 345 345 F F F
215452024-02-12T01:43:47.974Z 2 040-059 195 195 195 346 346 346 F F F
215462024-02-12T01:43:47.974Z 3 060-079 194 194 194 345 345 345 F F F
215472024-02-12T01:43:47.974Z 4 080-099 193 193 193 344 344 344 F F F
215482024-02-12T01:43:47.974Z 5 100-119 195 195 195 346 346 346 F F F
215492024-02-12T01:43:47.974Z 6 120-139 184 184 184 329 329 329 F F F
215502024-02-12T01:43:47.974Z 7 140-159 194 194 194 345 345 345 F F F
215512024-02-12T01:43:47.974Z 8 160-179 195 195 195 346 346 346 F F F
215522024-02-12T01:43:47.977Z 9 180-199 192 192 192 343 343 343 F F F
215532024-02-12T01:43:47.977Z 10 200-219 190 190 190 340 340 340 F F F
215542024-02-12T01:43:47.977Z 11 220-239 195 195 195 346 346 346 F F F
215552024-02-12T01:43:47.977Z 12 240-259 193 193 193 344 344 344 F F F
215562024-02-12T01:43:47.977Z 13 260-279 193 193 193 344 344 344 F F F
215572024-02-12T01:43:47.977Z 14 280-299 187 187 187 335 335 335 F F F
215582024-02-12T01:43:47.977Z 15 300-319 188 188 188 336 336 336 F F F
215592024-02-12T01:43:47.977Z 16 320-339 195 195 195 346 346 346 F F F
215602024-02-12T01:43:47.977Z 17 340-359 195 195 195 346 346 346 F F F
215612024-02-12T01:43:47.978Z 18 360-379 192 192 192 343 343 343 F F F
215622024-02-12T01:43:47.978Z 19 380-399 193 193 193 344 344 344 F F F
215632024-02-12T01:43:47.978Z 20 400-419 193 193 193 344 344 344 F F F
215642024-02-12T01:43:47.978Z 21 420-439 194 194 194 345 345 345 F F F
215652024-02-12T01:43:47.978Z 22 440-459 191 191 191 341 341 341 F F F
215662024-02-12T01:43:47.978Z 23 460-479 186 186 186 333 333 333 F F F
215672024-02-12T01:43:47.978Z 24 480-499 193 193 193 344 344 344 F F F
215682024-02-12T01:43:47.978Z 25 500-519 185 185 185 332 332 332 F F F
215692024-02-12T01:43:47.978Z 26 520-539 192 192 192 342 342 342 F F F
215702024-02-12T01:43:47.978Z 27 540-559 193 193 193 344 344 344 F F F
215712024-02-12T01:43:47.978Z 28 560-579 194 194 194 345 345 345 F F F
215722024-02-12T01:43:47.978Z 29 580-599 194 194 194 345 345 345 F F F
215732024-02-12T01:43:47.978ZMax gen: 195, Max flush: 346
215742024-02-12T01:43:47.978Z
215752024-02-12T01:43:47.978ZBegin loop 98
215762024-02-12T01:43:47.978ZDownstairs to restart: 2
215772024-02-12T01:43:47.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856030247Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
215782024-02-12T01:43:47.996Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
215792024-02-12T01:43:47.996Z,"time":"2024-02-12T01:43:46.856302854Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
215802024-02-12T01:43:48.001Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856346672Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
215812024-02-12T01:43:48.002Z{"msg":"Crucible 648e2dd5-e865-426f-9a70-d9fcfa734243 has session id: 9bb43644-c8e0-44c9-bcb2-5f6ead936d9a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.8563814Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215822024-02-12T01:43:48.002Z{"msg":"Upstairs opts: Upstairs UUID: 648e2dd5-e865-426f-9a70-d9fcfa734243, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856426328Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215832024-02-12T01:43:48.002Z{"msg":"Crucible stats registered with UUID: 648e2dd5-e865-426f-9a70-d9fcfa734243","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856451897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215842024-02-12T01:43:48.002Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856483046Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
215852024-02-12T01:43:48.002Z{"msg":"648e2dd5-e865-426f-9a70-d9fcfa734243 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856543413Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215862024-02-12T01:43:48.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856580291Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215872024-02-12T01:43:48.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85659844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215882024-02-12T01:43:48.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856620659Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215892024-02-12T01:43:48.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856635958Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215902024-02-12T01:43:48.002Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856657307Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215912024-02-12T01:43:48.002Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856674626Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215922024-02-12T01:43:48.002Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856707215Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215932024-02-12T01:43:48.002Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856730844Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215942024-02-12T01:43:48.003Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856795701Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215952024-02-12T01:43:48.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856889196Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215962024-02-12T01:43:48.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856933404Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215972024-02-12T01:43:48.003Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.856983912Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215982024-02-12T01:43:48.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.857596522Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
215992024-02-12T01:43:48.003Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85764822Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216002024-02-12T01:43:48.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.857680318Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216012024-02-12T01:43:48.003Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.857704227Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216022024-02-12T01:43:48.003Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858294789Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216032024-02-12T01:43:48.003Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858330927Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216042024-02-12T01:43:48.003Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858603265Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216052024-02-12T01:43:48.003Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858717559Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216062024-02-12T01:43:48.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858864722Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216072024-02-12T01:43:48.003Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85890148Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216082024-02-12T01:43:48.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858940758Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216092024-02-12T01:43:48.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858964827Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216102024-02-12T01:43:48.003Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.858990506Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216112024-02-12T01:43:48.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859004925Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216122024-02-12T01:43:48.003Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859179477Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216132024-02-12T01:43:48.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859584288Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216142024-02-12T01:43:48.003Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859623036Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216152024-02-12T01:43:48.003Z{"msg":"[0]R flush_numbers[0..12]: [335, 345, 346, 345, 344, 346, 329, 345, 346, 343, 340, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859639275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216162024-02-12T01:43:48.004Z{"msg":"[0]R generation[0..12]: [187, 194, 195, 194, 193, 195, 184, 194, 195, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859654294Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216172024-02-12T01:43:48.004Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859675713Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216182024-02-12T01:43:48.007Z{"msg":"[1]R flush_numbers[0..12]: [335, 345, 346, 345, 344, 346, 329, 345, 346, 343, 340, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859690503Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216192024-02-12T01:43:48.007Z{"msg":"[1]R generation[0..12]: [187, 194, 195, 194, 193, 195, 184, 194, 195, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859710982Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216202024-02-12T01:43:48.007Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859725651Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216212024-02-12T01:43:48.007Z{"msg":"[2]R flush_numbers[0..12]: [335, 345, 346, 345, 344, 346, 329, 345, 346, 343, 340, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85974605Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216222024-02-12T01:43:48.007Z{"msg":"[2]R generation[0..12]: [187, 194, 195, 194, 193, 195, 184, 194, 195, 192, 190, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859760079Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216232024-02-12T01:43:48.007Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859786478Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216242024-02-12T01:43:48.008Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859800107Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216252024-02-12T01:43:48.008Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859813897Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216262024-02-12T01:43:48.008Z{"msg":"Next flush: 347","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
216272024-02-12T01:43:48.008Z30,"time":"2024-02-12T01:43:46.859830176Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216282024-02-12T01:43:48.008Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859843105Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216292024-02-12T01:43:48.008Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859857085Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216302024-02-12T01:43:48.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859869124Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216312024-02-12T01:43:48.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859882933Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216322024-02-12T01:43:48.008Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859895993Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216332024-02-12T01:43:48.008Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859909462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216342024-02-12T01:43:48.008Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859922031Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216352024-02-12T01:43:48.008Z{"msg":"648e2dd5-e865-426f-9a70-d9fcfa734243 is now active with session: 9bb43644-c8e0-44c9-bcb2-5f6ead936d9a","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85994621Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216362024-02-12T01:43:48.008Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.85996163Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919,"session_id":"9bb43644-c8e0-44c9-bcb2-5f6ead936d9a"}
216372024-02-12T01:43:48.008Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:46.859973969Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
216382024-02-12T01:43:48.008ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
216392024-02-12T01:43:48.008ZLoading write count information from file "/tmp/test_repair_verify.data"
216402024-02-12T01:43:48.008ZRun Repair workload
216412024-02-12T01:43:48.008Z01/30 Write block 562 len 28672 data: 12 15 12 13 14 15 16
216422024-02-12T01:43:48.008Z02/30 Write block 331 len 20480 data: 9 11 8 8 8
216432024-02-12T01:43:48.008Z03/30 Write block 292 len 16384 data: 17 17 14 15
216442024-02-12T01:43:48.008Z04/30 Read block 374 len 28672
216452024-02-12T01:43:48.008Z05/30 Read block 547 len 4096
216462024-02-12T01:43:48.009Z06/30 Read block 509 len 4096
216472024-02-12T01:43:48.009Z07/30 Read block 169 len 36864
216482024-02-12T01:43:48.012Z08/30 Write block 305 len 40960 data: 15 16 18 18 17 17 16 12 16 15
216492024-02-12T01:43:48.012Z09/30 Read block 291 len 4096
216502024-02-12T01:43:48.012Z10/30 Write block 523 len 24576 data: 14 13 9 11 10 11
216512024-02-12T01:43:48.012Z11/30 Read block 351 len 36864
216522024-02-12T01:43:48.012Z12/30 Write block 527 len 16384 data: 11 12 12 16
216532024-02-12T01:43:48.015Z13/30 Write block 580 len 40960 data: 14 14 13 14 15 15 17 14 13 14
216542024-02-12T01:43:48.015Z14/30 Read block 194 len 16384
216552024-02-12T01:43:48.015Z15/30 Write block 382 len 32768 data: 13 13 14 17 19 18 19 19
216562024-02-12T01:43:48.020Z16/30 Write block 523 len 8192 data: 15 14
216572024-02-12T01:43:48.021Z17/30 Write block 548 len 8192 data: 14 18
216582024-02-12T01:43:48.021Z18/30 Write block 195 len 32768 data: 13 15 14 13 16 16 15 18
216592024-02-12T01:43:48.021Z19/30 Read block 222 len 20480
216602024-02-12T01:43:48.024Z20/30 Flush
216612024-02-12T01:43:48.024Z21/30 Write block 505 len 12288 data: 14 13 14
216622024-02-12T01:43:48.027Z22/30 Read block 93 len 4096
216632024-02-12T01:43:48.027Z23/30 Write block 15 len 12288 data: 13 9 8
216642024-02-12T01:43:48.027Z24/30 Write block 271 len 28672 data: 22 22 21 22 15 11 12
216652024-02-12T01:43:48.027Z25/30 Read block 83 len 36864
216662024-02-12T01:43:48.030Z26/30 Read block 68 len 4096
216672024-02-12T01:43:48.030Z27/30 Read block 367 len 40960
216682024-02-12T01:43:48.030Z28/30 Read block 112 len 12288
216692024-02-12T01:43:48.033Z29/30 Write block 85 len 20480 data: 21 17 20 16 13
216702024-02-12T01:43:48.033Z30/30 Read block 560 len 8192
216712024-02-12T01:43:48.033Z----------------------------------------------------------------
216722024-02-12T01:43:48.033Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
216732024-02-12T01:43:48.033ZStates: Active Active Active
216742024-02-12T01:43:48.033ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
216752024-02-12T01:43:48.033Z 1 Acked 1000 Write 7 Done Done Done false
216762024-02-12T01:43:48.033Z 2 Acked 1001 Write 5 Done Done Done false
216772024-02-12T01:43:48.033Z 3 Acked 1002 Write 4 Done Done Done false
216782024-02-12T01:43:48.033Z 4 Acked 1003 Read 7 Done Done Done false
216792024-02-12T01:43:48.034Z 5 Acked 1004 Read 1 Done Done Done false
216802024-02-12T01:43:48.034Z 6 Acked 1005 Read 1 Done Done Done false
216812024-02-12T01:43:48.034Z 7 Acked 1006 Read 9 Done Done Sent false
216822024-02-12T01:43:48.034Z 8 Acked 1007 Write 10 Done Done Sent false
216832024-02-12T01:43:48.034Z 9 Acked 1008 Read 1 Done Done Sent false
216842024-02-12T01:43:48.034Z 10 Acked 1009 Write 6 Done Done Sent false
216852024-02-12T01:43:48.034Z 11 Acked 1010 Read 9 Done Done Sent false
216862024-02-12T01:43:48.034Z 12 Acked 1011 Write 4 Done Done Sent false
216872024-02-12T01:43:48.034Z 13 Acked 1012 Write 10 Done Done Sent false
216882024-02-12T01:43:48.034Z 14 Acked 1013 Read 4 Done Done Sent false
216892024-02-12T01:43:48.034Z 15 Acked 1014 Write 8 Done Done Sent false
216902024-02-12T01:43:48.034Z 16 Acked 1015 Write 2 Done Done Sent false
216912024-02-12T01:43:48.034Z 17 Acked 1016 Write 2 Done Done Sent false
216922024-02-12T01:43:48.034Z 18 Acked 1017 Write 8 Done Done Sent false
216932024-02-12T01:43:48.034Z 19 Acked 1018 Read 5 Done Done Sent false
216942024-02-12T01:43:48.034Z 20 Acked 1019 Flush 0 Done Done Sent false
216952024-02-12T01:43:48.034Z 21 Acked 1020 Write 3 Done Done Sent false
216962024-02-12T01:43:48.034Z 22 Acked 1021 Read 1 Done Done Sent false
216972024-02-12T01:43:48.034Z 23 Acked 1022 Write 3 Done Done Sent false
216982024-02-12T01:43:48.034Z 24 Acked 1023 Write 7 Done Done Sent false
216992024-02-12T01:43:48.034Z 25 Acked 1024 Read 9 Done Done Sent false
217002024-02-12T01:43:48.034Z 26 Acked 1025 Read 1 Done Done Sent false
217012024-02-12T01:43:48.034Z 27 Acked 1026 Read 10 Done Done Sent false
217022024-02-12T01:43:48.035Z 28 Acked 1027 Read 3 Done Done Sent false
217032024-02-12T01:43:48.035Z 29 Acked 1028 Write 5 Done Done Sent false
217042024-02-12T01:43:48.035Z 30 Acked 1029 Read 2 Sent Done Sent false
217052024-02-12T01:43:48.035Z STATES DS:0 DS:1 DS:2 TOTAL
217062024-02-12T01:43:48.035Z New 0 0 0 0
217072024-02-12T01:43:48.035Z Sent 1 0 24 25
217082024-02-12T01:43:48.035Z Done 29 30 6 65
217092024-02-12T01:43:48.035Z Skipped 0 0 0 0
217102024-02-12T01:43:48.035Z Error 0 0 0 0
217112024-02-12T01:43:48.035ZLast Flush: 1019 1019 0
217122024-02-12T01:43:48.035ZDownstairs last five completed:
217132024-02-12T01:43:48.035ZUpstairs last five completed: 30 29 28 27 26
217142024-02-12T01:43:48.035Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:46.892049719Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1919}
217152024-02-12T01:43:48.035ZWrote out file "/tmp/test_repair_verify.data"
217162024-02-12T01:43:48.038Z
217172024-02-12T01:43:48.042ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
217182024-02-12T01:43:48.056ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217192024-02-12T01:43:48.056Z 0 000-019 187 187 187 335 335 335 T T F <---
217202024-02-12T01:43:48.056Z 1 020-039 194 194 194 345 345 345 F F F
217212024-02-12T01:43:48.057Z 2 040-059 195 195 195 346 346 346 F F F
217222024-02-12T01:43:48.057Z 3 060-079 194 194 194 345 345 345 F F F
217232024-02-12T01:43:48.057Z 4 080-099 193 193 193 344 344 344 T T F <---
217242024-02-12T01:43:48.059Z 5 100-119 195 195 195 346 346 346 F F F
217252024-02-12T01:43:48.060Z 6 120-139 184 184 184 329 329 329 F F F
217262024-02-12T01:43:48.060Z 7 140-159 194 194 194 345 345 345 F F F
217272024-02-12T01:43:48.060Z 8 160-179 195 195 195 346 346 346 F F F
217282024-02-12T01:43:48.060Z 9 180-199 196 196 192 347 347 343 F F F <---
217292024-02-12T01:43:48.060Z 10 200-219 196 196 190 347 347 340 F F F <---
217302024-02-12T01:43:48.060Z 11 220-239 195 195 195 346 346 346 F F F
217312024-02-12T01:43:48.060Z 12 240-259 193 193 193 344 344 344 F F F
217322024-02-12T01:43:48.060Z 13 260-279 193 193 193 344 344 344 T T F <---
217332024-02-12T01:43:48.060Z 14 280-299 196 196 187 347 347 335 F F T <---
217342024-02-12T01:43:48.060Z 15 300-319 196 196 188 347 347 336 F F F <---
217352024-02-12T01:43:48.060Z 16 320-339 196 196 195 347 347 346 F F T <---
217362024-02-12T01:43:48.060Z 17 340-359 195 195 195 346 346 346 F F F
217372024-02-12T01:43:48.060Z 18 360-379 192 192 192 343 343 343 F F F
217382024-02-12T01:43:48.060Z 19 380-399 196 196 193 347 347 344 F F F <---
217392024-02-12T01:43:48.060Z 20 400-419 193 193 193 344 344 344 F F F
217402024-02-12T01:43:48.060Z 21 420-439 194 194 194 345 345 345 F F F
217412024-02-12T01:43:48.060Z 22 440-459 191 191 191 341 341 341 F F F
217422024-02-12T01:43:48.060Z 23 460-479 186 186 186 333 333 333 F F F
217432024-02-12T01:43:48.060Z 24 480-499 193 193 193 344 344 344 F F F
217442024-02-12T01:43:48.060Z 25 500-519 185 185 185 332 332 332 T T F <---
217452024-02-12T01:43:48.060Z 26 520-539 196 196 192 347 347 342 F F F <---
217462024-02-12T01:43:48.060Z 27 540-559 196 196 193 347 347 344 F F F <---
217472024-02-12T01:43:48.060Z 28 560-579 196 196 194 347 347 345 F F T <---
217482024-02-12T01:43:48.061Z 29 580-599 196 196 194 347 347 345 F F F <---
217492024-02-12T01:43:48.061ZMax gen: 196, Max flush: 347
217502024-02-12T01:43:48.061ZError: Difference in extent metadata found!
217512024-02-12T01:43:48.064ZA Difference in extent metadata is expected here
217522024-02-12T01:43:48.064ZOn loop 98
217532024-02-12T01:43:48.064Z
217542024-02-12T01:43:48.064ZVerifying data now
217552024-02-12T01:43:53.733ZNo change after verify
217562024-02-12T01:43:53.734ZLoop: 98 Downstairs dump after verify (and repair):
217572024-02-12T01:43:53.749ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217582024-02-12T01:43:53.749Z 0 000-019 197 197 197 348 348 348 F F F
217592024-02-12T01:43:53.749Z 1 020-039 194 194 194 345 345 345 F F F
217602024-02-12T01:43:53.749Z 2 040-059 195 195 195 346 346 346 F F F
217612024-02-12T01:43:53.749Z 3 060-079 194 194 194 345 345 345 F F F
217622024-02-12T01:43:53.749Z 4 080-099 197 197 197 348 348 348 F F F
217632024-02-12T01:43:53.752Z 5 100-119 195 195 195 346 346 346 F F F
217642024-02-12T01:43:53.752Z 6 120-139 184 184 184 329 329 329 F F F
217652024-02-12T01:43:53.752Z 7 140-159 194 194 194 345 345 345 F F F
217662024-02-12T01:43:53.752Z 8 160-179 195 195 195 346 346 346 F F F
217672024-02-12T01:43:53.752Z 9 180-199 196 196 196 347 347 347 F F F
217682024-02-12T01:43:53.752Z 10 200-219 196 196 196 347 347 347 F F F
217692024-02-12T01:43:53.752Z 11 220-239 195 195 195 346 346 346 F F F
217702024-02-12T01:43:53.752Z 12 240-259 193 193 193 344 344 344 F F F
217712024-02-12T01:43:53.752Z 13 260-279 197 197 197 348 348 348 F F F
217722024-02-12T01:43:53.752Z 14 280-299 196 196 196 347 347 347 F F F
217732024-02-12T01:43:53.752Z 15 300-319 196 196 196 347 347 347 F F F
217742024-02-12T01:43:53.752Z 16 320-339 196 196 196 347 347 347 F F F
217752024-02-12T01:43:53.752Z 17 340-359 195 195 195 346 346 346 F F F
217762024-02-12T01:43:53.753Z 18 360-379 192 192 192 343 343 343 F F F
217772024-02-12T01:43:53.753Z 19 380-399 196 196 196 347 347 347 F F F
217782024-02-12T01:43:53.753Z 20 400-419 193 193 193 344 344 344 F F F
217792024-02-12T01:43:53.753Z 21 420-439 194 194 194 345 345 345 F F F
217802024-02-12T01:43:53.753Z 22 440-459 191 191 191 341 341 341 F F F
217812024-02-12T01:43:53.753Z 23 460-479 186 186 186 333 333 333 F F F
217822024-02-12T01:43:53.753Z 24 480-499 193 193 193 344 344 344 F F F
217832024-02-12T01:43:53.753Z 25 500-519 197 197 197 348 348 348 F F F
217842024-02-12T01:43:53.753Z 26 520-539 196 196 196 347 347 347 F F F
217852024-02-12T01:43:53.753Z 27 540-559 196 196 196 347 347 347 F F F
217862024-02-12T01:43:53.753Z 28 560-579 196 196 196 347 347 347 F F F
217872024-02-12T01:43:53.753Z 29 580-599 196 196 196 347 347 347 F F F
217882024-02-12T01:43:53.753ZMax gen: 197, Max flush: 348
217892024-02-12T01:43:53.753Z
217902024-02-12T01:43:53.753ZBegin loop 99
217912024-02-12T01:43:53.753ZDownstairs to restart: 0
217922024-02-12T01:43:53.771Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.629882671Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927}
217932024-02-12T01:43:53.771Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"9f74bb5fe2be09de9f7a47f2db64d7dce836b79c\",\n git_commit_timestamp: \"2024-02-12T01:11:14.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630191756Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927Crucible runtime is spawned
217942024-02-12T01:43:53.771Z}
217952024-02-12T01:43:53.775Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630238914Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927}
217962024-02-12T01:43:53.775Z{"msg":"Crucible ae16d987-bb01-4ab5-b000-b2b8e76e4888 has session id: 42b9f4aa-a865-4ec9-9a52-092544249007","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630283462Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
217972024-02-12T01:43:53.775Z{"msg":"Upstairs opts: Upstairs UUID: ae16d987-bb01-4ab5-b000-b2b8e76e4888, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630340519Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
217982024-02-12T01:43:53.775Z{"msg":"Crucible stats registered with UUID: ae16d987-bb01-4ab5-b000-b2b8e76e4888","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630378267Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
217992024-02-12T01:43:53.775Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630407876Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927}
218002024-02-12T01:43:53.775Z{"msg":"ae16d987-bb01-4ab5-b000-b2b8e76e4888 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630431565Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218012024-02-12T01:43:53.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630453044Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218022024-02-12T01:43:53.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630468373Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218032024-02-12T01:43:53.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630483022Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218042024-02-12T01:43:53.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630496901Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218052024-02-12T01:43:53.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630510081Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218062024-02-12T01:43:53.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63052662Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218072024-02-12T01:43:53.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630651664Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218082024-02-12T01:43:53.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630689972Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218092024-02-12T01:43:53.776Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63073493Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218102024-02-12T01:43:53.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.630904882Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218112024-02-12T01:43:53.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.631095903Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218122024-02-12T01:43:53.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63115172Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218132024-02-12T01:43:53.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63157629Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218142024-02-12T01:43:53.776Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.631612528Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218152024-02-12T01:43:53.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.631631317Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218162024-02-12T01:43:53.776Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.631663676Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218172024-02-12T01:43:53.776Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632167732Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218182024-02-12T01:43:53.776Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632301275Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218192024-02-12T01:43:53.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63262157Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218202024-02-12T01:43:53.776Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632706286Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218212024-02-12T01:43:53.776Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632736725Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218222024-02-12T01:43:53.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632779303Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218232024-02-12T01:43:53.776Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632817961Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218242024-02-12T01:43:53.776Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.632852419Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218252024-02-12T01:43:53.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.633210812Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218262024-02-12T01:43:53.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.633283928Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218272024-02-12T01:43:53.780Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.633782995Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218282024-02-12T01:43:53.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634190645Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218292024-02-12T01:43:53.781Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634219084Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218302024-02-12T01:43:53.781Z{"msg":"[0]R flush_numbers[0..12]: [348, 345, 346, 345, 348, 346, 329, 345, 346, 347, 347, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634241753Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218312024-02-12T01:43:53.781Z{"msg":"[0]R generation[0..12]: [197, 194, 195, 194, 197, 195, 184, 194, 195, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634262942Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218322024-02-12T01:43:53.781Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634278071Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218332024-02-12T01:43:53.781Z{"msg":"[1]R flush_numbers[0..12]: [348, 345, 346, 345, 348, 346, 329, 345, 346, 347, 347, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63429894Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218342024-02-12T01:43:53.781Z{"msg":"[1]R generation[0..12]: [197, 194, 195, 194, 197, 195, 184, 194, 195, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63431325Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218352024-02-12T01:43:53.781Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634333578Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218362024-02-12T01:43:53.781Z{"msg":"[2]R flush_numbers[0..12]: [348, 345, 346, 345, 348, 346, 329, 345, 346, 347, 347, 346]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634348408Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218372024-02-12T01:43:53.781Z{"msg":"[2]R generation[0..12]: [197, 194, 195, 194, 197, 195, 184, 194, 195, 196, 196, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634381156Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927Wait for a query_work_queue command to finish before sending IO
218382024-02-12T01:43:53.781Z,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218392024-02-12T01:43:53.781Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634416274Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218402024-02-12T01:43:53.781Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634444013Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218412024-02-12T01:43:53.781Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634458583Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218422024-02-12T01:43:53.781Z{"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634480502Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218432024-02-12T01:43:53.781Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63450625Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218442024-02-12T01:43:53.781Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634520769Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218452024-02-12T01:43:53.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634543899Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218462024-02-12T01:43:53.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634571257Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218472024-02-12T01:43:53.781Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634594536Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218482024-02-12T01:43:53.781Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634622015Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218492024-02-12T01:43:53.781Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634644284Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218502024-02-12T01:43:53.781Z{"msg":"ae16d987-bb01-4ab5-b000-b2b8e76e4888 is now active with session: 42b9f4aa-a865-4ec9-9a52-092544249007","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634672442Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218512024-02-12T01:43:53.781Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.634693681Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927,"session_id":"42b9f4aa-a865-4ec9-9a52-092544249007"}
218522024-02-12T01:43:53.781Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-12T01:43:52.63471977Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927}
218532024-02-12T01:43:53.782ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
218542024-02-12T01:43:53.782ZLoading write count information from file "/tmp/test_repair_verify.data"
218552024-02-12T01:43:53.782ZRun Repair workload
218562024-02-12T01:43:53.782Z01/30 Write block 174 len 20480 data: 18 19 21 19 21
218572024-02-12T01:43:53.782Z02/30 Read block 311 len 32768
218582024-02-12T01:43:53.782Z03/30 Read block 15 len 40960
218592024-02-12T01:43:53.782Z04/30 Read block 26 len 4096
218602024-02-12T01:43:53.785Z05/30 Flush
218612024-02-12T01:43:53.785Z06/30 Write block 544 len 8192 data: 12 14
218622024-02-12T01:43:53.785Z07/30 Read block 396 len 12288
218632024-02-12T01:43:53.785Z08/30 Write block 72 len 20480 data: 13 12 13 14 15
218642024-02-12T01:43:53.785Z09/30 Write block 97 len 36864 data: 11 11 9 8 11 10 10 14 16
218652024-02-12T01:43:53.788Z10/30 Read block 237 len 20480
218662024-02-12T01:43:53.788Z11/30 Read block 333 len 8192
218672024-02-12T01:43:53.788Z12/30 Read block 128 len 32768
218682024-02-12T01:43:53.788Z13/30 Read block 340 len 20480
218692024-02-12T01:43:53.791Z14/30 Read block 64 len 16384
218702024-02-12T01:43:53.791Z15/30 Read block 520 len 8192
218712024-02-12T01:43:53.791Z16/30 Write block 469 len 36864 data: 14 14 13 15 19 20 19 16 15
218722024-02-12T01:43:53.791Z17/30 Read block 329 len 20480
218732024-02-12T01:43:53.795Z18/30 Read block 19 len 24576
218742024-02-12T01:43:53.795Z19/30 Read block 11 len 8192
218752024-02-12T01:43:53.795Z20/30 Write block 6 len 32768 data: 13 11 11 12 11 12 13 13
218762024-02-12T01:43:53.797Z21/30 Write block 2 len 20480 data: 7 8 9 12 14
218772024-02-12T01:43:53.798Z22/30 Write block 214 len 32768 data: 12 12 16 12 15 15 13 13
218782024-02-12T01:43:53.798Z23/30 Write block 58 len 32768 data: 18 14 12 10 15 17 18 20
218792024-02-12T01:43:53.798Z24/30 Flush
218802024-02-12T01:43:53.802Z25/30 Write block 331 len 40960 data: 10 12 9 9 9 8 11 10 11 11
218812024-02-12T01:43:53.805Z26/30 Write block 587 len 24576 data: 15 14 15 18 15 15
218822024-02-12T01:43:53.805Z27/30 Write block 433 len 20480 data: 9 11 10 10 12
218832024-02-12T01:43:53.805Z28/30 Write block 541 len 20480 data: 15 17 13 13 15
218842024-02-12T01:43:53.805Z29/30 Read block 339 len 12288
218852024-02-12T01:43:53.808Z30/30 Write block 25 len 20480 data: 17 16 15 17 21
218862024-02-12T01:43:53.808Z----------------------------------------------------------------
218872024-02-12T01:43:53.809Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:25
218882024-02-12T01:43:53.809ZStates: Active Active Active
218892024-02-12T01:43:53.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
218902024-02-12T01:43:53.809Z 6 Acked 1005 Write 2 Sent Done Done false
218912024-02-12T01:43:53.809Z 7 Acked 1006 Read 3 Sent Done Done false
218922024-02-12T01:43:53.809Z 8 Acked 1007 Write 5 Sent Done Done false
218932024-02-12T01:43:53.809Z 9 Acked 1008 Write 9 Sent Done Done false
218942024-02-12T01:43:53.809Z 10 Acked 1009 Read 5 Sent Done Done false
218952024-02-12T01:43:53.809Z 11 Acked 1010 Read 2 Sent Done Done false
218962024-02-12T01:43:53.809Z 12 Acked 1011 Read 8 Sent Done Done false
218972024-02-12T01:43:53.809Z 13 Acked 1012 Read 5 Sent Done Done false
218982024-02-12T01:43:53.809Z 14 Acked 1013 Read 4 Sent Done Done false
218992024-02-12T01:43:53.809Z 15 Acked 1014 Read 2 Sent Done Done false
219002024-02-12T01:43:53.809Z 16 Acked 1015 Write 9 Sent Done Done false
219012024-02-12T01:43:53.809Z 17 Acked 1016 Read 5 Sent Done Done false
219022024-02-12T01:43:53.809Z 18 Acked 1017 Read 6 Sent Done Done false
219032024-02-12T01:43:53.809Z 19 Acked 1018 Read 2 Sent Done Done false
219042024-02-12T01:43:53.809Z 20 Acked 1019 Write 8 Sent Done Done false
219052024-02-12T01:43:53.809Z 21 Acked 1020 Write 5 Sent Done Done false
219062024-02-12T01:43:53.809Z 22 Acked 1021 Write 8 Sent Done Done false
219072024-02-12T01:43:53.809Z 23 Acked 1022 Write 8 Sent Done Done false
219082024-02-12T01:43:53.809Z 24 Acked 1023 Flush 0 Sent Done Done false
219092024-02-12T01:43:53.809Z 25 Acked 1024 Write 10 Sent Done Done false
219102024-02-12T01:43:53.810Z 26 Acked 1025 Write 6 Sent Done Done false
219112024-02-12T01:43:53.810Z 27 Acked 1026 Write 5 Sent Done Done false
219122024-02-12T01:43:53.810Z 28 Acked 1027 Write 5 Sent Done Done false
219132024-02-12T01:43:53.810Z 29 Acked 1028 Read 3 Sent Done Done false
219142024-02-12T01:43:53.810Z 30 Acked 1029 Write 5 Sent Sent Sent false
219152024-02-12T01:43:53.810Z STATES DS:0 DS:1 DS:2 TOTAL
219162024-02-12T01:43:53.810Z New 0 0 0 0
219172024-02-12T01:43:53.810Z Sent 25 1 1 27
219182024-02-12T01:43:53.810Z Done 0 24 24 48
219192024-02-12T01:43:53.810Z Skipped 0 0 0 0
219202024-02-12T01:43:53.810Z Error 0 0 0 0
219212024-02-12T01:43:53.810ZLast Flush: 1004 1023 1023
219222024-02-12T01:43:53.810ZDownstairs last five completed: 1004 1003 1002 1001 1000
219232024-02-12T01:43:53.810ZUpstairs last five completed: 30 29 28 27 26
219242024-02-12T01:43:53.810Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-12T01:43:52.666460956Z","hostname":"ip-10-150-1-24.us-west-2.compute.internal","pid":1927}
219252024-02-12T01:43:53.810ZWrote out file "/tmp/test_repair_verify.data"
219262024-02-12T01:43:53.813Z
219272024-02-12T01:43:53.816ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
219282024-02-12T01:43:53.833ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
219292024-02-12T01:43:53.833Z 0 000-019 197 198 198 348 350 350 F F F <---
219302024-02-12T01:43:53.833Z 1 020-039 194 194 194 345 345 345 F T T <---
219312024-02-12T01:43:53.833Z 2 040-059 195 198 198 346 350 350 F F F <---
219322024-02-12T01:43:53.833Z 3 060-079 194 198 198 345 350 350 F F F <---
219332024-02-12T01:43:53.833Z 4 080-099 197 198 198 348 350 350 F F F <---
219342024-02-12T01:43:53.833Z 5 100-119 195 198 198 346 350 350 F F F <---
219352024-02-12T01:43:53.833Z 6 120-139 184 184 184 329 329 329 F F F
219362024-02-12T01:43:53.833Z 7 140-159 194 194 194 345 345 345 F F F
219372024-02-12T01:43:53.833Z 8 160-179 198 198 198 349 349 349 F F F
219382024-02-12T01:43:53.834Z 9 180-199 196 196 196 347 347 347 F F F
219392024-02-12T01:43:53.834Z 10 200-219 196 198 198 347 350 350 F F F <---
219402024-02-12T01:43:53.834Z 11 220-239 195 198 198 346 350 350 F F F <---
219412024-02-12T01:43:53.834Z 12 240-259 193 193 193 344 344 344 F F F
219422024-02-12T01:43:53.834Z 13 260-279 197 197 197 348 348 348 F F F
219432024-02-12T01:43:53.837Z 14 280-299 196 196 196 347 347 347 F F F
219442024-02-12T01:43:53.837Z 15 300-319 196 196 196 347 347 347 F F F
219452024-02-12T01:43:53.837Z 16 320-339 196 196 196 347 347 347 F T T <---
219462024-02-12T01:43:53.837Z 17 340-359 195 195 195 346 346 346 F T T <---
219472024-02-12T01:43:53.837Z 18 360-379 192 192 192 343 343 343 F F F
219482024-02-12T01:43:53.837Z 19 380-399 196 196 196 347 347 347 F F F
219492024-02-12T01:43:53.837Z 20 400-419 193 193 193 344 344 344 F F F
219502024-02-12T01:43:53.837Z 21 420-439 194 194 194 345 345 345 F T T <---
219512024-02-12T01:43:53.837Z 22 440-459 191 191 191 341 341 341 F F F
219522024-02-12T01:43:53.837Z 23 460-479 186 198 198 333 350 350 F F F <---
219532024-02-12T01:43:53.837Z 24 480-499 193 193 193 344 344 344 F F F
219542024-02-12T01:43:53.838Z 25 500-519 197 197 197 348 348 348 F F F
219552024-02-12T01:43:53.838Z 26 520-539 196 196 196 347 347 347 F F F
219562024-02-12T01:43:53.838Z 27 540-559 196 198 198 347 350 350 F T T <---
219572024-02-12T01:43:53.838Z 28 560-579 196 196 196 347 347 347 F F F
219582024-02-12T01:43:53.838Z 29 580-599 196 196 196 347 347 347 F T T <---
219592024-02-12T01:43:53.838ZMax gen: 198, Max flush: 350
219602024-02-12T01:43:53.838ZError: Difference in extent metadata found!
219612024-02-12T01:43:53.838ZA Difference in extent metadata is expected here
219622024-02-12T01:43:53.838ZOn loop 99
219632024-02-12T01:43:53.838Z
219642024-02-12T01:43:53.841ZVerifying data now
219652024-02-12T01:43:58.529ZNo change after verify
219662024-02-12T01:43:58.529ZLoop: 99 Downstairs dump after verify (and repair):
219672024-02-12T01:43:58.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
219682024-02-12T01:43:58.544Z 0 000-019 198 198 198 350 350 350 F F F
219692024-02-12T01:43:58.545Z 1 020-039 199 199 199 351 351 351 F F F
219702024-02-12T01:43:58.545Z 2 040-059 198 198 198 350 350 350 F F F
219712024-02-12T01:43:58.545Z 3 060-079 198 198 198 350 350 350 F F F
219722024-02-12T01:43:58.548Z 4 080-099 198 198 198 350 350 350 F F F
219732024-02-12T01:43:58.548Z 5 100-119 198 198 198 350 350 350 F F F
219742024-02-12T01:43:58.548Z 6 120-139 184 184 184 329 329 329 F F F
219752024-02-12T01:43:58.549Z 7 140-159 194 194 194 345 345 345 F F F
219762024-02-12T01:43:58.549Z 8 160-179 198 198 198 349 349 349 F F F
219772024-02-12T01:43:58.549Z 9 180-199 196 196 196 347 347 347 F F F
219782024-02-12T01:43:58.549Z 10 200-219 198 198 198 350 350 350 F F F
219792024-02-12T01:43:58.549Z 11 220-239 198 198 198 350 350 350 F F F
219802024-02-12T01:43:58.549Z 12 240-259 193 193 193 344 344 344 F F F
219812024-02-12T01:43:58.549Z 13 260-279 197 197 197 348 348 348 F F F
219822024-02-12T01:43:58.549Z 14 280-299 196 196 196 347 347 347 F F F
219832024-02-12T01:43:58.549Z 15 300-319 196 196 196 347 347 347 F F F
219842024-02-12T01:43:58.549Z 16 320-339 199 199 199 351 351 351 F F F
219852024-02-12T01:43:58.549Z 17 340-359 199 199 199 351 351 351 F F F
219862024-02-12T01:43:58.549Z 18 360-379 192 192 192 343 343 343 F F F
219872024-02-12T01:43:58.550Z 19 380-399 196 196 196 347 347 347 F F F
219882024-02-12T01:43:58.550Z 20 400-419 193 193 193 344 344 344 F F F
219892024-02-12T01:43:58.550Z 21 420-439 199 199 199 351 351 351 F F F
219902024-02-12T01:43:58.550Z 22 440-459 191 191 191 341 341 341 F F F
219912024-02-12T01:43:58.550Z 23 460-479 198 198 198 350 350 350 F F F
219922024-02-12T01:43:58.550Z 24 480-499 193 193 193 344 344 344 F F F
219932024-02-12T01:43:58.550Z 25 500-519 197 197 197 348 348 348 F F F
219942024-02-12T01:43:58.550Z 26 520-539 196 196 196 347 347 347 F F F
219952024-02-12T01:43:58.550Z 27 540-559 199 199 199 351 351 351 F F F
219962024-02-12T01:43:58.550Z 28 560-579 196 196 196 347 347 347 F F F
219972024-02-12T01:43:58.550Z 29 580-599 199 199 199 351 351 351 F F F
219982024-02-12T01:43:58.550ZMax gen: 199, Max flush: 351
219992024-02-12T01:43:58.550Z9:46 Test duration
220002024-02-12T01:43:58.551ZTest completed
220012024-02-12T01:43:58.554Z
220022024-02-12T01:43:58.554Zreal 9:46.115648567
220032024-02-12T01:43:58.554Zuser 1:34.424405532
220042024-02-12T01:43:58.554Zsys 17.869313044
220052024-02-12T01:43:58.555Ztrap 0.085396503
220062024-02-12T01:43:58.555Ztflt 0.058246340
220072024-02-12T01:43:58.555Zdflt 0.053543067
220082024-02-12T01:43:58.555Zkflt 0.003195080
220092024-02-12T01:43:58.555Zlock 11:07:36.593409911
220102024-02-12T01:43:58.555Zslp 46:54.745861452
220112024-02-12T01:43:58.555Zlat 6.567394837
220122024-02-12T01:43:58.555Zstop 0.050299420
220132024-02-12T01:43:58.555Zprocess exited: duration 595222 ms, exit code 0
 
220142024-02-12T01:43:58.563Zfound 4 output files
220152024-02-12T01:43:58.563Zuploading: /tmp/test_repair_ds8810.txt (22086 bytes)
220162024-02-12T01:43:59.580Zuploaded: /tmp/test_repair_ds8810.txt
220172024-02-12T01:43:59.580Zuploading: /tmp/test_repair_ds8820.txt (177498 bytes)
220182024-02-12T01:44:00.599Zuploaded: /tmp/test_repair_ds8820.txt
220192024-02-12T01:44:00.599Zuploading: /tmp/test_repair_ds8830.txt (34208 bytes)
220202024-02-12T01:44:01.616Zuploaded: /tmp/test_repair_ds8830.txt
220212024-02-12T01:44:01.616Zuploading: /tmp/test_repair_out.txt (92580 bytes)
220222024-02-12T01:44:02.632Zuploaded: /tmp/test_repair_out.txt