01HQRV876V5YM89X3WSTG9HDYB: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HQRV8M2MBD5MGYXEWFC2D9KB

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-28T22:09:22.271Zjob dependencies complete; ready to run (waiting for 16 m 59 s)
22024-02-28T22:10:19.073Zjob assigned to worker 01HQRW7VG98Z5J3ARANQF1DRW4 (queued for 56 s)
32024-02-28T22:10:19.787Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-28T22:10:48.528Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-28T22:10:48.528Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-28T22:11:09.444Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-28T22:11:09.445Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-28T22:11:39.196Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-28T22:11:39.196Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-28T22:11:52.791Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-28T22:11:52.807Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-28T22:11:52.825Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-28T22:11:52.826Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-28T22:11:53.265Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-28T22:11:53.265Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-28T22:11:53.292Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-28T22:11:53.292Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-28T22:11:53.321Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-28T22:11:53.321Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-28T22:11:53.969Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-28T22:11:53.969Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-28T22:11:54.129Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-28T22:11:54.129Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-28T22:11:54.148Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-28T22:11:54.157Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-28T22:12:41.857Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-28T22:12:41.858Zstarting task 0: "setup"
282024-02-28T22:12:42.056Z++ uname -s
292024-02-28T22:12:42.069Z+ kern=SunOS
302024-02-28T22:12:42.069Z+ case "$kern" in
312024-02-28T22:12:42.069Z+ groupadd -g 12345 build
322024-02-28T22:12:42.069Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-28T22:12:44.073Z+ zfs create -o mountpoint=/work rpool/work
342024-02-28T22:12:44.509Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-28T22:12:44.509Z+ home_fs=zfs
362024-02-28T22:12:44.509Z+ [[ zfs == autofs ]]
372024-02-28T22:12:44.509Z+ mkdir -p /home/build
382024-02-28T22:12:44.509Z+ chown build:build /home/build /work
392024-02-28T22:12:45.194Z+ chmod 0700 /home/build /work
402024-02-28T22:12:45.195Zprocess exited: duration 3332 ms, exit code 0
 
412024-02-28T22:12:45.215Zstarting task 1: "authentication"
422024-02-28T22:12:45.236Zprocess exited: duration 24 ms, exit code 0
 
432024-02-28T22:12:45.269Zstarting task 2: "build"
442024-02-28T22:12:45.298Z+ banner cores
452024-02-28T22:12:45.315Z
462024-02-28T22:12:45.315Z #### #### ##### ###### ####
472024-02-28T22:12:45.315Z # # # # # # # #
482024-02-28T22:12:45.316Z # # # # # ##### ####
492024-02-28T22:12:45.316Z # # # ##### # #
502024-02-28T22:12:45.316Z # # # # # # # # #
512024-02-28T22:12:45.316Z #### #### # # ###### ####
522024-02-28T22:12:45.316Z
532024-02-28T22:12:45.339Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-28T22:12:45.339Z+ echo 'input bins dir contains:'
552024-02-28T22:12:45.339Zinput bins dir contains:
562024-02-28T22:12:45.339Z+ ls -ltr /input/build/work/bins
572024-02-28T22:12:45.339Ztotal 740779
582024-02-28T22:12:45.340Z-rw-r--r-- 1 root root 109740032 Feb 28 22:10 crucible-downstairs.gz
592024-02-28T22:12:45.340Z-rw-r--r-- 1 root root 91529491 Feb 28 22:11 crucible-hammer.gz
602024-02-28T22:12:45.340Z-rw-r--r-- 1 root root 105639604 Feb 28 22:11 crutest.gz
612024-02-28T22:12:45.340Z-rw-r--r-- 1 root root 72121864 Feb 28 22:11 dsc.gz
622024-02-28T22:12:45.340Z+ echo 'input script dir contains:'
632024-02-28T22:12:45.340Zinput script dir contains:
642024-02-28T22:12:45.341Z+ ls -ltr /input/build/work/scripts
652024-02-28T22:12:45.341Ztotal 39
662024-02-28T22:12:45.341Z-rw-r--r-- 1 root root 3056 Feb 28 22:11 perf-downstairs-tick.d
672024-02-28T22:12:45.341Z-rw-r--r-- 1 root root 1738 Feb 28 22:11 test_ds.sh
682024-02-28T22:12:45.342Z-rw-r--r-- 1 root root 3746 Feb 28 22:11 test_live_repair.sh
692024-02-28T22:12:45.342Z-rw-r--r-- 1 root root 6416 Feb 28 22:11 test_repair.sh
702024-02-28T22:12:45.342Z-rw-r--r-- 1 root root 2969 Feb 28 22:11 test_replay.sh
712024-02-28T22:12:45.343Z-rw-r--r-- 1 root root 9465 Feb 28 22:11 test_up.sh
722024-02-28T22:12:45.343Z-rw-r--r-- 1 root root 2824 Feb 28 22:11 upstairs_info.d
732024-02-28T22:12:45.343Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-28T22:12:45.343Z+ echo ' chmod input script dir contains:'
752024-02-28T22:12:45.343Z chmod input script dir contains:
762024-02-28T22:12:45.344Z+ ls -ltr /input/build/work/scripts
772024-02-28T22:12:45.366Ztotal 39
782024-02-28T22:12:45.366Z-rwxr-xr-x 1 root root 3056 Feb 28 22:11 perf-downstairs-tick.d
792024-02-28T22:12:45.366Z-rwxr-xr-x 1 root root 1738 Feb 28 22:11 test_ds.sh
802024-02-28T22:12:45.366Z-rwxr-xr-x 1 root root 3746 Feb 28 22:11 test_live_repair.sh
812024-02-28T22:12:45.367Z-rwxr-xr-x 1 root root 6416 Feb 28 22:11 test_repair.sh
822024-02-28T22:12:45.367Z-rwxr-xr-x 1 root root 2969 Feb 28 22:11 test_replay.sh
832024-02-28T22:12:45.367Z-rwxr-xr-x 1 root root 9465 Feb 28 22:11 test_up.sh
842024-02-28T22:12:45.367Z-rwxr-xr-x 1 root root 2824 Feb 28 22:11 upstairs_info.d
852024-02-28T22:12:45.368Z+ banner unpack
862024-02-28T22:12:45.368Z
872024-02-28T22:12:45.368Z # # # # ##### ## #### # #
882024-02-28T22:12:45.368Z # # ## # # # # # # # # #
892024-02-28T22:12:45.368Z # # # # # # # # # # ####
902024-02-28T22:12:45.369Z # # # # # ##### ###### # # #
912024-02-28T22:12:45.369Z # # # ## # # # # # # #
922024-02-28T22:12:45.369Z #### # # # # # #### # #
932024-02-28T22:12:45.369Z
942024-02-28T22:12:45.369Z+ mkdir -p /var/tmp/bins
952024-02-28T22:12:45.370Z+ for t in "$input/bins/"*.gz
962024-02-28T22:12:45.370Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-28T22:12:45.370Z+ b=crucible-downstairs.gz
982024-02-28T22:12:45.370Z+ b=crucible-downstairs
992024-02-28T22:12:45.370Z+ gunzip
1002024-02-28T22:12:47.928Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-28T22:12:47.931Z+ for t in "$input/bins/"*.gz
1022024-02-28T22:12:47.931Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-28T22:12:47.935Z+ b=crucible-hammer.gz
1042024-02-28T22:12:47.935Z+ b=crucible-hammer
1052024-02-28T22:12:47.935Z+ gunzip
1062024-02-28T22:12:50.073Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-28T22:12:50.104Z+ for t in "$input/bins/"*.gz
1082024-02-28T22:12:50.104Z++ basename /input/build/work/bins/crutest.gz
1092024-02-28T22:12:50.105Z+ b=crutest.gz
1102024-02-28T22:12:50.105Z+ b=crutest
1112024-02-28T22:12:50.105Z+ gunzip
1122024-02-28T22:12:52.533Z+ chmod +x /var/tmp/bins/crutest
1132024-02-28T22:12:52.544Z+ for t in "$input/bins/"*.gz
1142024-02-28T22:12:52.545Z++ basename /input/build/work/bins/dsc.gz
1152024-02-28T22:12:52.545Z+ b=dsc.gz
1162024-02-28T22:12:52.545Z+ b=dsc
1172024-02-28T22:12:52.545Z+ gunzip
1182024-02-28T22:12:54.185Z+ chmod +x /var/tmp/bins/dsc
1192024-02-28T22:12:54.198Z+ export BINDIR=/var/tmp/bins
1202024-02-28T22:12:54.198Z+ BINDIR=/var/tmp/bins
1212024-02-28T22:12:54.198Z+ banner repair
1222024-02-28T22:12:54.199Z
1232024-02-28T22:12:54.199Z ##### ###### ##### ## # #####
1242024-02-28T22:12:54.199Z # # # # # # # # # #
1252024-02-28T22:12:54.199Z # # ##### # # # # # # #
1262024-02-28T22:12:54.199Z ##### # ##### ###### # #####
1272024-02-28T22:12:54.199Z # # # # # # # # #
1282024-02-28T22:12:54.199Z # # ###### # # # # # #
1292024-02-28T22:12:54.199Z
1302024-02-28T22:12:54.200Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-28T22:12:54.275ZTurn off color for downstairs dump
1322024-02-28T22:12:54.289ZCreating region directory at: /var/tmp/test_repair
1332024-02-28T22:12:54.292ZCreating dsc directory at: /tmp/dsc
1342024-02-28T22:12:54.402ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.10151099
1352024-02-28T22:12:54.454ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.049274273
1362024-02-28T22:12:54.488ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.041225657
1372024-02-28T22:12:54.488ZRegion set with 3 regions was created
1382024-02-28T22:12:54.507Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1137 1138 1139
1392024-02-28T22:12:54.508ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-28T22:12:54.530Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.422366621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
1412024-02-28T22:12:54.536Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423170589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
1422024-02-28T22:12:54.536Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423218888Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
1432024-02-28T22:12:54.537Z{"msg":"Crucible 8527f1a5-2f18-4300-8d91-d3d3f8a2b8b0 has session id: f63fb890-beac-490a-a331-2bb4ecd1b2e1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423245487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1442024-02-28T22:12:54.537Z{"msg":"Upstairs opts: Upstairs UUID: 8527f1a5-2f18-4300-8d91-d3d3f8a2b8b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423275996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1452024-02-28T22:12:54.537Z{"msg":"Crucible stats registered with UUID: 8527f1a5-2f18-4300-8d91-d3d3f8a2b8b0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423298976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1462024-02-28T22:12:54.537ZCrucible runtime is spawned
1472024-02-28T22:12:54.537Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423556839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
1482024-02-28T22:12:54.537Z{"msg":"8527f1a5-2f18-4300-8d91-d3d3f8a2b8b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423739244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1492024-02-28T22:12:54.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423767063Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1502024-02-28T22:12:54.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423789912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1512024-02-28T22:12:54.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423825501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1522024-02-28T22:12:54.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423850381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1532024-02-28T22:12:54.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42386452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1542024-02-28T22:12:54.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42387924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1552024-02-28T22:12:54.537Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.423983787Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1562024-02-28T22:12:54.538Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.424015556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1572024-02-28T22:12:54.538Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.424040805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1582024-02-28T22:12:54.538Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42422722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1592024-02-28T22:12:54.538Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.424278839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1602024-02-28T22:12:54.538Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.424310868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1612024-02-28T22:12:54.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426110838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1622024-02-28T22:12:54.538Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426188736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1632024-02-28T22:12:54.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426222625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1642024-02-28T22:12:54.538Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426260684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1652024-02-28T22:12:54.538Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426278574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1662024-02-28T22:12:54.538Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.426292653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1672024-02-28T22:12:54.538Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.427935568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1682024-02-28T22:12:54.538Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.427981247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1692024-02-28T22:12:54.538Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428060365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1702024-02-28T22:12:54.538Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428839163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1712024-02-28T22:12:54.538Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428878432Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1722024-02-28T22:12:54.539Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428894562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1732024-02-28T22:12:54.539Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428915361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1742024-02-28T22:12:54.539Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42894579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1752024-02-28T22:12:54.539Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42896176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1762024-02-28T22:12:54.539Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.428993019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1772024-02-28T22:12:54.539Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429022628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1782024-02-28T22:12:54.539Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429194464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1792024-02-28T22:12:54.539Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429233122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1802024-02-28T22:12:54.539Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429265882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1812024-02-28T22:12:54.539Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429290311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1822024-02-28T22:12:54.539Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42930573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1832024-02-28T22:12:54.539Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42932863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1842024-02-28T22:12:54.539Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429343079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1852024-02-28T22:12:54.539Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429364179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1862024-02-28T22:12:54.540Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429378858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1872024-02-28T22:12:54.540Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429392948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1882024-02-28T22:12:54.540Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429406798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1892024-02-28T22:12:54.540Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429427677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1902024-02-28T22:12:54.540Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429442447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"downstairs","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1912024-02-28T22:12:54.540Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429456026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1922024-02-28T22:12:54.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429482596Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"Wait for a query_work_queue command to finish before sending IO
1932024-02-28T22:12:54.540Z}
1942024-02-28T22:12:54.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429571443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1952024-02-28T22:12:54.540Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429600372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"client":"2","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1962024-02-28T22:12:54.540Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429624661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1972024-02-28T22:12:54.540Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429656581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1982024-02-28T22:12:54.540Z{"msg":"8527f1a5-2f18-4300-8d91-d3d3f8a2b8b0 is now active with session: f63fb890-beac-490a-a331-2bb4ecd1b2e1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.42967567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
1992024-02-28T22:12:54.540Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429705989Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2002024-02-28T22:12:54.540Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:53.429720629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
2012024-02-28T22:12:54.540ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-28T22:12:54.541ZFill test
2032024-02-28T22:12:54.577ZRead and Verify all blocks (0..600 range:false)
2042024-02-28T22:12:54.647ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-28T22:12:54.647ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-28T22:12:54.647Z----------------------------------------------------------------
2072024-02-28T22:12:54.647Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-28T22:12:54.647ZStates: Active Active Active
2092024-02-28T22:12:54.647ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-28T22:12:54.648Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-28T22:12:54.648Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-28T22:12:54.648Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-28T22:12:54.648Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-28T22:12:54.648Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-28T22:12:54.648Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-28T22:12:54.648Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-28T22:12:54.649Z New 0 0 0 0
2182024-02-28T22:12:54.664Z Sent 0 0 0 0
2192024-02-28T22:12:54.664Z Done 6 6 6 18
2202024-02-28T22:12:54.664Z Skipped 0 0 0 0
2212024-02-28T22:12:54.664Z Error 0 0 0 0
2222024-02-28T22:12:54.665ZLast Flush: 1006 1006 1006
2232024-02-28T22:12:54.665ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-28T22:12:54.665ZUpstairs last five completed: 13 12 11 10 9
2252024-02-28T22:12:54.665ZCLIENT: Up:0 ds:6 act:3
2262024-02-28T22:12:58.653Z----------------------------------------------------------------
2272024-02-28T22:12:58.653Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-28T22:12:58.653ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-28T22:12:58.654ZUpstairs last five completed: 14 13 12 11 10
2302024-02-28T22:12:58.654ZCLIENT: Up:0 ds:0 act:3
2312024-02-28T22:12:58.654ZCLIENT: All crucible jobs finished, exiting program
2322024-02-28T22:12:58.674Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.549879203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141}
2332024-02-28T22:12:58.674Z{"msg":"client request queue closed unexpectedly; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.549946341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2342024-02-28T22:12:58.675Z{"msg":"client_stop_rx closed unexpectedly: RecvError(())","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.550096717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2352024-02-28T22:12:58.675Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.550142686Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2362024-02-28T22:12:58.675Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.550171925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2372024-02-28T22:12:58.675Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.550226004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"0","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2382024-02-28T22:12:58.676Z{"msg":"client task is sending Done(QueueClosed)","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.550253403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2392024-02-28T22:12:58.676Z{"msg":"client task could not reply to main task; shutting down?","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.550278472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2402024-02-28T22:12:58.676Z{"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.550302732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1141,"":"io task","client":"1","":"downstairs","session_id":"f63fb890-beac-490a-a331-2bb4ecd1b2e1"}
2412024-02-28T22:12:58.676Z
2422024-02-28T22:12:58.676ZBegin loop 1
2432024-02-28T22:12:58.676ZDownstairs to restart: 0
2442024-02-28T22:12:58.720Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.573533701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
2452024-02-28T22:12:58.720Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"Crucible runtime is spawned
2462024-02-28T22:12:58.720Zcrucible","level":30,"time":"2024-02-28T22:12:57.573897621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
2472024-02-28T22:12:58.720Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.57394409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
2482024-02-28T22:12:58.721Z{"msg":"Crucible 4d57f746-be8c-4192-8564-5f6785711f8f has session id: 6052d394-26b9-4a71-bafe-3b5a534e792e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.573988008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2492024-02-28T22:12:58.721Z{"msg":"Upstairs opts: Upstairs UUID: 4d57f746-be8c-4192-8564-5f6785711f8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574021307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2502024-02-28T22:12:58.721Z{"msg":"Crucible stats registered with UUID: 4d57f746-be8c-4192-8564-5f6785711f8f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574062786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2512024-02-28T22:12:58.721Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574081906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
2522024-02-28T22:12:58.721Z{"msg":"4d57f746-be8c-4192-8564-5f6785711f8f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574141754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2532024-02-28T22:12:58.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574193533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2542024-02-28T22:12:58.721Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574214812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2552024-02-28T22:12:58.721Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574231402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2562024-02-28T22:12:58.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574269501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2572024-02-28T22:12:58.722Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.5743025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2582024-02-28T22:12:58.722Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574341809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2592024-02-28T22:12:58.722Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574359388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2602024-02-28T22:12:58.722Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574403697Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2612024-02-28T22:12:58.723Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574437936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2622024-02-28T22:12:58.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574475365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2632024-02-28T22:12:58.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574512164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2642024-02-28T22:12:58.723Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.574543563Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"io task","client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2652024-02-28T22:12:58.723Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575325492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2662024-02-28T22:12:58.723Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575364241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2672024-02-28T22:12:58.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.5754058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2682024-02-28T22:12:58.724Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575441908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2692024-02-28T22:12:58.724Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575480747Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2702024-02-28T22:12:58.724Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575509806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2712024-02-28T22:12:58.725Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575952874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2722024-02-28T22:12:58.725Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.575991303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2732024-02-28T22:12:58.725Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576264466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2742024-02-28T22:12:58.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576555408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2752024-02-28T22:12:58.725Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576602016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2762024-02-28T22:12:58.725Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576637826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2772024-02-28T22:12:58.725Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576669375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2782024-02-28T22:12:58.726Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576707553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2792024-02-28T22:12:58.726Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576733583Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2802024-02-28T22:12:58.726Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576793671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2812024-02-28T22:12:58.726Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.57682458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2822024-02-28T22:12:58.726Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.57684555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2832024-02-28T22:12:58.727Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576875079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2842024-02-28T22:12:58.727Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576891919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2852024-02-28T22:12:58.727Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576914118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2862024-02-28T22:12:58.727Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576928307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2872024-02-28T22:12:58.727Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576942327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2882024-02-28T22:12:58.727Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576967617Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2892024-02-28T22:12:58.728Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.576983646Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2902024-02-28T22:12:58.728ZWait for a query_work_queue command to finish before sending IO
2912024-02-28T22:12:58.728Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577026975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2922024-02-28T22:12:58.728Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577050044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2932024-02-28T22:12:58.728Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577064614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2942024-02-28T22:12:58.728Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577079153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2952024-02-28T22:12:58.729Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577093383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"":"downstairs","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2962024-02-28T22:12:58.729Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577119852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2972024-02-28T22:12:58.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577135192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"0","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2982024-02-28T22:12:58.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577169081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"1","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
2992024-02-28T22:12:58.729Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.57720456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"client":"2","":"downstairs","session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
3002024-02-28T22:12:58.729Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577237629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
3012024-02-28T22:12:58.729Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577271578Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
3022024-02-28T22:12:58.729Z{"msg":"4d57f746-be8c-4192-8564-5f6785711f8f is now active with session: 6052d394-26b9-4a71-bafe-3b5a534e792e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577288028Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
3032024-02-28T22:12:58.729Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577311007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143,"session_id":"6052d394-26b9-4a71-bafe-3b5a534e792e"}
3042024-02-28T22:12:58.730Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:57.577343256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
3052024-02-28T22:12:58.730ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3062024-02-28T22:12:58.730ZLoading write count information from file "/tmp/test_repair_verify.data"
3072024-02-28T22:12:58.730ZRun Repair workload
3082024-02-28T22:12:58.730Z01/30 Flush
3092024-02-28T22:12:58.730Z02/30 Flush
3102024-02-28T22:12:58.731Z03/30 Write block 45 len 12288 data: 2 2 2
3112024-02-28T22:12:58.731Z04/30 Read block 34 len 8192
3122024-02-28T22:12:58.731Z05/30 Read block 379 len 32768
3132024-02-28T22:12:58.731Z06/30 Read block 491 len 28672
3142024-02-28T22:12:58.731Z07/30 Write block 108 len 4096 data: 2
3152024-02-28T22:12:58.731Z08/30 Write block 238 len 16384 data: 2 2 2 2
3162024-02-28T22:12:58.731Z09/30 Write block 313 len 20480 data: 2 2 2 2 2
3172024-02-28T22:12:58.731Z10/30 Write block 395 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3182024-02-28T22:12:58.732Z11/30 Write block 420 len 24576 data: 2 2 2 2 2 2
3192024-02-28T22:12:58.732Z12/30 Read block 327 len 36864
3202024-02-28T22:12:58.732Z13/30 Read block 497 len 28672
3212024-02-28T22:12:58.732Z14/30 Write block 103 len 28672 data: 2 2 2 2 2 3 2
3222024-02-28T22:12:58.732Z15/30 Read block 156 len 40960
3232024-02-28T22:12:58.732Z16/30 Flush
3242024-02-28T22:12:58.732Z17/30 Write block 232 len 4096 data: 2
3252024-02-28T22:12:58.732Z18/30 Read block 491 len 36864
3262024-02-28T22:12:58.732Z19/30 Read block 557 len 40960
3272024-02-28T22:12:58.733Z20/30 Write block 56 len 28672 data: 2 2 2 2 2 2 2
3282024-02-28T22:12:59.207Z21/30 Write block 495 len 28672 data: 2 2 2 2 2 2 2
3292024-02-28T22:12:59.207Z22/30 Write block 384 len 36864 data: 2 2 2 2 2 2 2 2 2
3302024-02-28T22:12:59.208Z23/30 Write block 226 len 28672 data: 2 2 2 2 2 2 3
3312024-02-28T22:12:59.208Z24/30 Read block 586 len 32768
3322024-02-28T22:12:59.208Z25/30 Read block 39 len 40960
3332024-02-28T22:12:59.208Z26/30 Read block 492 len 40960
3342024-02-28T22:12:59.208Z27/30 Write block 451 len 36864 data: 2 2 2 2 2 2 2 2 2
3352024-02-28T22:12:59.208Z28/30 Read block 24 len 8192
3362024-02-28T22:12:59.209Z29/30 Write block 45 len 20480 data: 3 3 3 2 2
3372024-02-28T22:12:59.209Z30/30 Write block 533 len 8192 data: 2 2
3382024-02-28T22:12:59.209Z----------------------------------------------------------------
3392024-02-28T22:12:59.209Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:28
3402024-02-28T22:12:59.209ZStates: Active Active Active
3412024-02-28T22:12:59.209ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3422024-02-28T22:12:59.209Z 3 Acked 1002 Write 3 Done Done Done false
3432024-02-28T22:12:59.210Z 4 Acked 1003 Read 2 Done Done Done false
3442024-02-28T22:12:59.210Z 5 Acked 1004 Read 8 Done Done Done false
3452024-02-28T22:12:59.210Z 6 Acked 1005 Read 7 Sent Done Done false
3462024-02-28T22:12:59.210Z 7 Acked 1006 Write 1 Sent Done Done false
3472024-02-28T22:12:59.210Z 8 Acked 1007 Write 4 Sent Done Done false
3482024-02-28T22:12:59.210Z 9 Acked 1008 Write 5 Sent Done Done false
3492024-02-28T22:12:59.210Z 10 Acked 1009 Write 10 Sent Done Done false
3502024-02-28T22:12:59.210Z 11 Acked 1010 Write 6 Sent Done Done false
3512024-02-28T22:12:59.211Z 12 Acked 1011 Read 9 Sent Done Done false
3522024-02-28T22:12:59.211Z 13 Acked 1012 Read 7 Sent Done Done false
3532024-02-28T22:12:59.211Z 14 Acked 1013 Write 7 Sent Done Done false
3542024-02-28T22:12:59.211Z 15 Acked 1014 Read 10 Sent Done Done false
3552024-02-28T22:12:59.211Z 16 Acked 1015 Flush 0 Sent Done Done false
3562024-02-28T22:12:59.211Z 17 Acked 1016 Write 1 Sent Done Done false
3572024-02-28T22:12:59.211Z 18 Acked 1017 Read 9 Sent Done Done false
3582024-02-28T22:12:59.212Z 19 Acked 1018 Read 10 Sent Done Done false
3592024-02-28T22:12:59.212Z 20 Acked 1019 Write 7 Sent Done Done false
3602024-02-28T22:12:59.212Z 21 Acked 1020 Write 7 Sent Done Done false
3612024-02-28T22:12:59.212Z 22 Acked 1021 Write 9 Sent Done Done false
3622024-02-28T22:12:59.212Z 23 Acked 1022 Write 7 Sent Done Done false
3632024-02-28T22:12:59.212Z 24 Acked 1023 Read 8 Sent Done Done false
3642024-02-28T22:12:59.212Z 25 Acked 1024 Read 10 Sent Done Done false
3652024-02-28T22:12:59.212Z 26 Acked 1025 Read 10 Sent Done Done false
3662024-02-28T22:12:59.213Z 27 Acked 1026 Write 9 Sent Done Done false
3672024-02-28T22:12:59.213Z 28 Acked 1027 Read 2 Sent Done Done false
3682024-02-28T22:12:59.213Z 29 Acked 1028 Write 5 Sent Sent Sent false
3692024-02-28T22:12:59.213Z 30 Acked 1029 Write 2 Sent Sent Sent false
3702024-02-28T22:12:59.213Z STATES DS:0 DS:1 DS:2 TOTAL
3712024-02-28T22:12:59.213Z New 0 0 0 0
3722024-02-28T22:12:59.214Z Sent 25 2 2 29
3732024-02-28T22:12:59.214Z Done 3 26 26 55
3742024-02-28T22:12:59.214Z Skipped 0 0 0 0
3752024-02-28T22:12:59.214Z Error 0 0 0 0
3762024-02-28T22:12:59.214ZLast Flush: 1001 1015 1015
3772024-02-28T22:12:59.214ZDownstairs last five completed: 1001 1000
3782024-02-28T22:12:59.214ZUpstairs last five completed: 30 29 28 27 26
3792024-02-28T22:12:59.214Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:12:57.608093939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1143}
3802024-02-28T22:12:59.215ZWrote out file "/tmp/test_repair_verify.data"
3812024-02-28T22:12:59.215Z
3822024-02-28T22:12:59.215ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3832024-02-28T22:12:59.215ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3842024-02-28T22:12:59.215Z 0 000-019 1 1 1 1 1 1 F F F
3852024-02-28T22:12:59.215Z 1 020-039 1 1 1 1 1 1 F F F
3862024-02-28T22:12:59.215Z 2 040-059 1 2 2 1 4 4 T T T <---
3872024-02-28T22:12:59.215Z 3 060-079 1 1 1 1 1 1 F T T <---
3882024-02-28T22:12:59.216Z 4 080-099 1 1 1 1 1 1 F F F
3892024-02-28T22:12:59.216Z 5 100-119 1 2 2 1 4 4 F F F <---
3902024-02-28T22:12:59.216Z 6 120-139 1 1 1 1 1 1 F F F
3912024-02-28T22:12:59.216Z 7 140-159 1 1 1 1 1 1 F F F
3922024-02-28T22:12:59.216Z 8 160-179 1 1 1 1 1 1 F F F
3932024-02-28T22:12:59.216Z 9 180-199 1 1 1 1 1 1 F F F
3942024-02-28T22:12:59.216Z 10 200-219 1 1 1 1 1 1 F F F
3952024-02-28T22:12:59.217Z 11 220-239 1 2 2 1 4 4 F T T <---
3962024-02-28T22:12:59.217Z 12 240-259 1 2 2 1 4 4 F F F <---
3972024-02-28T22:12:59.217Z 13 260-279 1 1 1 1 1 1 F F F
3982024-02-28T22:12:59.217Z 14 280-299 1 1 1 1 1 1 F F F
3992024-02-28T22:12:59.217Z 15 300-319 1 2 2 1 4 4 F F F <---
4002024-02-28T22:12:59.217Z 16 320-339 1 1 1 1 1 1 F F F
4012024-02-28T22:12:59.217Z 17 340-359 1 1 1 1 1 1 F F F
4022024-02-28T22:12:59.218Z 18 360-379 1 1 1 1 1 1 F F F
4032024-02-28T22:12:59.218Z 19 380-399 1 2 2 1 4 4 F T T <---
4042024-02-28T22:12:59.218Z 20 400-419 1 2 2 1 4 4 F F F <---
4052024-02-28T22:12:59.218Z 21 420-439 1 2 2 1 4 4 F F F <---
4062024-02-28T22:12:59.218Z 22 440-459 1 1 1 1 1 1 F T T <---
4072024-02-28T22:12:59.218Z 23 460-479 1 1 1 1 1 1 F F F
4082024-02-28T22:12:59.218Z 24 480-499 1 1 1 1 1 1 F T T <---
4092024-02-28T22:12:59.218Z 25 500-519 1 1 1 1 1 1 F T T <---
4102024-02-28T22:12:59.219Z 26 520-539 1 1 1 1 1 1 F T T <---
4112024-02-28T22:12:59.219Z 27 540-559 1 1 1 1 1 1 F F F
4122024-02-28T22:12:59.219Z 28 560-579 1 1 1 1 1 1 F F F
4132024-02-28T22:12:59.219Z 29 580-599 1 1 1 1 1 1 F F F
4142024-02-28T22:12:59.219ZMax gen: 2, Max flush: 4
4152024-02-28T22:12:59.219ZError: Difference in extent metadata found!
4162024-02-28T22:12:59.219ZA Difference in extent metadata is expected here
4172024-02-28T22:12:59.220ZOn loop 1
4182024-02-28T22:12:59.220Z
4192024-02-28T22:12:59.220ZVerifying data now
4202024-02-28T22:13:05.227ZNo change after verify
4212024-02-28T22:13:05.228ZLoop: 1 Downstairs dump after verify (and repair):
4222024-02-28T22:13:05.251ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4232024-02-28T22:13:05.251Z 0 000-019 1 1 1 1 1 1 F F F
4242024-02-28T22:13:05.251Z 1 020-039 1 1 1 1 1 1 F F F
4252024-02-28T22:13:05.252Z 2 040-059 3 3 3 5 5 5 F F F
4262024-02-28T22:13:05.252Z 3 060-079 3 3 3 5 5 5 F F F
4272024-02-28T22:13:05.252Z 4 080-099 1 1 1 1 1 1 F F F
4282024-02-28T22:13:05.252Z 5 100-119 2 2 2 4 4 4 F F F
4292024-02-28T22:13:05.252Z 6 120-139 1 1 1 1 1 1 F F F
4302024-02-28T22:13:05.252Z 7 140-159 1 1 1 1 1 1 F F F
4312024-02-28T22:13:05.252Z 8 160-179 1 1 1 1 1 1 F F F
4322024-02-28T22:13:05.252Z 9 180-199 1 1 1 1 1 1 F F F
4332024-02-28T22:13:05.253Z 10 200-219 1 1 1 1 1 1 F F F
4342024-02-28T22:13:05.253Z 11 220-239 3 3 3 5 5 5 F F F
4352024-02-28T22:13:05.253Z 12 240-259 2 2 2 4 4 4 F F F
4362024-02-28T22:13:05.253Z 13 260-279 1 1 1 1 1 1 F F F
4372024-02-28T22:13:05.253Z 14 280-299 1 1 1 1 1 1 F F F
4382024-02-28T22:13:05.253Z 15 300-319 2 2 2 4 4 4 F F F
4392024-02-28T22:13:05.254Z 16 320-339 1 1 1 1 1 1 F F F
4402024-02-28T22:13:05.254Z 17 340-359 1 1 1 1 1 1 F F F
4412024-02-28T22:13:05.254Z 18 360-379 1 1 1 1 1 1 F F F
4422024-02-28T22:13:05.254Z 19 380-399 3 3 3 5 5 5 F F F
4432024-02-28T22:13:05.254Z 20 400-419 2 2 2 4 4 4 F F F
4442024-02-28T22:13:05.254Z 21 420-439 2 2 2 4 4 4 F F F
4452024-02-28T22:13:05.254Z 22 440-459 3 3 3 5 5 5 F F F
4462024-02-28T22:13:05.254Z 23 460-479 1 1 1 1 1 1 F F F
4472024-02-28T22:13:05.254Z 24 480-499 3 3 3 5 5 5 F F F
4482024-02-28T22:13:05.255Z 25 500-519 3 3 3 5 5 5 F F F
4492024-02-28T22:13:05.255Z 26 520-539 3 3 3 5 5 5 F F F
4502024-02-28T22:13:05.255Z 27 540-559 1 1 1 1 1 1 F F F
4512024-02-28T22:13:05.255Z 28 560-579 1 1 1 1 1 1 F F F
4522024-02-28T22:13:05.255Z 29 580-599 1 1 1 1 1 1 F F F
4532024-02-28T22:13:05.255ZMax gen: 3, Max flush: 5
4542024-02-28T22:13:05.255Z
4552024-02-28T22:13:05.255ZBegin loop 2
4562024-02-28T22:13:05.255ZDownstairs to restart: 2
4572024-02-28T22:13:05.278Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156306002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
4582024-02-28T22:13:05.278Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156582055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
4592024-02-28T22:13:05.278Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156616494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
4602024-02-28T22:13:05.278Z{"msg":"Crucible runtime is spawned
4612024-02-28T22:13:05.279ZCrucible 32e4e4db-a4ca-4e7f-aaac-afba557e1510 has session id: f899ad01-55a0-4d86-94f0-ad405fcdcb1d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156649073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4622024-02-28T22:13:05.279Z{"msg":"Upstairs opts: Upstairs UUID: 32e4e4db-a4ca-4e7f-aaac-afba557e1510, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156693522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4632024-02-28T22:13:05.279Z{"msg":"Crucible stats registered with UUID: 32e4e4db-a4ca-4e7f-aaac-afba557e1510","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156719111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4642024-02-28T22:13:05.279Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.15674078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
4652024-02-28T22:13:05.279Z{"msg":"32e4e4db-a4ca-4e7f-aaac-afba557e1510 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156792179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4662024-02-28T22:13:05.279Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156821778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4672024-02-28T22:13:05.279Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156847817Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4682024-02-28T22:13:05.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156883936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4692024-02-28T22:13:05.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156911786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4702024-02-28T22:13:05.280Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156946765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4712024-02-28T22:13:05.280Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.156972694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4722024-02-28T22:13:05.280Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157054082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4732024-02-28T22:13:05.280Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157100921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4742024-02-28T22:13:05.281Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157139149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4752024-02-28T22:13:05.281Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157214387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4762024-02-28T22:13:05.281Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157253206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4772024-02-28T22:13:05.281Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157280845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"io task","client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4782024-02-28T22:13:05.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157917618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4792024-02-28T22:13:05.282Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.157965777Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4802024-02-28T22:13:05.282Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158007345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4812024-02-28T22:13:05.282Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158052774Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4822024-02-28T22:13:05.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158105073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4832024-02-28T22:13:05.283Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158149452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4842024-02-28T22:13:05.283Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158514022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4852024-02-28T22:13:05.284Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158726206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4862024-02-28T22:13:05.284Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.158768964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4872024-02-28T22:13:05.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159041007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4882024-02-28T22:13:05.284Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159082046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4892024-02-28T22:13:05.284Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159109395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4902024-02-28T22:13:05.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159246171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4912024-02-28T22:13:05.284Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.15928197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4922024-02-28T22:13:05.284Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.15929758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4932024-02-28T22:13:05.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159350728Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4942024-02-28T22:13:05.284Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159365598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4952024-02-28T22:13:05.285Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 5, 5, 1, 4, 1, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159387047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4962024-02-28T22:13:05.285Z{"msg":"[0]R generation[0..12]: [1, 1, 3, 3, 1, 2, 1, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159401247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4972024-02-28T22:13:05.285Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159423437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4982024-02-28T22:13:05.285Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 5, 5, 1, 4, 1, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159437676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
4992024-02-28T22:13:05.285Z{"msg":"[1]R generation[0..12]: [1, 1, 3, 3, 1, 2, 1, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159459096Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5002024-02-28T22:13:05.285Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159474025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5012024-02-28T22:13:05.285ZWait for a query_work_queue command to finish before sending IO
5022024-02-28T22:13:05.285Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 5, 5, 1, 4, 1, 1, 1, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159501294Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5032024-02-28T22:13:05.285Z{"msg":"[2]R generation[0..12]: [1, 1, 3, 3, 1, 2, 1, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159522194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5042024-02-28T22:13:05.286Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159544653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5052024-02-28T22:13:05.286Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159559633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5062024-02-28T22:13:05.286Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159574262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5072024-02-28T22:13:05.286Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159594982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5082024-02-28T22:13:05.287Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159608671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"":"downstairs","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5092024-02-28T22:13:05.287Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159633291Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5102024-02-28T22:13:05.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.15964843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"0","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5112024-02-28T22:13:05.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.15966281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"1","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5122024-02-28T22:13:05.287Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159676759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"client":"2","":"downstairs","session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5132024-02-28T22:13:05.287Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159690179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5142024-02-28T22:13:05.287Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159703099Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5152024-02-28T22:13:05.287Z{"msg":"32e4e4db-a4ca-4e7f-aaac-afba557e1510 is now active with session: f899ad01-55a0-4d86-94f0-ad405fcdcb1d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159716298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5162024-02-28T22:13:05.287Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159728928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151,"session_id":"f899ad01-55a0-4d86-94f0-ad405fcdcb1d"}
5172024-02-28T22:13:05.288Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:04.159742108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
5182024-02-28T22:13:05.288ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5192024-02-28T22:13:05.288ZLoading write count information from file "/tmp/test_repair_verify.data"
5202024-02-28T22:13:05.288ZRun Repair workload
5212024-02-28T22:13:05.288Z01/30 Write block 165 len 40960 data: 2 2 2 2 2 2 2 2 2 2
5222024-02-28T22:13:05.288Z02/30 Flush
5232024-02-28T22:13:05.302Z03/30 Write block 500 len 36864 data: 3 3 2 2 2 2 2 2 2
5242024-02-28T22:13:05.302Z04/30 Write block 542 len 24576 data: 2 2 2 2 2 2
5252024-02-28T22:13:05.302Z05/30 Write block 348 len 4096 data: 2
5262024-02-28T22:13:05.303Z06/30 Read block 108 len 4096
5272024-02-28T22:13:05.303Z07/30 Read block 183 len 40960
5282024-02-28T22:13:05.303Z08/30 Flush
5292024-02-28T22:13:05.303Z09/30 Write block 474 len 24576 data: 2 2 2 2 2 2
5302024-02-28T22:13:05.303Z10/30 Read block 339 len 12288
5312024-02-28T22:13:05.304Z11/30 Write block 359 len 20480 data: 2 2 2 2 2
5322024-02-28T22:13:05.304Z12/30 Read block 54 len 36864
5332024-02-28T22:13:05.304Z13/30 Flush
5342024-02-28T22:13:05.313Z14/30 Write block 49 len 20480 data: 3 2 2 2 2
5352024-02-28T22:13:05.313Z15/30 Read block 560 len 24576
5362024-02-28T22:13:05.313Z16/30 Read block 55 len 12288
5372024-02-28T22:13:05.313Z17/30 Write block 282 len 20480 data: 2 2 2 2 2
5382024-02-28T22:13:05.314Z18/30 Read block 566 len 40960
5392024-02-28T22:13:05.314Z19/30 Read block 207 len 12288
5402024-02-28T22:13:05.314Z20/30 Read block 512 len 36864
5412024-02-28T22:13:05.314Z21/30 Flush
5422024-02-28T22:13:05.314Z22/30 Write block 590 len 16384 data: 2 2 2 2
5432024-02-28T22:13:05.314Z23/30 Read block 97 len 20480
5442024-02-28T22:13:05.314Z24/30 Write block 295 len 12288 data: 2 2 2
5452024-02-28T22:13:05.314Z25/30 Write block 590 len 4096 data: 3
5462024-02-28T22:13:05.314Z26/30 Write block 45 len 4096 data: 4
5472024-02-28T22:13:05.314Z27/30 Read block 294 len 36864
5482024-02-28T22:13:05.314Z28/30 Read block 353 len 12288
5492024-02-28T22:13:05.314Z29/30 Read block 452 len 36864
5502024-02-28T22:13:05.323Z30/30 Read block 457 len 40960
5512024-02-28T22:13:05.323Z----------------------------------------------------------------
5522024-02-28T22:13:05.323Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:28
5532024-02-28T22:13:05.323ZStates: Active Active Active
5542024-02-28T22:13:05.323ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5552024-02-28T22:13:05.324Z 3 Acked 1002 Write 9 Done Done Done false
5562024-02-28T22:13:05.324Z 4 Acked 1003 Write 6 Done Done Sent false
5572024-02-28T22:13:05.324Z 5 Acked 1004 Write 1 Done Done Sent false
5582024-02-28T22:13:05.324Z 6 Acked 1005 Read 1 Done Done Sent false
5592024-02-28T22:13:05.324Z 7 Acked 1006 Read 10 Done Done Sent false
5602024-02-28T22:13:05.324Z 8 Acked 1007 Flush 0 Done Done Sent false
5612024-02-28T22:13:05.324Z 9 Acked 1008 Write 6 Done Done Sent false
5622024-02-28T22:13:05.325Z 10 Acked 1009 Read 3 Done Done Sent false
5632024-02-28T22:13:05.325Z 11 Acked 1010 Write 5 Done Done Sent false
5642024-02-28T22:13:05.325Z 12 Acked 1011 Read 9 Done Done Sent false
5652024-02-28T22:13:05.325Z 13 Acked 1012 Flush 0 Done Done Sent false
5662024-02-28T22:13:05.326Z 14 Acked 1013 Write 5 Done Done Sent false
5672024-02-28T22:13:05.326Z 15 Acked 1014 Read 6 Done Done Sent false
5682024-02-28T22:13:05.326Z 16 Acked 1015 Read 3 Done Done Sent false
5692024-02-28T22:13:05.326Z 17 Acked 1016 Write 5 Done Done Sent false
5702024-02-28T22:13:05.326Z 18 Acked 1017 Read 10 Done Done Sent false
5712024-02-28T22:13:05.326Z 19 Acked 1018 Read 3 Done Done Sent false
5722024-02-28T22:13:05.326Z 20 Acked 1019 Read 9 Done Done Sent false
5732024-02-28T22:13:05.327Z 21 Acked 1020 Flush 0 Done Done Sent false
5742024-02-28T22:13:05.327Z 22 Acked 1021 Write 4 Done Done Sent false
5752024-02-28T22:13:05.327Z 23 Acked 1022 Read 5 Done Done Sent false
5762024-02-28T22:13:05.327Z 24 Acked 1023 Write 3 Done Done Sent false
5772024-02-28T22:13:05.327Z 25 Acked 1024 Write 1 Done Done Sent false
5782024-02-28T22:13:05.327Z 26 Acked 1025 Write 1 Done Done Sent false
5792024-02-28T22:13:05.327Z 27 Acked 1026 Read 9 Done Done Sent false
5802024-02-28T22:13:05.327Z 28 Acked 1027 Read 3 Done Done Sent false
5812024-02-28T22:13:05.327Z 29 Acked 1028 Read 9 Done Done Sent false
5822024-02-28T22:13:05.327Z 30 Acked 1029 Read 10 Sent Done Sent false
5832024-02-28T22:13:05.327Z STATES DS:0 DS:1 DS:2 TOTAL
5842024-02-28T22:13:05.328Z New 0 0 0 0
5852024-02-28T22:13:05.328Z Sent 1 0 27 28
5862024-02-28T22:13:05.328Z Done 27 28 1 56
5872024-02-28T22:13:05.328Z Skipped 0 0 0 0
5882024-02-28T22:13:05.328Z Error 0 0 0 0
5892024-02-28T22:13:05.328ZLast Flush: 1020 1020 1001
5902024-02-28T22:13:05.328ZDownstairs last five completed: 1001 1000
5912024-02-28T22:13:05.328ZUpstairs last five completed: 30 29 28 27 26
5922024-02-28T22:13:05.328Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:04.215964159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1151}
5932024-02-28T22:13:05.328ZWrote out file "/tmp/test_repair_verify.data"
5942024-02-28T22:13:05.328Z
5952024-02-28T22:13:05.328ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5962024-02-28T22:13:05.336ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5972024-02-28T22:13:05.336Z 0 000-019 1 1 1 1 1 1 F F F
5982024-02-28T22:13:05.336Z 1 020-039 1 1 1 1 1 1 F F F
5992024-02-28T22:13:05.337Z 2 040-059 4 4 3 9 9 5 T T F <---
6002024-02-28T22:13:05.337Z 3 060-079 3 3 3 5 5 5 F F F
6012024-02-28T22:13:05.337Z 4 080-099 1 1 1 1 1 1 F F F
6022024-02-28T22:13:05.337Z 5 100-119 2 2 2 4 4 4 F F F
6032024-02-28T22:13:05.344Z 6 120-139 1 1 1 1 1 1 F F F
6042024-02-28T22:13:05.344Z 7 140-159 1 1 1 1 1 1 F F F
6052024-02-28T22:13:05.345Z 8 160-179 4 4 4 6 6 6 F F F
6062024-02-28T22:13:05.345Z 9 180-199 1 1 1 1 1 1 F F F
6072024-02-28T22:13:05.345Z 10 200-219 1 1 1 1 1 1 F F F
6082024-02-28T22:13:05.345Z 11 220-239 3 3 3 5 5 5 F F F
6092024-02-28T22:13:05.345Z 12 240-259 2 2 2 4 4 4 F F F
6102024-02-28T22:13:05.345Z 13 260-279 1 1 1 1 1 1 F F F
6112024-02-28T22:13:05.346Z 14 280-299 4 4 1 9 9 1 T T F <---
6122024-02-28T22:13:05.346Z 15 300-319 2 2 2 4 4 4 F F F
6132024-02-28T22:13:05.346Z 16 320-339 1 1 1 1 1 1 F F F
6142024-02-28T22:13:05.346Z 17 340-359 4 4 1 8 8 1 F F F <---
6152024-02-28T22:13:05.346Z 18 360-379 4 4 1 8 8 1 F F F <---
6162024-02-28T22:13:05.346Z 19 380-399 3 3 3 5 5 5 F F F
6172024-02-28T22:13:05.346Z 20 400-419 2 2 2 4 4 4 F F F
6182024-02-28T22:13:05.346Z 21 420-439 2 2 2 4 4 4 F F F
6192024-02-28T22:13:05.346Z 22 440-459 3 3 3 5 5 5 F F F
6202024-02-28T22:13:05.346Z 23 460-479 4 4 1 8 8 1 F F F <---
6212024-02-28T22:13:05.347Z 24 480-499 3 3 3 5 5 5 F F F
6222024-02-28T22:13:05.347Z 25 500-519 4 4 3 7 7 5 F F T <---
6232024-02-28T22:13:05.347Z 26 520-539 3 3 3 5 5 5 F F F
6242024-02-28T22:13:05.347Z 27 540-559 4 4 1 7 7 1 F F F <---
6252024-02-28T22:13:05.347Z 28 560-579 1 1 1 1 1 1 F F F
6262024-02-28T22:13:05.347Z 29 580-599 1 1 1 1 1 1 T T F <---
6272024-02-28T22:13:05.347ZMax gen: 4, Max flush: 9
6282024-02-28T22:13:05.347ZError: Difference in extent metadata found!
6292024-02-28T22:13:05.347ZA Difference in extent metadata is expected here
6302024-02-28T22:13:05.347ZOn loop 2
6312024-02-28T22:13:05.348Z
6322024-02-28T22:13:05.357ZVerifying data now
6332024-02-28T22:13:10.644ZNo change after verify
6342024-02-28T22:13:10.644ZLoop: 2 Downstairs dump after verify (and repair):
6352024-02-28T22:13:10.659ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6362024-02-28T22:13:10.659Z 0 000-019 1 1 1 1 1 1 F F F
6372024-02-28T22:13:10.659Z 1 020-039 1 1 1 1 1 1 F F F
6382024-02-28T22:13:10.659Z 2 040-059 5 5 5 10 10 10 F F F
6392024-02-28T22:13:10.660Z 3 060-079 3 3 3 5 5 5 F F F
6402024-02-28T22:13:10.660Z 4 080-099 1 1 1 1 1 1 F F F
6412024-02-28T22:13:10.660Z 5 100-119 2 2 2 4 4 4 F F F
6422024-02-28T22:13:10.660Z 6 120-139 1 1 1 1 1 1 F F F
6432024-02-28T22:13:10.661Z 7 140-159 1 1 1 1 1 1 F F F
6442024-02-28T22:13:10.661Z 8 160-179 4 4 4 6 6 6 F F F
6452024-02-28T22:13:10.661Z 9 180-199 1 1 1 1 1 1 F F F
6462024-02-28T22:13:10.661Z 10 200-219 1 1 1 1 1 1 F F F
6472024-02-28T22:13:10.662Z 11 220-239 3 3 3 5 5 5 F F F
6482024-02-28T22:13:10.662Z 12 240-259 2 2 2 4 4 4 F F F
6492024-02-28T22:13:10.662Z 13 260-279 1 1 1 1 1 1 F F F
6502024-02-28T22:13:10.662Z 14 280-299 5 5 5 10 10 10 F F F
6512024-02-28T22:13:10.662Z 15 300-319 2 2 2 4 4 4 F F F
6522024-02-28T22:13:10.662Z 16 320-339 1 1 1 1 1 1 F F F
6532024-02-28T22:13:10.662Z 17 340-359 4 4 4 8 8 8 F F F
6542024-02-28T22:13:10.663Z 18 360-379 4 4 4 8 8 8 F F F
6552024-02-28T22:13:10.663Z 19 380-399 3 3 3 5 5 5 F F F
6562024-02-28T22:13:10.663Z 20 400-419 2 2 2 4 4 4 F F F
6572024-02-28T22:13:10.663Z 21 420-439 2 2 2 4 4 4 F F F
6582024-02-28T22:13:10.663Z 22 440-459 3 3 3 5 5 5 F F F
6592024-02-28T22:13:10.663Z 23 460-479 4 4 4 8 8 8 F F F
6602024-02-28T22:13:10.663Z 24 480-499 3 3 3 5 5 5 F F F
6612024-02-28T22:13:10.664Z 25 500-519 4 4 4 7 7 7 F F F
6622024-02-28T22:13:10.664Z 26 520-539 3 3 3 5 5 5 F F F
6632024-02-28T22:13:10.664Z 27 540-559 4 4 4 7 7 7 F F F
6642024-02-28T22:13:10.664Z 28 560-579 1 1 1 1 1 1 F F F
6652024-02-28T22:13:10.664Z 29 580-599 5 5 5 10 10 10 F F F
6662024-02-28T22:13:10.664ZMax gen: 5, Max flush: 10
6672024-02-28T22:13:10.664Z
6682024-02-28T22:13:10.664ZBegin loop 3
6692024-02-28T22:13:10.665ZDownstairs to restart: 1
6702024-02-28T22:13:10.665Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.164730326Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
6712024-02-28T22:13:10.665Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165031737Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
6722024-02-28T22:13:10.665Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":Crucible runtime is spawned
6732024-02-28T22:13:10.665Z0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165078656Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
6742024-02-28T22:13:10.665Z{"msg":"Crucible b329e603-8d35-4003-a2bc-a0a54871cba4 has session id: 9a364bc4-5e36-414e-bd82-19b088a46f65","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165097355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6752024-02-28T22:13:10.665Z{"msg":"Upstairs opts: Upstairs UUID: b329e603-8d35-4003-a2bc-a0a54871cba4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165129255Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6762024-02-28T22:13:10.665Z{"msg":"Crucible stats registered with UUID: b329e603-8d35-4003-a2bc-a0a54871cba4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165144184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6772024-02-28T22:13:10.666Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165157684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
6782024-02-28T22:13:10.666Z{"msg":"b329e603-8d35-4003-a2bc-a0a54871cba4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165255431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6792024-02-28T22:13:10.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16530416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6802024-02-28T22:13:10.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165340269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6812024-02-28T22:13:10.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165401017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6822024-02-28T22:13:10.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165431226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6832024-02-28T22:13:10.667Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165459835Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6842024-02-28T22:13:10.667Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165477525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6852024-02-28T22:13:10.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165510464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6862024-02-28T22:13:10.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165551503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6872024-02-28T22:13:10.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16563613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6882024-02-28T22:13:10.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16566529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6892024-02-28T22:13:10.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165690769Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6902024-02-28T22:13:10.668Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.165736908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"io task","client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6912024-02-28T22:13:10.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166335041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6922024-02-28T22:13:10.668Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166363241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6932024-02-28T22:13:10.668Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16637811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6942024-02-28T22:13:10.669Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166412259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6952024-02-28T22:13:10.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166457648Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6962024-02-28T22:13:10.669Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166497627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6972024-02-28T22:13:10.669Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166918055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6982024-02-28T22:13:10.669Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.166964724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
6992024-02-28T22:13:10.669Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16710445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7002024-02-28T22:13:10.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167492709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7012024-02-28T22:13:10.670Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167534778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7022024-02-28T22:13:10.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167551318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7032024-02-28T22:13:10.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167564537Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7042024-02-28T22:13:10.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167587117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7052024-02-28T22:13:10.671Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167601346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7062024-02-28T22:13:10.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167613996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7072024-02-28T22:13:10.671Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167627816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7082024-02-28T22:13:10.671Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 10, 5, 1, 4, 1, 1, 6, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167649055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7092024-02-28T22:13:10.671Z{"msg":"[0]R generation[0..12]: [1, 1, 5, 3, 1, 2, 1, 1, 4, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167663995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7102024-02-28T22:13:10.671Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167686574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7112024-02-28T22:13:10.671Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 10, 5, 1, 4, 1, 1, 6, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167701373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7122024-02-28T22:13:10.672Z{"msg":"[1]R generation[0..12]: [1, 1, 5, 3, 1, 2, 1, 1, 4, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167715623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7132024-02-28T22:13:10.672Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167737323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7142024-02-28T22:13:10.672Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 10, 5, 1, 4, 1, 1, 6, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167751362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7152024-02-28T22:13:10.672Z{"msg":"[2]R generation[0..12]: [1, 1, 5, 3, 1, 2, 1, 1, 4, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167764982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7162024-02-28T22:13:10.672Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167786311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7172024-02-28T22:13:10.673Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
7182024-02-28T22:13:10.673Z,"time":"2024-02-28T22:13:09.167800111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7192024-02-28T22:13:10.673Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16781465Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7202024-02-28T22:13:10.673Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.16783493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7212024-02-28T22:13:10.673Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167848369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"":"downstairs","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7222024-02-28T22:13:10.674Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167861779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7232024-02-28T22:13:10.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167874129Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"0","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7242024-02-28T22:13:10.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167895618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"1","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7252024-02-28T22:13:10.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167909328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"client":"2","":"downstairs","session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7262024-02-28T22:13:10.674Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167922667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7272024-02-28T22:13:10.674Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167942717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7282024-02-28T22:13:10.674Z{"msg":"b329e603-8d35-4003-a2bc-a0a54871cba4 is now active with session: 9a364bc4-5e36-414e-bd82-19b088a46f65","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167956357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7292024-02-28T22:13:10.674Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167968206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159,"session_id":"9a364bc4-5e36-414e-bd82-19b088a46f65"}
7302024-02-28T22:13:10.675Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:09.167987536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
7312024-02-28T22:13:10.675ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7322024-02-28T22:13:10.675ZLoading write count information from file "/tmp/test_repair_verify.data"
7332024-02-28T22:13:10.675ZRun Repair workload
7342024-02-28T22:13:10.675Z01/30 Write block 590 len 40960 data: 4 3 3 3 2 2 2 2 2 2
7352024-02-28T22:13:10.689Z02/30 Write block 456 len 4096 data: 3
7362024-02-28T22:13:10.689Z03/30 Write block 62 len 12288 data: 3 2 2
7372024-02-28T22:13:10.689Z04/30 Write block 63 len 40960 data: 3 3 2 2 2 2 2 2 2 2
7382024-02-28T22:13:10.689Z05/30 Write block 385 len 28672 data: 3 3 3 3 3 3 3
7392024-02-28T22:13:10.690Z06/30 Read block 538 len 12288
7402024-02-28T22:13:10.690Z07/30 Write block 227 len 36864 data: 3 3 3 3 3 4 2 2 2
7412024-02-28T22:13:10.690Z08/30 Write block 229 len 36864 data: 4 4 4 5 3 3 3 2 2
7422024-02-28T22:13:10.690Z09/30 Write block 100 len 32768 data: 2 2 2 3 3 3 3 3
7432024-02-28T22:13:10.690Z10/30 Read block 551 len 24576
7442024-02-28T22:13:10.690Z11/30 Read block 489 len 8192
7452024-02-28T22:13:10.690Z12/30 Write block 121 len 36864 data: 2 2 2 2 2 2 2 2 2
7462024-02-28T22:13:10.691Z13/30 Read block 289 len 28672
7472024-02-28T22:13:10.691Z14/30 Read block 580 len 28672
7482024-02-28T22:13:10.691Z15/30 Write block 549 len 28672 data: 2 2 2 2 2 2 2
7492024-02-28T22:13:10.691Z16/30 Read block 444 len 12288
7502024-02-28T22:13:10.691Z17/30 Read block 233 len 4096
7512024-02-28T22:13:10.691Z18/30 Read block 279 len 28672
7522024-02-28T22:13:10.691Z19/30 Write block 71 len 16384 data: 3 3 2 2
7532024-02-28T22:13:10.691Z20/30 Write block 407 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7542024-02-28T22:13:10.692Z21/30 Read block 530 len 24576
7552024-02-28T22:13:10.692Z22/30 Read block 579 len 16384
7562024-02-28T22:13:10.692Z23/30 Write block 244 len 4096 data: 2
7572024-02-28T22:13:10.692Z24/30 Write block 475 len 8192 data: 3 3
7582024-02-28T22:13:10.692Z25/30 Write block 322 len 24576 data: 2 2 2 2 2 2
7592024-02-28T22:13:10.692Z26/30 Write block 485 len 36864 data: 2 2 2 2 2 2 2 2 2
7602024-02-28T22:13:10.693Z27/30 Write block 103 len 40960 data: 4 4 4 4 4 4 3 2 2 2
7612024-02-28T22:13:10.693Z28/30 Read block 173 len 8192
7622024-02-28T22:13:10.693Z29/30 Write block 304 len 4096 data: 2
7632024-02-28T22:13:10.693Z30/30 Read block 327 len 28672
7642024-02-28T22:13:10.693Z----------------------------------------------------------------
7652024-02-28T22:13:10.694Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7662024-02-28T22:13:10.694ZStates: Active Active Active
7672024-02-28T22:13:10.694ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7682024-02-28T22:13:10.694Z 1 Acked 1000 Write 10 Done Sent Done false
7692024-02-28T22:13:10.695Z 2 Acked 1001 Write 1 Done Sent Done false
7702024-02-28T22:13:10.695Z 3 Acked 1002 Write 3 Done Sent Done false
7712024-02-28T22:13:10.696Z 4 Acked 1003 Write 10 Done Sent Done false
7722024-02-28T22:13:10.696Z 5 Acked 1004 Write 7 Done Sent Done false
7732024-02-28T22:13:10.696Z 6 Acked 1005 Read 3 Done Sent Done false
7742024-02-28T22:13:10.697Z 7 Acked 1006 Write 9 Done Sent Done false
7752024-02-28T22:13:10.697Z 8 Acked 1007 Write 9 Done Sent Done false
7762024-02-28T22:13:10.697Z 9 Acked 1008 Write 8 Done Sent Done false
7772024-02-28T22:13:10.697Z 10 Acked 1009 Read 6 Done Sent Done false
7782024-02-28T22:13:10.698Z 11 Acked 1010 Read 2 Done Sent Done false
7792024-02-28T22:13:10.698Z 12 Acked 1011 Write 9 Done Sent Done false
7802024-02-28T22:13:10.699Z 13 Acked 1012 Read 7 Done Sent Done false
7812024-02-28T22:13:10.699Z 14 Acked 1013 Read 7 Done Sent Done false
7822024-02-28T22:13:10.699Z 15 Acked 1014 Write 7 Done Sent Done false
7832024-02-28T22:13:10.699Z 16 Acked 1015 Read 3 Done Sent Done false
7842024-02-28T22:13:10.699Z 17 Acked 1016 Read 1 Done Sent Done false
7852024-02-28T22:13:10.700Z 18 Acked 1017 Read 7 Done Sent Done false
7862024-02-28T22:13:10.700Z 19 Acked 1018 Write 4 Done Sent Done false
7872024-02-28T22:13:10.700Z 20 Acked 1019 Write 10 Done Sent Done false
7882024-02-28T22:13:10.700Z 21 Acked 1020 Read 6 Done Sent Done false
7892024-02-28T22:13:10.701Z 22 Acked 1021 Read 4 Done Sent Done false
7902024-02-28T22:13:10.701Z 23 Acked 1022 Write 1 Done Sent Done false
7912024-02-28T22:13:10.701Z 24 Acked 1023 Write 2 Done Sent Done false
7922024-02-28T22:13:10.702Z 25 Acked 1024 Write 6 Done Sent Done false
7932024-02-28T22:13:10.702Z 26 Acked 1025 Write 9 Done Sent Done false
7942024-02-28T22:13:10.702Z 27 Acked 1026 Write 10 Done Sent Done false
7952024-02-28T22:13:10.703Z 28 Acked 1027 Read 2 Done Sent Done false
7962024-02-28T22:13:10.703Z 29 Acked 1028 Write 1 Done Sent Done false
7972024-02-28T22:13:10.703Z 30 Acked 1029 Read 7 Sent Sent Done false
7982024-02-28T22:13:10.703Z STATES DS:0 DS:1 DS:2 TOTAL
7992024-02-28T22:13:10.704Z New 0 0 0 0
8002024-02-28T22:13:10.704Z Sent 1 30 0 31
8012024-02-28T22:13:10.705Z Done 29 0 30 59
8022024-02-28T22:13:10.705Z Skipped 0 0 0 0
8032024-02-28T22:13:10.705Z Error 0 0 0 0
8042024-02-28T22:13:10.705ZLast Flush: 0 0 0
8052024-02-28T22:13:10.705ZDownstairs last five completed:
8062024-02-28T22:13:10.706ZUpstairs last five completed: 30 29 28 27 26
8072024-02-28T22:13:10.706Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:09.195886147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1159}
8082024-02-28T22:13:10.706ZWrote out file "/tmp/test_repair_verify.data"
8092024-02-28T22:13:10.706Z
8102024-02-28T22:13:10.707ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8112024-02-28T22:13:10.707ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8122024-02-28T22:13:10.707Z 0 000-019 1 1 1 1 1 1 F F F
8132024-02-28T22:13:10.707Z 1 020-039 1 1 1 1 1 1 F F F
8142024-02-28T22:13:10.707Z 2 040-059 5 5 5 10 10 10 F F F
8152024-02-28T22:13:10.708Z 3 060-079 3 3 3 5 5 5 T F T <---
8162024-02-28T22:13:10.708Z 4 080-099 1 1 1 1 1 1 F F F
8172024-02-28T22:13:10.708Z 5 100-119 2 2 2 4 4 4 T F T <---
8182024-02-28T22:13:10.708Z 6 120-139 1 1 1 1 1 1 T F T <---
8192024-02-28T22:13:10.709Z 7 140-159 1 1 1 1 1 1 F F F
8202024-02-28T22:13:10.709Z 8 160-179 4 4 4 6 6 6 F F F
8212024-02-28T22:13:10.709Z 9 180-199 1 1 1 1 1 1 F F F
8222024-02-28T22:13:10.709Z 10 200-219 1 1 1 1 1 1 F F F
8232024-02-28T22:13:10.709Z 11 220-239 3 3 3 5 5 5 T F T <---
8242024-02-28T22:13:10.710Z 12 240-259 2 2 2 4 4 4 T F T <---
8252024-02-28T22:13:10.710Z 13 260-279 1 1 1 1 1 1 F F F
8262024-02-28T22:13:10.710Z 14 280-299 5 5 5 10 10 10 F F F
8272024-02-28T22:13:10.710Z 15 300-319 2 2 2 4 4 4 T F T <---
8282024-02-28T22:13:10.710Z 16 320-339 1 1 1 1 1 1 T F T <---
8292024-02-28T22:13:10.711Z 17 340-359 4 4 4 8 8 8 F F F
8302024-02-28T22:13:10.711Z 18 360-379 4 4 4 8 8 8 F F F
8312024-02-28T22:13:10.711Z 19 380-399 3 3 3 5 5 5 T F T <---
8322024-02-28T22:13:10.711Z 20 400-419 2 2 2 4 4 4 T F T <---
8332024-02-28T22:13:10.711Z 21 420-439 2 2 2 4 4 4 F F F
8342024-02-28T22:13:10.711Z 22 440-459 3 3 3 5 5 5 T F T <---
8352024-02-28T22:13:11.132Z 23 460-479 4 4 4 8 8 8 T F T <---
8362024-02-28T22:13:11.132Z 24 480-499 3 3 3 5 5 5 T F T <---
8372024-02-28T22:13:11.132Z 25 500-519 4 4 4 7 7 7 F F F
8382024-02-28T22:13:11.132Z 26 520-539 3 3 3 5 5 5 F F F
8392024-02-28T22:13:11.133Z 27 540-559 4 4 4 7 7 7 T F T <---
8402024-02-28T22:13:11.133Z 28 560-579 1 1 1 1 1 1 F F F
8412024-02-28T22:13:11.133Z 29 580-599 5 5 5 10 10 10 T F T <---
8422024-02-28T22:13:11.133ZMax gen: 5, Max flush: 10
8432024-02-28T22:13:11.133ZError: Difference in extent metadata found!
8442024-02-28T22:13:11.133ZA Difference in extent metadata is expected here
8452024-02-28T22:13:11.133ZOn loop 3
8462024-02-28T22:13:11.133Z
8472024-02-28T22:13:11.134ZVerifying data now
8482024-02-28T22:13:16.015ZNo change after verify
8492024-02-28T22:13:16.015ZLoop: 3 Downstairs dump after verify (and repair):
8502024-02-28T22:13:16.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8512024-02-28T22:13:16.025Z 0 000-019 1 1 1 1 1 1 F F F
8522024-02-28T22:13:16.025Z 1 020-039 1 1 1 1 1 1 F F F
8532024-02-28T22:13:16.025Z 2 040-059 5 5 5 10 10 10 F F F
8542024-02-28T22:13:16.026Z 3 060-079 6 6 6 11 11 11 F F F
8552024-02-28T22:13:16.026Z 4 080-099 1 1 1 1 1 1 F F F
8562024-02-28T22:13:16.026Z 5 100-119 6 6 6 11 11 11 F F F
8572024-02-28T22:13:16.026Z 6 120-139 6 6 6 11 11 11 F F F
8582024-02-28T22:13:16.026Z 7 140-159 1 1 1 1 1 1 F F F
8592024-02-28T22:13:16.027Z 8 160-179 4 4 4 6 6 6 F F F
8602024-02-28T22:13:16.027Z 9 180-199 1 1 1 1 1 1 F F F
8612024-02-28T22:13:16.027Z 10 200-219 1 1 1 1 1 1 F F F
8622024-02-28T22:13:16.027Z 11 220-239 6 6 6 11 11 11 F F F
8632024-02-28T22:13:16.027Z 12 240-259 6 6 6 11 11 11 F F F
8642024-02-28T22:13:16.027Z 13 260-279 1 1 1 1 1 1 F F F
8652024-02-28T22:13:16.028Z 14 280-299 5 5 5 10 10 10 F F F
8662024-02-28T22:13:16.028Z 15 300-319 6 6 6 11 11 11 F F F
8672024-02-28T22:13:16.028Z 16 320-339 6 6 6 11 11 11 F F F
8682024-02-28T22:13:16.028Z 17 340-359 4 4 4 8 8 8 F F F
8692024-02-28T22:13:16.028Z 18 360-379 4 4 4 8 8 8 F F F
8702024-02-28T22:13:16.029Z 19 380-399 6 6 6 11 11 11 F F F
8712024-02-28T22:13:16.029Z 20 400-419 6 6 6 11 11 11 F F F
8722024-02-28T22:13:16.029Z 21 420-439 2 2 2 4 4 4 F F F
8732024-02-28T22:13:16.029Z 22 440-459 6 6 6 11 11 11 F F F
8742024-02-28T22:13:16.030Z 23 460-479 6 6 6 11 11 11 F F F
8752024-02-28T22:13:16.030Z 24 480-499 6 6 6 11 11 11 F F F
8762024-02-28T22:13:16.030Z 25 500-519 4 4 4 7 7 7 F F F
8772024-02-28T22:13:16.030Z 26 520-539 3 3 3 5 5 5 F F F
8782024-02-28T22:13:16.031Z 27 540-559 6 6 6 11 11 11 F F F
8792024-02-28T22:13:16.031Z 28 560-579 1 1 1 1 1 1 F F F
8802024-02-28T22:13:16.031Z 29 580-599 6 6 6 11 11 11 F F F
8812024-02-28T22:13:16.031ZMax gen: 6, Max flush: 11
8822024-02-28T22:13:16.043Z
8832024-02-28T22:13:16.043ZBegin loop 4
8842024-02-28T22:13:16.044ZDownstairs to restart: 2
8852024-02-28T22:13:16.475Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94626616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
8862024-02-28T22:13:16.475Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946572651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
8872024-02-28T22:13:16.475Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
8882024-02-28T22:13:16.476Z,"time":"2024-02-28T22:13:14.94660576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
8892024-02-28T22:13:16.476Z{"msg":"Crucible 30802b15-d405-4c35-8fab-315038fe315d has session id: 46ca72f9-8665-4ca0-8828-073488a05263","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94663016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8902024-02-28T22:13:16.476Z{"msg":"Upstairs opts: Upstairs UUID: 30802b15-d405-4c35-8fab-315038fe315d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946675578Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8912024-02-28T22:13:16.476Z{"msg":"Crucible stats registered with UUID: 30802b15-d405-4c35-8fab-315038fe315d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946711067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8922024-02-28T22:13:16.476Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946775056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
8932024-02-28T22:13:16.476Z{"msg":"30802b15-d405-4c35-8fab-315038fe315d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946803735Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8942024-02-28T22:13:16.476Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946829294Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8952024-02-28T22:13:16.476Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946866133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8962024-02-28T22:13:16.476Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946901962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8972024-02-28T22:13:16.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.946930021Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8982024-02-28T22:13:16.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94697693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
8992024-02-28T22:13:16.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947008069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9002024-02-28T22:13:16.477Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947037808Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9012024-02-28T22:13:16.477Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947077597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9022024-02-28T22:13:16.477Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947121546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9032024-02-28T22:13:16.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947176494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9042024-02-28T22:13:16.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947205954Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9052024-02-28T22:13:16.478Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947228453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"io task","client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9062024-02-28T22:13:16.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947877155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9072024-02-28T22:13:16.478Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947925254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9082024-02-28T22:13:16.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.947965243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9092024-02-28T22:13:16.478Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.948007142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9102024-02-28T22:13:16.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.948038631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9112024-02-28T22:13:16.478Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94806834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9122024-02-28T22:13:16.478Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.948479969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9132024-02-28T22:13:16.478Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.948521337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9142024-02-28T22:13:16.478Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.948757021Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9152024-02-28T22:13:16.478Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949046663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9162024-02-28T22:13:16.478Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949089582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9172024-02-28T22:13:16.479Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949115261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9182024-02-28T22:13:16.479Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94913814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9192024-02-28T22:13:16.479Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9202024-02-28T22:13:16.479Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94916867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9212024-02-28T22:13:16.479Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949188459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9222024-02-28T22:13:16.479Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949202879Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9232024-02-28T22:13:16.479Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 10, 11, 1, 11, 11, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949238078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9242024-02-28T22:13:16.479Z{"msg":"[0]R generation[0..12]: [1, 1, 5, 6, 1, 6, 6, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949271327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9252024-02-28T22:13:16.479Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949287486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9262024-02-28T22:13:16.479Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 10, 11, 1, 11, 11, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949302286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9272024-02-28T22:13:16.479Z{"msg":"[1]R generation[0..12]: [1, 1, 5, 6, 1, 6, 6, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949324265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9282024-02-28T22:13:16.480Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949338955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9292024-02-28T22:13:16.480Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 10, 11, 1, 11, 11, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949366404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9302024-02-28T22:13:16.480Z{"msg":"[2]R generation[0..12]: [1, 1, 5, 6, 1, 6, 6, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949380604Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9312024-02-28T22:13:16.480Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949394924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}Wait for a query_work_queue command to finish before sending IO
9322024-02-28T22:13:16.480Z
9332024-02-28T22:13:16.480Z{"msg":"Max found gen is 7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949424943Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9342024-02-28T22:13:16.480Z{"msg":"Generation requested: 8 >= found:7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949438972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9352024-02-28T22:13:16.480Z{"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949460112Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9362024-02-28T22:13:16.480Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949480311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"":"downstairs","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9372024-02-28T22:13:16.480Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94951303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9382024-02-28T22:13:16.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.94952765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"0","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9392024-02-28T22:13:16.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949554909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"1","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9402024-02-28T22:13:16.481Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949587268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"client":"2","":"downstairs","session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9412024-02-28T22:13:16.481Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949612027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9422024-02-28T22:13:16.481Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949632577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9432024-02-28T22:13:16.481Z{"msg":"30802b15-d405-4c35-8fab-315038fe315d is now active with session: 46ca72f9-8665-4ca0-8828-073488a05263","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949659966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9442024-02-28T22:13:16.481Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949691215Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167,"session_id":"46ca72f9-8665-4ca0-8828-073488a05263"}
9452024-02-28T22:13:16.481Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:14.949715804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
9462024-02-28T22:13:16.481ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9472024-02-28T22:13:16.482ZLoading write count information from file "/tmp/test_repair_verify.data"
9482024-02-28T22:13:16.482ZRun Repair workload
9492024-02-28T22:13:16.482Z01/30 Flush
9502024-02-28T22:13:16.482Z02/30 Write block 567 len 8192 data: 2 2
9512024-02-28T22:13:16.482Z03/30 Write block 65 len 12288 data: 3 3 3
9522024-02-28T22:13:16.482Z04/30 Write block 130 len 32768 data: 2 2 2 2 2 2 2 2
9532024-02-28T22:13:16.482Z05/30 Write block 376 len 36864 data: 2 2 2 2 2 2 2 2 3
9542024-02-28T22:13:16.511Z06/30 Write block 63 len 32768 data: 4 4 4 4 4 3 3 3
9552024-02-28T22:13:16.511Z07/30 Read block 533 len 20480
9562024-02-28T22:13:16.511Z08/30 Read block 553 len 36864
9572024-02-28T22:13:16.511Z09/30 Read block 61 len 32768
9582024-02-28T22:13:16.511Z10/30 Write block 463 len 16384 data: 2 2 2 2
9592024-02-28T22:13:16.511Z11/30 Flush
9602024-02-28T22:13:16.511Z12/30 Write block 247 len 28672 data: 2 2 2 2 2 2 2
9612024-02-28T22:13:16.511Z13/30 Write block 328 len 32768 data: 2 2 2 2 2 2 2 2
9622024-02-28T22:13:16.512Z14/30 Read block 212 len 32768
9632024-02-28T22:13:16.512Z15/30 Flush
9642024-02-28T22:13:16.512Z16/30 Write block 356 len 12288 data: 2 2 2
9652024-02-28T22:13:16.512Z17/30 Read block 274 len 36864
9662024-02-28T22:13:16.512Z18/30 Write block 495 len 12288 data: 3 3 3
9672024-02-28T22:13:16.513Z19/30 Read block 350 len 24576
9682024-02-28T22:13:16.513Z20/30 Flush
9692024-02-28T22:13:16.513Z21/30 Write block 14 len 28672 data: 2 2 2 2 2 2 2
9702024-02-28T22:13:16.513Z22/30 Write block 355 len 8192 data: 2 3
9712024-02-28T22:13:16.513Z23/30 Flush
9722024-02-28T22:13:16.513Z24/30 Flush
9732024-02-28T22:13:16.513Z25/30 Write block 301 len 32768 data: 2 2 2 3 2 2 2 2
9742024-02-28T22:13:16.513Z26/30 Read block 297 len 40960
9752024-02-28T22:13:16.513Z27/30 Read block 381 len 28672
9762024-02-28T22:13:16.513Z28/30 Read block 372 len 16384
9772024-02-28T22:13:16.513Z29/30 Write block 556 len 24576 data: 2 2 2 2 2 2
9782024-02-28T22:13:16.513Z30/30 Read block 286 len 36864
9792024-02-28T22:13:16.513Z----------------------------------------------------------------
9802024-02-28T22:13:16.514Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
9812024-02-28T22:13:16.514ZStates: Active Active Active
9822024-02-28T22:13:16.514ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9832024-02-28T22:13:16.514Z 1 Acked 1000 Flush 0 Done Done Sent false
9842024-02-28T22:13:16.514Z 2 Acked 1001 Write 2 Done Done Sent false
9852024-02-28T22:13:16.514Z 3 Acked 1002 Write 3 Done Done Sent false
9862024-02-28T22:13:16.514Z 4 Acked 1003 Write 8 Done Done Sent false
9872024-02-28T22:13:16.514Z 5 Acked 1004 Write 9 Done Done Sent false
9882024-02-28T22:13:16.514Z 6 Acked 1005 Write 8 Done Done Sent false
9892024-02-28T22:13:16.514Z 7 Acked 1006 Read 5 Done Done Sent false
9902024-02-28T22:13:16.514Z 8 Acked 1007 Read 9 Done Done Sent false
9912024-02-28T22:13:16.514Z 9 Acked 1008 Read 8 Done Done Sent false
9922024-02-28T22:13:16.515Z 10 Acked 1009 Write 4 Done Done Sent false
9932024-02-28T22:13:16.515Z 11 Acked 1010 Flush 0 Done Done Sent false
9942024-02-28T22:13:16.515Z 12 Acked 1011 Write 7 Done Done Sent false
9952024-02-28T22:13:16.515Z 13 Acked 1012 Write 8 Done Done Sent false
9962024-02-28T22:13:16.515Z 14 Acked 1013 Read 8 Done Done Sent false
9972024-02-28T22:13:16.515Z 15 Acked 1014 Flush 0 Done Done Sent false
9982024-02-28T22:13:16.515Z 16 Acked 1015 Write 3 Done Done Sent false
9992024-02-28T22:13:16.515Z 17 Acked 1016 Read 9 Done Done Sent false
10002024-02-28T22:13:16.516Z 18 Acked 1017 Write 3 Done Done Sent false
10012024-02-28T22:13:16.516Z 19 Acked 1018 Read 6 Done Done Sent false
10022024-02-28T22:13:16.516Z 20 Acked 1019 Flush 0 Done Done Sent false
10032024-02-28T22:13:16.516Z 21 Acked 1020 Write 7 Done Done Sent false
10042024-02-28T22:13:16.516Z 22 Acked 1021 Write 2 Done Done Sent false
10052024-02-28T22:13:16.516Z 23 Acked 1022 Flush 0 Done Done Sent false
10062024-02-28T22:13:16.516Z 24 Acked 1023 Flush 0 Done Done Sent false
10072024-02-28T22:13:16.517Z 25 Acked 1024 Write 8 Done Done Sent false
10082024-02-28T22:13:16.517Z 26 Acked 1025 Read 10 Done Done Sent false
10092024-02-28T22:13:16.517Z 27 Acked 1026 Read 7 Done Done Sent false
10102024-02-28T22:13:16.517Z 28 Acked 1027 Read 4 Done Done Sent false
10112024-02-28T22:13:16.517Z 29 Acked 1028 Write 6 Done Done Sent false
10122024-02-28T22:13:16.517Z 30 Acked 1029 Read 9 Done Sent Sent false
10132024-02-28T22:13:16.517Z STATES DS:0 DS:1 DS:2 TOTAL
10142024-02-28T22:13:16.517Z New 0 0 0 0
10152024-02-28T22:13:16.517Z Sent 0 1 30 31
10162024-02-28T22:13:16.517Z Done 30 29 0 59
10172024-02-28T22:13:16.518Z Skipped 0 0 0 0
10182024-02-28T22:13:16.518Z Error 0 0 0 0
10192024-02-28T22:13:16.518ZLast Flush: 1023 1023 0
10202024-02-28T22:13:16.518ZDownstairs last five completed:
10212024-02-28T22:13:16.518ZUpstairs last five completed: 30 29 28 27 26
10222024-02-28T22:13:16.518Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:15.031989138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1167}
10232024-02-28T22:13:16.518ZWrote out file "/tmp/test_repair_verify.data"
10242024-02-28T22:13:16.518Z
10252024-02-28T22:13:16.519ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10262024-02-28T22:13:16.519ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10272024-02-28T22:13:16.519Z 0 000-019 8 8 1 16 16 1 F F F <---
10282024-02-28T22:13:16.519Z 1 020-039 8 8 1 16 16 1 F F F <---
10292024-02-28T22:13:16.519Z 2 040-059 5 5 5 10 10 10 F F F
10302024-02-28T22:13:16.519Z 3 060-079 8 8 6 13 13 11 F F F <---
10312024-02-28T22:13:16.519Z 4 080-099 1 1 1 1 1 1 F F F
10322024-02-28T22:13:16.519Z 5 100-119 6 6 6 11 11 11 F F F
10332024-02-28T22:13:16.520Z 6 120-139 8 8 6 13 13 11 F F F <---
10342024-02-28T22:13:16.520Z 7 140-159 1 1 1 1 1 1 F F F
10352024-02-28T22:13:16.520Z 8 160-179 4 4 4 6 6 6 F F F
10362024-02-28T22:13:16.520Z 9 180-199 1 1 1 1 1 1 F F F
10372024-02-28T22:13:16.520Z 10 200-219 1 1 1 1 1 1 F F F
10382024-02-28T22:13:16.520Z 11 220-239 6 6 6 11 11 11 F F F
10392024-02-28T22:13:16.520Z 12 240-259 8 8 6 14 14 11 F F F <---
10402024-02-28T22:13:16.520Z 13 260-279 1 1 1 1 1 1 F F F
10412024-02-28T22:13:16.520Z 14 280-299 5 5 5 10 10 10 F F F
10422024-02-28T22:13:16.520Z 15 300-319 6 6 6 11 11 11 T T F <---
10432024-02-28T22:13:16.521Z 16 320-339 8 8 6 14 14 11 F F F <---
10442024-02-28T22:13:16.521Z 17 340-359 8 8 4 16 16 8 F F F <---
10452024-02-28T22:13:16.521Z 18 360-379 8 8 4 13 13 8 F F F <---
10462024-02-28T22:13:16.521Z 19 380-399 8 8 6 13 13 11 F F F <---
10472024-02-28T22:13:16.521Z 20 400-419 6 6 6 11 11 11 F F F
10482024-02-28T22:13:16.521Z 21 420-439 2 2 2 4 4 4 F F F
10492024-02-28T22:13:16.522Z 22 440-459 6 6 6 11 11 11 F F F
10502024-02-28T22:13:16.522Z 23 460-479 8 8 6 13 13 11 F F F <---
10512024-02-28T22:13:16.522Z 24 480-499 8 8 6 15 15 11 F F F <---
10522024-02-28T22:13:16.522Z 25 500-519 4 4 4 7 7 7 F F F
10532024-02-28T22:13:16.522Z 26 520-539 3 3 3 5 5 5 F F F
10542024-02-28T22:13:16.536Z 27 540-559 6 6 6 11 11 11 T T F <---
10552024-02-28T22:13:16.536Z 28 560-579 8 8 1 13 13 1 T T F <---
10562024-02-28T22:13:16.536Z 29 580-599 6 6 6 11 11 11 F F F
10572024-02-28T22:13:16.536ZMax gen: 8, Max flush: 16
10582024-02-28T22:13:16.537ZError: Difference in extent metadata found!
10592024-02-28T22:13:16.537ZA Difference in extent metadata is expected here
10602024-02-28T22:13:16.537ZOn loop 4
10612024-02-28T22:13:16.537Z
10622024-02-28T22:13:16.537ZVerifying data now
10632024-02-28T22:13:21.486ZNo change after verify
10642024-02-28T22:13:21.486ZLoop: 4 Downstairs dump after verify (and repair):
10652024-02-28T22:13:21.505ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10662024-02-28T22:13:21.505Z 0 000-019 8 8 8 16 16 16 F F F
10672024-02-28T22:13:21.506Z 1 020-039 8 8 8 16 16 16 F F F
10682024-02-28T22:13:21.506Z 2 040-059 5 5 5 10 10 10 F F F
10692024-02-28T22:13:21.506Z 3 060-079 8 8 8 13 13 13 F F F
10702024-02-28T22:13:21.506Z 4 080-099 1 1 1 1 1 1 F F F
10712024-02-28T22:13:21.506Z 5 100-119 6 6 6 11 11 11 F F F
10722024-02-28T22:13:21.507Z 6 120-139 8 8 8 13 13 13 F F F
10732024-02-28T22:13:21.507Z 7 140-159 1 1 1 1 1 1 F F F
10742024-02-28T22:13:21.507Z 8 160-179 4 4 4 6 6 6 F F F
10752024-02-28T22:13:21.507Z 9 180-199 1 1 1 1 1 1 F F F
10762024-02-28T22:13:21.507Z 10 200-219 1 1 1 1 1 1 F F F
10772024-02-28T22:13:21.507Z 11 220-239 6 6 6 11 11 11 F F F
10782024-02-28T22:13:21.507Z 12 240-259 8 8 8 14 14 14 F F F
10792024-02-28T22:13:21.508Z 13 260-279 1 1 1 1 1 1 F F F
10802024-02-28T22:13:21.508Z 14 280-299 5 5 5 10 10 10 F F F
10812024-02-28T22:13:21.508Z 15 300-319 9 9 9 17 17 17 F F F
10822024-02-28T22:13:21.508Z 16 320-339 8 8 8 14 14 14 F F F
10832024-02-28T22:13:21.508Z 17 340-359 8 8 8 16 16 16 F F F
10842024-02-28T22:13:21.508Z 18 360-379 8 8 8 13 13 13 F F F
10852024-02-28T22:13:21.508Z 19 380-399 8 8 8 13 13 13 F F F
10862024-02-28T22:13:21.509Z 20 400-419 6 6 6 11 11 11 F F F
10872024-02-28T22:13:21.509Z 21 420-439 2 2 2 4 4 4 F F F
10882024-02-28T22:13:21.509Z 22 440-459 6 6 6 11 11 11 F F F
10892024-02-28T22:13:21.509Z 23 460-479 8 8 8 13 13 13 F F F
10902024-02-28T22:13:21.509Z 24 480-499 8 8 8 15 15 15 F F F
10912024-02-28T22:13:21.509Z 25 500-519 4 4 4 7 7 7 F F F
10922024-02-28T22:13:21.509Z 26 520-539 3 3 3 5 5 5 F F F
10932024-02-28T22:13:21.510Z 27 540-559 9 9 9 17 17 17 F F F
10942024-02-28T22:13:21.510Z 28 560-579 9 9 9 17 17 17 F F F
10952024-02-28T22:13:21.510Z 29 580-599 6 6 6 11 11 11 F F F
10962024-02-28T22:13:21.510ZMax gen: 9, Max flush: 17
10972024-02-28T22:13:21.571Z
10982024-02-28T22:13:21.571ZBegin loop 5
10992024-02-28T22:13:21.571ZDownstairs to restart: 1
11002024-02-28T22:13:21.571Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.421794124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
11012024-02-28T22:13:21.571Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422081466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
11022024-02-28T22:13:21.572Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422099486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
11032024-02-28T22:13:21.572Z{"msg":"Crucible 00580385-687f-408b-b604-32da3c0b4503 has session id: 4dd83071-3e3e-4b8f-ad4a-2ef35da60573","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422116695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11042024-02-28T22:13:21.572ZCrucible runtime is spawned
11052024-02-28T22:13:21.572Z{"msg":"Upstairs opts: Upstairs UUID: 00580385-687f-408b-b604-32da3c0b4503, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422175254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11062024-02-28T22:13:21.572Z{"msg":"Crucible stats registered with UUID: 00580385-687f-408b-b604-32da3c0b4503","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422210843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11072024-02-28T22:13:21.572Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422227002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
11082024-02-28T22:13:21.572Z{"msg":"00580385-687f-408b-b604-32da3c0b4503 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422282681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11092024-02-28T22:13:21.572Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.42231864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11102024-02-28T22:13:21.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422342239Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11112024-02-28T22:13:21.573Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422370588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11122024-02-28T22:13:21.573Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422390158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11132024-02-28T22:13:21.574Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422433067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11142024-02-28T22:13:21.574Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422477255Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11152024-02-28T22:13:21.574Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422515014Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11162024-02-28T22:13:21.574Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422545544Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11172024-02-28T22:13:21.575Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422581603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11182024-02-28T22:13:21.575Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422759278Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11192024-02-28T22:13:21.575Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422797637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11202024-02-28T22:13:21.575Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.422824876Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"io task","client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11212024-02-28T22:13:21.575Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.423547316Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11222024-02-28T22:13:21.575Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.423590925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11232024-02-28T22:13:21.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.423624454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11242024-02-28T22:13:21.576Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.423666193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11252024-02-28T22:13:21.576Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424169819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11262024-02-28T22:13:21.577Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424310055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11272024-02-28T22:13:21.577Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424433441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11282024-02-28T22:13:21.577Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424455241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11292024-02-28T22:13:21.577Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424720574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11302024-02-28T22:13:21.577Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424758823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11312024-02-28T22:13:21.578Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424775562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11322024-02-28T22:13:21.578Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.42483627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11332024-02-28T22:13:21.578Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424878859Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11342024-02-28T22:13:21.578Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.424896899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11352024-02-28T22:13:21.578Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425138332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11362024-02-28T22:13:21.578Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.42559882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11372024-02-28T22:13:21.578Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425633939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11382024-02-28T22:13:21.578Z{"msg":"[0]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 11, 13, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425650328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11392024-02-28T22:13:21.578Z{"msg":"[0]R generation[0..12]: [8, 8, 5, 8, 1, 6, 8, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425672267Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11402024-02-28T22:13:21.578Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425687297Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11412024-02-28T22:13:21.579Z{"msg":"[1]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 11, 13, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425707877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11422024-02-28T22:13:21.579Z{"msg":"[1]R generation[0..12]: [8, 8, 5, 8, 1, 6, 8, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425721706Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11432024-02-28T22:13:21.579Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425744336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11442024-02-28T22:13:21.579Z{"msg":"[2]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 11, 13, 1, 6, 1, 1, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425758605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11452024-02-28T22:13:21.579Z{"msg":"[2]R generation[0..12]: [8, 8, 5, 8, 1, 6, 8, 1, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425778915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11462024-02-28T22:13:21.579Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425793344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11472024-02-28T22:13:21.579Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425824563Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
11482024-02-28T22:13:21.580Zdownstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11492024-02-28T22:13:21.580Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425856062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11502024-02-28T22:13:21.580Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425890892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11512024-02-28T22:13:21.580Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425906721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"":"downstairs","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11522024-02-28T22:13:21.580Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425920731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11532024-02-28T22:13:21.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.42594008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"0","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11542024-02-28T22:13:21.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.425969409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"1","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11552024-02-28T22:13:21.580Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426007558Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"client":"2","":"downstairs","session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11562024-02-28T22:13:21.581Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426047797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11572024-02-28T22:13:21.581Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426077416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11582024-02-28T22:13:21.581Z{"msg":"00580385-687f-408b-b604-32da3c0b4503 is now active with session: 4dd83071-3e3e-4b8f-ad4a-2ef35da60573","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426091846Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11592024-02-28T22:13:21.581Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426117395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175,"session_id":"4dd83071-3e3e-4b8f-ad4a-2ef35da60573"}
11602024-02-28T22:13:21.581Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:20.426152814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
11612024-02-28T22:13:21.581ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11622024-02-28T22:13:21.582ZLoading write count information from file "/tmp/test_repair_verify.data"
11632024-02-28T22:13:21.582ZRun Repair workload
11642024-02-28T22:13:21.582Z01/30 Write block 324 len 8192 data: 3 3
11652024-02-28T22:13:21.582Z02/30 Read block 78 len 16384
11662024-02-28T22:13:21.582Z03/30 Write block 474 len 12288 data: 3 4 4
11672024-02-28T22:13:21.582Z04/30 Write block 131 len 12288 data: 3 3 3
11682024-02-28T22:13:21.582Z05/30 Read block 189 len 20480
11692024-02-28T22:13:21.582Z06/30 Write block 498 len 28672 data: 3 3 4 4 3 3 3
11702024-02-28T22:13:21.583Z07/30 Read block 249 len 28672
11712024-02-28T22:13:21.605Z08/30 Read block 570 len 8192
11722024-02-28T22:13:21.605Z09/30 Write block 211 len 8192 data: 2 2
11732024-02-28T22:13:21.606Z10/30 Read block 222 len 20480
11742024-02-28T22:13:21.606Z11/30 Write block 393 len 36864 data: 2 2 3 3 3 3 3 3 3
11752024-02-28T22:13:21.606Z12/30 Read block 313 len 32768
11762024-02-28T22:13:21.606Z13/30 Read block 455 len 4096
11772024-02-28T22:13:21.606Z14/30 Read block 551 len 16384
11782024-02-28T22:13:21.606Z15/30 Read block 213 len 32768
11792024-02-28T22:13:21.606Z16/30 Flush
11802024-02-28T22:13:21.606Z17/30 Write block 115 len 28672 data: 2 2 2 2 2 2 3
11812024-02-28T22:13:21.606Z18/30 Write block 257 len 4096 data: 2
11822024-02-28T22:13:21.606Z19/30 Read block 114 len 4096
11832024-02-28T22:13:21.606Z20/30 Write block 244 len 8192 data: 3 2
11842024-02-28T22:13:21.607Z21/30 Read block 285 len 16384
11852024-02-28T22:13:21.607Z22/30 Read block 468 len 28672
11862024-02-28T22:13:21.607Z23/30 Write block 518 len 28672 data: 2 2 2 2 2 2 2
11872024-02-28T22:13:21.607Z24/30 Write block 108 len 40960 data: 5 4 3 3 3 2 2 3 3 3
11882024-02-28T22:13:21.607Z25/30 Read block 139 len 32768
11892024-02-28T22:13:21.607Z26/30 Read block 496 len 24576
11902024-02-28T22:13:21.607Z27/30 Write block 352 len 24576 data: 2 2 2 3 4 3
11912024-02-28T22:13:21.607Z28/30 Read block 375 len 40960
11922024-02-28T22:13:21.607Z29/30 Read block 350 len 4096
11932024-02-28T22:13:21.607Z30/30 Write block 451 len 16384 data: 3 3 3 3
11942024-02-28T22:13:21.607Z----------------------------------------------------------------
11952024-02-28T22:13:21.607Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11962024-02-28T22:13:21.607ZStates: Active Active Active
11972024-02-28T22:13:21.608ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11982024-02-28T22:13:21.608Z 1 Acked 1000 Write 2 Done Sent Done false
11992024-02-28T22:13:21.608Z 2 Acked 1001 Read 4 Done Sent Done false
12002024-02-28T22:13:21.608Z 3 Acked 1002 Write 3 Done Sent Done false
12012024-02-28T22:13:21.608Z 4 Acked 1003 Write 3 Done Sent Done false
12022024-02-28T22:13:21.608Z 5 Acked 1004 Read 5 Done Sent Done false
12032024-02-28T22:13:21.608Z 6 Acked 1005 Write 7 Done Sent Done false
12042024-02-28T22:13:21.608Z 7 Acked 1006 Read 7 Done Sent Done false
12052024-02-28T22:13:21.608Z 8 Acked 1007 Read 2 Done Sent Done false
12062024-02-28T22:13:21.608Z 9 Acked 1008 Write 2 Done Sent Done false
12072024-02-28T22:13:21.608Z 10 Acked 1009 Read 5 Done Sent Done false
12082024-02-28T22:13:21.609Z 11 Acked 1010 Write 9 Done Sent Done false
12092024-02-28T22:13:21.609Z 12 Acked 1011 Read 8 Done Sent Done false
12102024-02-28T22:13:21.609Z 13 Acked 1012 Read 1 Done Sent Done false
12112024-02-28T22:13:21.609Z 14 Acked 1013 Read 4 Done Sent Done false
12122024-02-28T22:13:21.609Z 15 Acked 1014 Read 8 Done Sent Done false
12132024-02-28T22:13:21.609Z 16 Acked 1015 Flush 0 Done Sent Done false
12142024-02-28T22:13:21.609Z 17 Acked 1016 Write 7 Done Sent Done false
12152024-02-28T22:13:21.610Z 18 Acked 1017 Write 1 Done Sent Done false
12162024-02-28T22:13:21.610Z 19 Acked 1018 Read 1 Done Sent Done false
12172024-02-28T22:13:21.610Z 20 Acked 1019 Write 2 Done Sent Done false
12182024-02-28T22:13:21.610Z 21 Acked 1020 Read 4 Done Sent Done false
12192024-02-28T22:13:21.610Z 22 Acked 1021 Read 7 Done Sent Done false
12202024-02-28T22:13:21.610Z 23 Acked 1022 Write 7 Done Sent Done false
12212024-02-28T22:13:21.611Z 24 Acked 1023 Write 10 Done Sent Done false
12222024-02-28T22:13:21.611Z 25 Acked 1024 Read 8 Done Sent Done false
12232024-02-28T22:13:21.611Z 26 Acked 1025 Read 6 Done Sent Done false
12242024-02-28T22:13:21.611Z 27 Acked 1026 Write 6 Done Sent Done false
12252024-02-28T22:13:21.611Z 28 Acked 1027 Read 10 Done Sent Done false
12262024-02-28T22:13:21.611Z 29 Acked 1028 Read 1 Done Sent Done false
12272024-02-28T22:13:21.612Z 30 Acked 1029 Write 4 Sent Sent Sent false
12282024-02-28T22:13:21.612Z STATES DS:0 DS:1 DS:2 TOTAL
12292024-02-28T22:13:21.612Z New 0 0 0 0
12302024-02-28T22:13:21.612Z Sent 1 30 1 32
12312024-02-28T22:13:21.612Z Done 29 0 29 58
12322024-02-28T22:13:21.612Z Skipped 0 0 0 0
12332024-02-28T22:13:21.613Z Error 0 0 0 0
12342024-02-28T22:13:21.613ZLast Flush: 1015 0 1015
12352024-02-28T22:13:21.613ZDownstairs last five completed:
12362024-02-28T22:13:21.613ZUpstairs last five completed: 30 29 28 27 26
12372024-02-28T22:13:21.613Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:20.455107756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1175}
12382024-02-28T22:13:21.613ZWrote out file "/tmp/test_repair_verify.data"
12392024-02-28T22:13:21.613Z
12402024-02-28T22:13:21.613ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12412024-02-28T22:13:21.613ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12422024-02-28T22:13:21.613Z 0 000-019 8 8 8 16 16 16 F F F
12432024-02-28T22:13:21.614Z 1 020-039 8 8 8 16 16 16 F F F
12442024-02-28T22:13:21.614Z 2 040-059 5 5 5 10 10 10 F F F
12452024-02-28T22:13:21.614Z 3 060-079 8 8 8 13 13 13 F F F
12462024-02-28T22:13:21.614Z 4 080-099 1 1 1 1 1 1 F F F
12472024-02-28T22:13:21.614Z 5 100-119 6 6 6 11 11 11 T F T <---
12482024-02-28T22:13:21.614Z 6 120-139 10 8 10 18 13 18 T F T <---
12492024-02-28T22:13:21.614Z 7 140-159 1 1 1 1 1 1 F F F
12502024-02-28T22:13:21.614Z 8 160-179 4 4 4 6 6 6 F F F
12512024-02-28T22:13:21.614Z 9 180-199 1 1 1 1 1 1 F F F
12522024-02-28T22:13:21.614Z 10 200-219 10 1 10 18 1 18 F F F <---
12532024-02-28T22:13:21.614Z 11 220-239 6 6 6 11 11 11 F F F
12542024-02-28T22:13:21.614Z 12 240-259 8 8 8 14 14 14 T F T <---
12552024-02-28T22:13:21.614Z 13 260-279 1 1 1 1 1 1 F F F
12562024-02-28T22:13:21.614Z 14 280-299 5 5 5 10 10 10 F F F
12572024-02-28T22:13:21.614Z 15 300-319 9 9 9 17 17 17 F F F
12582024-02-28T22:13:21.614Z 16 320-339 10 8 10 18 14 18 F F F <---
12592024-02-28T22:13:21.614Z 17 340-359 8 8 8 16 16 16 T F T <---
12602024-02-28T22:13:21.614Z 18 360-379 8 8 8 13 13 13 F F F
12612024-02-28T22:13:21.614Z 19 380-399 10 8 10 18 13 18 F F F <---
12622024-02-28T22:13:21.614Z 20 400-419 10 6 10 18 11 18 F F F <---
12632024-02-28T22:13:21.614Z 21 420-439 2 2 2 4 4 4 F F F
12642024-02-28T22:13:21.614Z 22 440-459 6 6 6 11 11 11 T F T <---
12652024-02-28T22:13:21.615Z 23 460-479 10 8 10 18 13 18 F F F <---
12662024-02-28T22:13:21.615Z 24 480-499 10 8 10 18 15 18 F F F <---
12672024-02-28T22:13:21.615Z 25 500-519 10 4 10 18 7 18 T F T <---
12682024-02-28T22:13:21.615Z 26 520-539 3 3 3 5 5 5 T F T <---
12692024-02-28T22:13:21.615Z 27 540-559 9 9 9 17 17 17 F F F
12702024-02-28T22:13:21.615Z 28 560-579 9 9 9 17 17 17 F F F
12712024-02-28T22:13:21.635Z 29 580-599 6 6 6 11 11 11 F F F
12722024-02-28T22:13:21.635ZMax gen: 10, Max flush: 18
12732024-02-28T22:13:21.635ZError: Difference in extent metadata found!
12742024-02-28T22:13:21.636ZA Difference in extent metadata is expected here
12752024-02-28T22:13:21.636ZOn loop 5
12762024-02-28T22:13:21.636Z
12772024-02-28T22:13:21.636ZVerifying data now
12782024-02-28T22:13:27.029ZNo change after verify
12792024-02-28T22:13:27.029ZLoop: 5 Downstairs dump after verify (and repair):
12802024-02-28T22:13:27.044ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12812024-02-28T22:13:27.044Z 0 000-019 8 8 8 16 16 16 F F F
12822024-02-28T22:13:27.044Z 1 020-039 8 8 8 16 16 16 F F F
12832024-02-28T22:13:27.044Z 2 040-059 5 5 5 10 10 10 F F F
12842024-02-28T22:13:27.044Z 3 060-079 8 8 8 13 13 13 F F F
12852024-02-28T22:13:27.044Z 4 080-099 1 1 1 1 1 1 F F F
12862024-02-28T22:13:27.044Z 5 100-119 11 11 11 19 19 19 F F F
12872024-02-28T22:13:27.044Z 6 120-139 11 11 11 19 19 19 F F F
12882024-02-28T22:13:27.044Z 7 140-159 1 1 1 1 1 1 F F F
12892024-02-28T22:13:27.044Z 8 160-179 4 4 4 6 6 6 F F F
12902024-02-28T22:13:27.044Z 9 180-199 1 1 1 1 1 1 F F F
12912024-02-28T22:13:27.044Z 10 200-219 10 10 10 18 18 18 F F F
12922024-02-28T22:13:27.044Z 11 220-239 6 6 6 11 11 11 F F F
12932024-02-28T22:13:27.044Z 12 240-259 11 11 11 19 19 19 F F F
12942024-02-28T22:13:27.044Z 13 260-279 1 1 1 1 1 1 F F F
12952024-02-28T22:13:27.044Z 14 280-299 5 5 5 10 10 10 F F F
12962024-02-28T22:13:27.044Z 15 300-319 9 9 9 17 17 17 F F F
12972024-02-28T22:13:27.044Z 16 320-339 10 10 10 18 18 18 F F F
12982024-02-28T22:13:27.048Z 17 340-359 11 11 11 19 19 19 F F F
12992024-02-28T22:13:27.048Z 18 360-379 8 8 8 13 13 13 F F F
13002024-02-28T22:13:27.048Z 19 380-399 10 10 10 18 18 18 F F F
13012024-02-28T22:13:27.048Z 20 400-419 10 10 10 18 18 18 F F F
13022024-02-28T22:13:27.048Z 21 420-439 2 2 2 4 4 4 F F F
13032024-02-28T22:13:27.048Z 22 440-459 11 11 11 19 19 19 F F F
13042024-02-28T22:13:27.048Z 23 460-479 10 10 10 18 18 18 F F F
13052024-02-28T22:13:27.048Z 24 480-499 10 10 10 18 18 18 F F F
13062024-02-28T22:13:27.048Z 25 500-519 11 11 11 19 19 19 F F F
13072024-02-28T22:13:27.048Z 26 520-539 11 11 11 19 19 19 F F F
13082024-02-28T22:13:27.048Z 27 540-559 9 9 9 17 17 17 F F F
13092024-02-28T22:13:27.048Z 28 560-579 9 9 9 17 17 17 F F F
13102024-02-28T22:13:27.048Z 29 580-599 6 6 6 11 11 11 F F F
13112024-02-28T22:13:27.048ZMax gen: 11, Max flush: 19
13122024-02-28T22:13:27.048Z
13132024-02-28T22:13:27.049ZBegin loop 6
13142024-02-28T22:13:27.049ZDownstairs to restart: 1
13152024-02-28T22:13:27.066Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.96881115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
13162024-02-28T22:13:27.066Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969073232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
13172024-02-28T22:13:27.072Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969112471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
13182024-02-28T22:13:27.072Z{Crucible runtime is spawned
13192024-02-28T22:13:27.072Z"msg":"Crucible 3718b755-3099-4df7-a94e-60f893656396 has session id: 7ae6478e-92b6-400c-b173-55ed880434bc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.96915662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13202024-02-28T22:13:27.073Z{"msg":"Upstairs opts: Upstairs UUID: 3718b755-3099-4df7-a94e-60f893656396, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969195809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13212024-02-28T22:13:27.073Z{"msg":"Crucible stats registered with UUID: 3718b755-3099-4df7-a94e-60f893656396","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969212748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13222024-02-28T22:13:27.073Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969241718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
13232024-02-28T22:13:27.073Z{"msg":"3718b755-3099-4df7-a94e-60f893656396 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969335725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13242024-02-28T22:13:27.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969367654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13252024-02-28T22:13:27.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969395054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13262024-02-28T22:13:27.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969454072Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13272024-02-28T22:13:27.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969483631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13282024-02-28T22:13:27.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969547989Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13292024-02-28T22:13:27.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969587258Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13302024-02-28T22:13:27.073Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969661436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13312024-02-28T22:13:27.073Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969732284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13322024-02-28T22:13:27.073Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969781363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13332024-02-28T22:13:27.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969802462Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13342024-02-28T22:13:27.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969836281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13352024-02-28T22:13:27.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.969857521Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"io task","client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13362024-02-28T22:13:27.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970420325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13372024-02-28T22:13:27.073Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970455454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13382024-02-28T22:13:27.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970471664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13392024-02-28T22:13:27.073Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970486253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13402024-02-28T22:13:27.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970507523Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13412024-02-28T22:13:27.074Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.970522023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13422024-02-28T22:13:27.074Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971053488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13432024-02-28T22:13:27.074Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971103007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13442024-02-28T22:13:27.074Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971123606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13452024-02-28T22:13:27.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971650831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13462024-02-28T22:13:27.074Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.97169422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13472024-02-28T22:13:27.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971721449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13482024-02-28T22:13:27.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971753298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13492024-02-28T22:13:27.074Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971769388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13502024-02-28T22:13:27.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971784818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13512024-02-28T22:13:27.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971797567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13522024-02-28T22:13:27.074Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971811147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13532024-02-28T22:13:27.074Z{"msg":"[0]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 19, 19, 1, 6, 1, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971825196Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13542024-02-28T22:13:27.074Z{"msg":"[0]R generation[0..12]: [8, 8, 5, 8, 1, 11, 11, 1, 4, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971846236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13552024-02-28T22:13:27.074Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971860846Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13562024-02-28T22:13:27.074Z{"msg":"[1]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 19, 19, 1, 6, 1, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971876405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13572024-02-28T22:13:27.074Z{"msg":"[1]R generation[0..12]: [8, 8, 5, 8, 1, 11, 11, 1, 4, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971889965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13582024-02-28T22:13:27.074Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971911054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13592024-02-28T22:13:27.074Z{"msg":"[2]R flush_numbers[0..12]: [16, 16, 10, 13, 1, 19, 19, 1, 6, 1, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971925504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13602024-02-28T22:13:27.074Z{"msg":"[2]R generation[0..12]: [8, 8, 5, 8, 1, 11, 11, 1, 4, 1, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971946553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13612024-02-28T22:13:27.074Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13622024-02-28T22:13:27.074Z,"time":"2024-02-28T22:13:25.971961113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13632024-02-28T22:13:27.075Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.971984422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13642024-02-28T22:13:27.075Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972017301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13652024-02-28T22:13:27.075Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972033131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13662024-02-28T22:13:27.075Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.97204744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"":"downstairs","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13672024-02-28T22:13:27.075Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.97206125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13682024-02-28T22:13:27.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972092019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"0","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13692024-02-28T22:13:27.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972111039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"1","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13702024-02-28T22:13:27.075Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972132418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"client":"2","":"downstairs","session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13712024-02-28T22:13:27.075Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972146228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13722024-02-28T22:13:27.075Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972159577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13732024-02-28T22:13:27.075Z{"msg":"3718b755-3099-4df7-a94e-60f893656396 is now active with session: 7ae6478e-92b6-400c-b173-55ed880434bc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972172657Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13742024-02-28T22:13:27.075Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972184677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183,"session_id":"7ae6478e-92b6-400c-b173-55ed880434bc"}
13752024-02-28T22:13:27.075Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:25.972197586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
13762024-02-28T22:13:27.079ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13772024-02-28T22:13:27.079ZLoading write count information from file "/tmp/test_repair_verify.data"
13782024-02-28T22:13:27.079ZRun Repair workload
13792024-02-28T22:13:27.079Z01/30 Write block 471 len 28672 data: 2 2 2 4 5 5 3
13802024-02-28T22:13:27.079Z02/30 Write block 334 len 20480 data: 3 3 2 2 2
13812024-02-28T22:13:27.079Z03/30 Read block 58 len 28672
13822024-02-28T22:13:27.079Z04/30 Read block 244 len 8192
13832024-02-28T22:13:27.079Z05/30 Write block 192 len 4096 data: 2
13842024-02-28T22:13:27.079Z06/30 Write block 430 len 36864 data: 2 2 2 2 2 2 2 2 2
13852024-02-28T22:13:27.079Z07/30 Write block 120 len 24576 data: 3 4 3 3 3 3
13862024-02-28T22:13:27.079Z08/30 Read block 569 len 24576
13872024-02-28T22:13:27.079Z09/30 Read block 336 len 28672
13882024-02-28T22:13:27.079Z10/30 Read block 297 len 16384
13892024-02-28T22:13:27.082Z11/30 Write block 251 len 4096 data: 3
13902024-02-28T22:13:27.082Z12/30 Flush
13912024-02-28T22:13:27.111Z13/30 Write block 258 len 20480 data: 2 2 2 2 2
13922024-02-28T22:13:27.115Z14/30 Read block 260 len 24576
13932024-02-28T22:13:27.115Z15/30 Write block 4 len 20480 data: 2 2 2 2 2
13942024-02-28T22:13:27.115Z16/30 Write block 405 len 24576 data: 2 2 3 3 3 3
13952024-02-28T22:13:27.118Z17/30 Flush
13962024-02-28T22:13:27.121Z18/30 Write block 2 len 8192 data: 2 2
13972024-02-28T22:13:27.121Z19/30 Write block 111 len 8192 data: 4 4
13982024-02-28T22:13:27.121Z20/30 Write block 56 len 4096 data: 3
13992024-02-28T22:13:27.121Z21/30 Flush
14002024-02-28T22:13:27.156Z22/30 Write block 429 len 40960 data: 2 3 3 3 3 3 3 3 3 3
14012024-02-28T22:13:27.165Z23/30 Write block 245 len 20480 data: 3 2 3 3 3
14022024-02-28T22:13:27.165Z24/30 Read block 59 len 24576
14032024-02-28T22:13:27.165Z25/30 Read block 330 len 20480
14042024-02-28T22:13:27.169Z26/30 Write block 262 len 16384 data: 3 2 2 2
14052024-02-28T22:13:27.169Z27/30 Write block 19 len 12288 data: 3 3 2
14062024-02-28T22:13:27.169Z28/30 Write block 37 len 20480 data: 2 2 2 2 2
14072024-02-28T22:13:27.169Z29/30 Read block 320 len 36864
14082024-02-28T22:13:27.169Z30/30 Read block 592 len 12288
14092024-02-28T22:13:27.169Z----------------------------------------------------------------
14102024-02-28T22:13:27.169Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14112024-02-28T22:13:27.169ZStates: Active Active Active
14122024-02-28T22:13:27.169ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14132024-02-28T22:13:27.169Z 1 Acked 1000 Write 7 Done Done Done false
14142024-02-28T22:13:27.169Z 2 Acked 1001 Write 5 Done Done Done false
14152024-02-28T22:13:27.169Z 3 Acked 1002 Read 7 Done Done Done false
14162024-02-28T22:13:27.170Z 4 Acked 1003 Read 2 Done Sent Done false
14172024-02-28T22:13:27.170Z 5 Acked 1004 Write 1 Done Sent Done false
14182024-02-28T22:13:27.170Z 6 Acked 1005 Write 9 Done Sent Done false
14192024-02-28T22:13:27.170Z 7 Acked 1006 Write 6 Done Sent Done false
14202024-02-28T22:13:27.170Z 8 Acked 1007 Read 6 Done Sent Done false
14212024-02-28T22:13:27.170Z 9 Acked 1008 Read 7 Done Sent Done false
14222024-02-28T22:13:27.170Z 10 Acked 1009 Read 4 Done Sent Done false
14232024-02-28T22:13:27.170Z 11 Acked 1010 Write 1 Done Sent Done false
14242024-02-28T22:13:27.170Z 12 Acked 1011 Flush 0 Done Sent Done false
14252024-02-28T22:13:27.170Z 13 Acked 1012 Write 5 Done Sent Done false
14262024-02-28T22:13:27.170Z 14 Acked 1013 Read 6 Done Sent Done false
14272024-02-28T22:13:27.170Z 15 Acked 1014 Write 5 Done Sent Done false
14282024-02-28T22:13:27.170Z 16 Acked 1015 Write 6 Done Sent Done false
14292024-02-28T22:13:27.170Z 17 Acked 1016 Flush 0 Done Sent Done false
14302024-02-28T22:13:27.170Z 18 Acked 1017 Write 2 Done Sent Done false
14312024-02-28T22:13:27.170Z 19 Acked 1018 Write 2 Done Sent Done false
14322024-02-28T22:13:27.170Z 20 Acked 1019 Write 1 Done Sent Done false
14332024-02-28T22:13:27.170Z 21 Acked 1020 Flush 0 Done Sent Done false
14342024-02-28T22:13:27.170Z 22 Acked 1021 Write 10 Done Sent Done false
14352024-02-28T22:13:27.170Z 23 Acked 1022 Write 5 Done Sent Done false
14362024-02-28T22:13:27.170Z 24 Acked 1023 Read 6 Done Sent Done false
14372024-02-28T22:13:27.170Z 25 Acked 1024 Read 5 Done Sent Done false
14382024-02-28T22:13:27.170Z 26 Acked 1025 Write 4 Done Sent Done false
14392024-02-28T22:13:27.171Z 27 Acked 1026 Write 3 Done Sent Done false
14402024-02-28T22:13:27.171Z 28 Acked 1027 Write 5 Done Sent Done false
14412024-02-28T22:13:27.171Z 29 Acked 1028 Read 9 Done Sent Done false
14422024-02-28T22:13:27.171Z 30 Acked 1029 Read 3 Done Sent Done false
14432024-02-28T22:13:27.171Z STATES DS:0 DS:1 DS:2 TOTAL
14442024-02-28T22:13:27.171Z New 0 0 0 0
14452024-02-28T22:13:27.171Z Sent 0 27 0 27
14462024-02-28T22:13:27.171Z Done 30 3 30 63
14472024-02-28T22:13:27.171Z Skipped 0 0 0 0
14482024-02-28T22:13:27.171Z Error 0 0 0 0
14492024-02-28T22:13:27.171ZLast Flush: 1020 0 1020
14502024-02-28T22:13:27.171ZDownstairs last five completed:
14512024-02-28T22:13:27.171ZUpstairs last five completed: 30 29 28 27 26
14522024-02-28T22:13:27.171Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:26.066165117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1183}
14532024-02-28T22:13:27.171ZWrote out file "/tmp/test_repair_verify.data"
14542024-02-28T22:13:27.171Z
14552024-02-28T22:13:27.175ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14562024-02-28T22:13:27.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14572024-02-28T22:13:27.188Z 0 000-019 12 8 12 22 16 22 T F T <---
14582024-02-28T22:13:27.188Z 1 020-039 8 8 8 16 16 16 T F T <---
14592024-02-28T22:13:27.188Z 2 040-059 12 5 12 22 10 22 T F T <---
14602024-02-28T22:13:27.188Z 3 060-079 8 8 8 13 13 13 F F F
14612024-02-28T22:13:27.188Z 4 080-099 1 1 1 1 1 1 F F F
14622024-02-28T22:13:27.189Z 5 100-119 12 11 12 22 19 22 F F F <---
14632024-02-28T22:13:27.189Z 6 120-139 12 11 12 20 19 20 F F F <---
14642024-02-28T22:13:27.189Z 7 140-159 1 1 1 1 1 1 F F F
14652024-02-28T22:13:27.189Z 8 160-179 4 4 4 6 6 6 F F F
14662024-02-28T22:13:27.189Z 9 180-199 12 1 12 20 1 20 F F F <---
14672024-02-28T22:13:27.189Z 10 200-219 10 10 10 18 18 18 F F F
14682024-02-28T22:13:27.189Z 11 220-239 6 6 6 11 11 11 F F F
14692024-02-28T22:13:27.189Z 12 240-259 12 11 12 21 19 21 T F T <---
14702024-02-28T22:13:27.189Z 13 260-279 12 1 12 21 1 21 T F T <---
14712024-02-28T22:13:27.192Z 14 280-299 5 5 5 10 10 10 F F F
14722024-02-28T22:13:27.192Z 15 300-319 9 9 9 17 17 17 F F F
14732024-02-28T22:13:27.192Z 16 320-339 12 10 12 20 18 20 F T F <---
14742024-02-28T22:13:27.192Z 17 340-359 11 11 11 19 19 19 F F F
14752024-02-28T22:13:27.192Z 18 360-379 8 8 8 13 13 13 F F F
14762024-02-28T22:13:27.192Z 19 380-399 10 10 10 18 18 18 F F F
14772024-02-28T22:13:27.192Z 20 400-419 12 10 12 21 18 21 F F F <---
14782024-02-28T22:13:27.192Z 21 420-439 12 2 12 20 4 20 T F T <---
14792024-02-28T22:13:27.192Z 22 440-459 11 11 11 19 19 19 F F F
14802024-02-28T22:13:27.192Z 23 460-479 12 10 12 20 18 20 F T F <---
14812024-02-28T22:13:27.192Z 24 480-499 10 10 10 18 18 18 F F F
14822024-02-28T22:13:27.193Z 25 500-519 11 11 11 19 19 19 F F F
14832024-02-28T22:13:27.193Z 26 520-539 11 11 11 19 19 19 F F F
14842024-02-28T22:13:27.193Z 27 540-559 9 9 9 17 17 17 F F F
14852024-02-28T22:13:27.193Z 28 560-579 9 9 9 17 17 17 F F F
14862024-02-28T22:13:27.193Z 29 580-599 6 6 6 11 11 11 F F F
14872024-02-28T22:13:27.193ZMax gen: 12, Max flush: 22
14882024-02-28T22:13:27.193ZError: Difference in extent metadata found!
14892024-02-28T22:13:27.193ZA Difference in extent metadata is expected here
14902024-02-28T22:13:27.193ZOn loop 6
14912024-02-28T22:13:27.193Z
14922024-02-28T22:13:27.199ZVerifying data now
14932024-02-28T22:13:33.341ZNo change after verify
14942024-02-28T22:13:33.342ZLoop: 6 Downstairs dump after verify (and repair):
14952024-02-28T22:13:33.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14962024-02-28T22:13:33.365Z 0 000-019 13 13 13 23 23 23 F F F
14972024-02-28T22:13:33.365Z 1 020-039 13 13 13 23 23 23 F F F
14982024-02-28T22:13:33.365Z 2 040-059 13 13 13 23 23 23 F F F
14992024-02-28T22:13:33.365Z 3 060-079 8 8 8 13 13 13 F F F
15002024-02-28T22:13:33.365Z 4 080-099 1 1 1 1 1 1 F F F
15012024-02-28T22:13:33.365Z 5 100-119 12 12 12 22 22 22 F F F
15022024-02-28T22:13:33.366Z 6 120-139 12 12 12 20 20 20 F F F
15032024-02-28T22:13:33.366Z 7 140-159 1 1 1 1 1 1 F F F
15042024-02-28T22:13:33.366Z 8 160-179 4 4 4 6 6 6 F F F
15052024-02-28T22:13:33.366Z 9 180-199 12 12 12 20 20 20 F F F
15062024-02-28T22:13:33.366Z 10 200-219 10 10 10 18 18 18 F F F
15072024-02-28T22:13:33.366Z 11 220-239 6 6 6 11 11 11 F F F
15082024-02-28T22:13:33.366Z 12 240-259 13 13 13 23 23 23 F F F
15092024-02-28T22:13:33.366Z 13 260-279 13 13 13 23 23 23 F F F
15102024-02-28T22:13:33.366Z 14 280-299 5 5 5 10 10 10 F F F
15112024-02-28T22:13:33.366Z 15 300-319 9 9 9 17 17 17 F F F
15122024-02-28T22:13:33.369Z 16 320-339 12 12 12 20 20 20 F F F
15132024-02-28T22:13:33.369Z 17 340-359 11 11 11 19 19 19 F F F
15142024-02-28T22:13:33.369Z 18 360-379 8 8 8 13 13 13 F F F
15152024-02-28T22:13:33.369Z 19 380-399 10 10 10 18 18 18 F F F
15162024-02-28T22:13:33.370Z 20 400-419 12 12 12 21 21 21 F F F
15172024-02-28T22:13:33.370Z 21 420-439 13 13 13 23 23 23 F F F
15182024-02-28T22:13:33.370Z 22 440-459 11 11 11 19 19 19 F F F
15192024-02-28T22:13:33.370Z 23 460-479 12 12 12 20 20 20 F F F
15202024-02-28T22:13:33.370Z 24 480-499 10 10 10 18 18 18 F F F
15212024-02-28T22:13:33.370Z 25 500-519 11 11 11 19 19 19 F F F
15222024-02-28T22:13:33.370Z 26 520-539 11 11 11 19 19 19 F F F
15232024-02-28T22:13:33.370Z 27 540-559 9 9 9 17 17 17 F F F
15242024-02-28T22:13:33.370Z 28 560-579 9 9 9 17 17 17 F F F
15252024-02-28T22:13:33.370Z 29 580-599 6 6 6 11 11 11 F F F
15262024-02-28T22:13:33.370ZMax gen: 13, Max flush: 23
15272024-02-28T22:13:33.370Z
15282024-02-28T22:13:33.370ZBegin loop 7
15292024-02-28T22:13:33.370ZDownstairs to restart: 2
15302024-02-28T22:13:33.378Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.279650412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
15312024-02-28T22:13:33.378Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.279910635Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
15322024-02-28T22:13:33.378Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.279947654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
15332024-02-28T22:13:33.385Z{"msg":"Crucible f83c376f-d903-466d-b0e6-f73f428ed535 has session id: 58e1a530-6ca5-418a-a110-6856271a2b62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.279973343Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15342024-02-28T22:13:33.385Z{"msg":"Upstairs opts: Upstairs UUID: f83c376f-d903-466d-b0e6-f73f428ed535, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
15352024-02-28T22:13:33.385Z,"time":"2024-02-28T22:13:32.280008582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15362024-02-28T22:13:33.385Z{"msg":"Crucible stats registered with UUID: f83c376f-d903-466d-b0e6-f73f428ed535","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280049221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15372024-02-28T22:13:33.385Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.28008936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
15382024-02-28T22:13:33.385Z{"msg":"f83c376f-d903-466d-b0e6-f73f428ed535 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280164338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15392024-02-28T22:13:33.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280223576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15402024-02-28T22:13:33.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280288844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15412024-02-28T22:13:33.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280322154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15422024-02-28T22:13:33.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280373912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15432024-02-28T22:13:33.385Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280403521Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15442024-02-28T22:13:33.385Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280472759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15452024-02-28T22:13:33.385Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280526628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15462024-02-28T22:13:33.385Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280575207Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15472024-02-28T22:13:33.385Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280610556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15482024-02-28T22:13:33.385Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280635325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15492024-02-28T22:13:33.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280660944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15502024-02-28T22:13:33.386Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.280763931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15512024-02-28T22:13:33.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.281323216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15522024-02-28T22:13:33.386Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.281358875Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15532024-02-28T22:13:33.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.281375495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15542024-02-28T22:13:33.386Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.281394684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15552024-02-28T22:13:33.386Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.281490142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15562024-02-28T22:13:33.386Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.28154856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15572024-02-28T22:13:33.386Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282040546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15582024-02-28T22:13:33.386Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282216351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15592024-02-28T22:13:33.386Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282391377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15602024-02-28T22:13:33.386Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282677149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15612024-02-28T22:13:33.386Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282732687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15622024-02-28T22:13:33.386Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282772696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15632024-02-28T22:13:33.386Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282811605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15642024-02-28T22:13:33.386Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282830985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15652024-02-28T22:13:33.386Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282852694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15662024-02-28T22:13:33.386Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282865504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15672024-02-28T22:13:33.386Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282885723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15682024-02-28T22:13:33.386Z{"msg":"[0]R flush_numbers[0..12]: [23, 23, 23, 13, 1, 22, 20, 1, 6, 20, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282899933Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15692024-02-28T22:13:33.386Z{"msg":"[0]R generation[0..12]: [13, 13, 13, 8, 1, 12, 12, 1, 4, 12, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282920522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15702024-02-28T22:13:33.386Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282935132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15712024-02-28T22:13:33.386Z{"msg":"[1]R flush_numbers[0..12]: [23, 23, 23, 13, 1, 22, 20, 1, 6, 20, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282949541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15722024-02-28T22:13:33.386Z{"msg":"[1]R generation[0..12]: [13, 13, 13, 8, 1, 12, 12, 1, 4, 12, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.282969661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15732024-02-28T22:13:33.387Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.28298415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15742024-02-28T22:13:33.387Z{"msg":"[2]R flush_numbers[0..12]: [23, 23, 23, 13, 1, 22, 20, 1, 6, 20, 18, 11]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.28301286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15752024-02-28T22:13:33.387ZWait for a query_work_queue command to finish before sending IO
15762024-02-28T22:13:33.387Z{"msg":"[2]R generation[0..12]: [13, 13, 13, 8, 1, 12, 12, 1, 4, 12, 10, 6]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283042499Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15772024-02-28T22:13:33.387Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283058568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15782024-02-28T22:13:33.387Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283090667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15792024-02-28T22:13:33.387Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283118097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15802024-02-28T22:13:33.387Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283137296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15812024-02-28T22:13:33.387Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283160865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"":"downstairs","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15822024-02-28T22:13:33.387Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283189265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15832024-02-28T22:13:33.387Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283222494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"0","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15842024-02-28T22:13:33.387Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283250863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"1","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15852024-02-28T22:13:33.387Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283282992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"client":"2","":"downstairs","session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15862024-02-28T22:13:33.387Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283315391Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15872024-02-28T22:13:33.387ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15882024-02-28T22:13:33.387Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.28334926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15892024-02-28T22:13:33.387Z{"msg":"f83c376f-d903-466d-b0e6-f73f428ed535 is now active with session: 58e1a530-6ca5-418a-a110-6856271a2b62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283380069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15902024-02-28T22:13:33.387Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283393829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191,"session_id":"58e1a530-6ca5-418a-a110-6856271a2b62"}
15912024-02-28T22:13:33.387Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:32.283412849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
15922024-02-28T22:13:33.387ZLoading write count information from file "/tmp/test_repair_verify.data"
15932024-02-28T22:13:33.387ZRun Repair workload
15942024-02-28T22:13:33.391Z01/30 Write block 205 len 24576 data: 2 2 2 2 2 2
15952024-02-28T22:13:33.391Z02/30 Write block 228 len 28672 data: 4 5 5 5 6 4 4
15962024-02-28T22:13:33.391Z03/30 Read block 354 len 16384
15972024-02-28T22:13:33.391Z04/30 Read block 420 len 28672
15982024-02-28T22:13:33.391Z05/30 Read block 141 len 16384
15992024-02-28T22:13:33.391Z06/30 Write block 30 len 36864 data: 2 2 2 2 2 2 2 3 3
16002024-02-28T22:13:33.391Z07/30 Read block 306 len 4096
16012024-02-28T22:13:33.391Z08/30 Read block 430 len 32768
16022024-02-28T22:13:33.391Z09/30 Write block 546 len 8192 data: 3 3
16032024-02-28T22:13:33.392Z10/30 Write block 106 len 28672 data: 5 5 6 5 4 5 5
16042024-02-28T22:13:33.392Z11/30 Flush
16052024-02-28T22:13:33.395Z12/30 Write block 589 len 28672 data: 2 5 4 4 4 3 3
16062024-02-28T22:13:33.398Z13/30 Flush
16072024-02-28T22:13:33.429Z14/30 Write block 477 len 16384 data: 4 3 3 2
16082024-02-28T22:13:33.442Z15/30 Write block 154 len 24576 data: 2 2 2 2 2 2
16092024-02-28T22:13:33.443Z16/30 Write block 67 len 20480 data: 5 4 4 4 4
16102024-02-28T22:13:33.443Z17/30 Write block 371 len 20480 data: 2 2 2 2 2
16112024-02-28T22:13:33.443Z18/30 Flush
16122024-02-28T22:13:33.443Z19/30 Write block 87 len 24576 data: 2 2 2 2 2 2
16132024-02-28T22:13:33.443Z20/30 Write block 368 len 32768 data: 2 2 2 3 3 3 3 3
16142024-02-28T22:13:33.443Z21/30 Write block 181 len 16384 data: 2 2 2 2
16152024-02-28T22:13:33.443Z22/30 Read block 170 len 32768
16162024-02-28T22:13:33.443Z23/30 Write block 171 len 24576 data: 3 3 3 3 2 2
16172024-02-28T22:13:33.443Z24/30 Read block 24 len 40960
16182024-02-28T22:13:33.444Z25/30 Write block 302 len 32768 data: 3 3 4 3 3 3 3 2
16192024-02-28T22:13:33.456Z26/30 Read block 481 len 28672
16202024-02-28T22:13:33.456Z27/30 Read block 150 len 8192
16212024-02-28T22:13:33.456Z28/30 Write block 545 len 12288 data: 3 4 4
16222024-02-28T22:13:33.457Z29/30 Read block 150 len 32768
16232024-02-28T22:13:33.457Z30/30 Read block 274 len 40960
16242024-02-28T22:13:33.457Z----------------------------------------------------------------
16252024-02-28T22:13:33.457Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16262024-02-28T22:13:33.457ZStates: Active Active Active
16272024-02-28T22:13:33.457ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16282024-02-28T22:13:33.457Z 1 Acked 1000 Write 6 Done Done Done false
16292024-02-28T22:13:33.457Z 2 Acked 1001 Write 7 Done Done Done false
16302024-02-28T22:13:33.457Z 3 Acked 1002 Read 4 Done Done Done false
16312024-02-28T22:13:33.457Z 4 Acked 1003 Read 7 Done Done Done false
16322024-02-28T22:13:33.457Z 5 Acked 1004 Read 4 Done Done Done false
16332024-02-28T22:13:33.457Z 6 Acked 1005 Write 9 Done Done Sent false
16342024-02-28T22:13:33.457Z 7 Acked 1006 Read 1 Done Done Sent false
16352024-02-28T22:13:33.457Z 8 Acked 1007 Read 8 Done Done Sent false
16362024-02-28T22:13:33.457Z 9 Acked 1008 Write 2 Done Done Sent false
16372024-02-28T22:13:33.457Z 10 Acked 1009 Write 7 Done Done Sent false
16382024-02-28T22:13:33.457Z 11 Acked 1010 Flush 0 Done Done Sent false
16392024-02-28T22:13:33.458Z 12 Acked 1011 Write 7 Done Done Sent false
16402024-02-28T22:13:33.458Z 13 Acked 1012 Flush 0 Done Done Sent false
16412024-02-28T22:13:33.458Z 14 Acked 1013 Write 4 Done Done Sent false
16422024-02-28T22:13:33.458Z 15 Acked 1014 Write 6 Done Done Sent false
16432024-02-28T22:13:33.458Z 16 Acked 1015 Write 5 Done Done Sent false
16442024-02-28T22:13:33.458Z 17 Acked 1016 Write 5 Done Done Sent false
16452024-02-28T22:13:33.458Z 18 Acked 1017 Flush 0 Done Done Sent false
16462024-02-28T22:13:33.459Z 19 Acked 1018 Write 6 Done Done Sent false
16472024-02-28T22:13:33.459Z 20 Acked 1019 Write 8 Done Done Sent false
16482024-02-28T22:13:33.459Z 21 Acked 1020 Write 4 Done Done Sent false
16492024-02-28T22:13:33.459Z 22 Acked 1021 Read 8 Done Done Sent false
16502024-02-28T22:13:33.459Z 23 Acked 1022 Write 6 Done Done Sent false
16512024-02-28T22:13:33.459Z 24 Acked 1023 Read 10 Done Done Sent false
16522024-02-28T22:13:33.459Z 25 Acked 1024 Write 8 Done Done Sent false
16532024-02-28T22:13:33.459Z 26 Acked 1025 Read 7 Done Done Sent false
16542024-02-28T22:13:33.459Z 27 Acked 1026 Read 2 Done Done Sent false
16552024-02-28T22:13:33.459Z 28 Acked 1027 Write 3 Done Done Sent false
16562024-02-28T22:13:33.459Z 29 Acked 1028 Read 8 Done Done Sent false
16572024-02-28T22:13:33.459Z 30 Acked 1029 Read 10 Done Sent Sent false
16582024-02-28T22:13:33.459Z STATES DS:0 DS:1 DS:2 TOTAL
16592024-02-28T22:13:33.459Z New 0 0 0 0
16602024-02-28T22:13:33.459Z Sent 0 1 25 26
16612024-02-28T22:13:33.459Z Done 30 29 5 64
16622024-02-28T22:13:33.459Z Skipped 0 0 0 0
16632024-02-28T22:13:33.459Z Error 0 0 0 0
16642024-02-28T22:13:33.459ZLast Flush: 1017 1017 0
16652024-02-28T22:13:33.459ZDownstairs last five completed:
16662024-02-28T22:13:33.460ZUpstairs last five completed: 30 29 28 27 26
16672024-02-28T22:13:33.460Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:32.343097894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1191}
16682024-02-28T22:13:33.460ZWrote out file "/tmp/test_repair_verify.data"
16692024-02-28T22:13:33.460Z
16702024-02-28T22:13:33.460ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16712024-02-28T22:13:33.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16722024-02-28T22:13:33.476Z 0 000-019 13 13 13 23 23 23 F F F
16732024-02-28T22:13:33.476Z 1 020-039 14 14 13 24 24 23 F F F <---
16742024-02-28T22:13:33.476Z 2 040-059 13 13 13 23 23 23 F F F
16752024-02-28T22:13:33.477Z 3 060-079 14 14 8 26 26 13 F F F <---
16762024-02-28T22:13:33.477Z 4 080-099 1 1 1 1 1 1 T T F <---
16772024-02-28T22:13:33.477Z 5 100-119 14 14 12 24 24 22 F F F <---
16782024-02-28T22:13:33.477Z 6 120-139 12 12 12 20 20 20 F F F
16792024-02-28T22:13:33.477Z 7 140-159 14 14 1 26 26 1 F F F <---
16802024-02-28T22:13:33.477Z 8 160-179 4 4 4 6 6 6 T T F <---
16812024-02-28T22:13:33.477Z 9 180-199 12 12 12 20 20 20 T T F <---
16822024-02-28T22:13:33.477Z 10 200-219 14 14 10 24 24 18 F F T <---
16832024-02-28T22:13:33.477Z 11 220-239 14 14 6 24 24 11 F F T <---
16842024-02-28T22:13:33.478Z 12 240-259 13 13 13 23 23 23 F F F
16852024-02-28T22:13:33.478Z 13 260-279 13 13 13 23 23 23 F F F
16862024-02-28T22:13:33.478Z 14 280-299 5 5 5 10 10 10 F F F
16872024-02-28T22:13:33.478Z 15 300-319 9 9 9 17 17 17 T T F <---
16882024-02-28T22:13:33.479Z 16 320-339 12 12 12 20 20 20 F F F
16892024-02-28T22:13:33.479Z 17 340-359 11 11 11 19 19 19 F F F
16902024-02-28T22:13:33.479Z 18 360-379 14 14 8 26 26 13 T T F <---
16912024-02-28T22:13:33.479Z 19 380-399 10 10 10 18 18 18 F F F
16922024-02-28T22:13:33.479Z 20 400-419 12 12 12 21 21 21 F F F
16932024-02-28T22:13:33.479Z 21 420-439 13 13 13 23 23 23 F F F
16942024-02-28T22:13:33.479Z 22 440-459 11 11 11 19 19 19 F F F
16952024-02-28T22:13:33.479Z 23 460-479 14 14 12 26 26 20 F F F <---
16962024-02-28T22:13:33.479Z 24 480-499 14 14 10 26 26 18 F F F <---
16972024-02-28T22:13:33.479Z 25 500-519 11 11 11 19 19 19 F F F
16982024-02-28T22:13:33.479Z 26 520-539 11 11 11 19 19 19 F F F
16992024-02-28T22:13:33.480Z 27 540-559 14 14 9 24 24 17 T T F <---
17002024-02-28T22:13:33.480Z 28 560-579 9 9 9 17 17 17 F F F
17012024-02-28T22:13:33.480Z 29 580-599 14 14 6 25 25 11 F F F <---
17022024-02-28T22:13:33.480ZMax gen: 14, Max flush: 26
17032024-02-28T22:13:33.480ZError: Difference in extent metadata found!
17042024-02-28T22:13:33.503ZA Difference in extent metadata is expected here
17052024-02-28T22:13:33.503ZOn loop 7
17062024-02-28T22:13:33.503Z
17072024-02-28T22:13:33.503ZVerifying data now
17082024-02-28T22:13:39.977ZNo change after verify
17092024-02-28T22:13:39.978ZLoop: 7 Downstairs dump after verify (and repair):
17102024-02-28T22:13:39.992ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17112024-02-28T22:13:39.992Z 0 000-019 13 13 13 23 23 23 F F F
17122024-02-28T22:13:39.992Z 1 020-039 14 14 14 24 24 24 F F F
17132024-02-28T22:13:39.992Z 2 040-059 13 13 13 23 23 23 F F F
17142024-02-28T22:13:39.992Z 3 060-079 14 14 14 26 26 26 F F F
17152024-02-28T22:13:39.992Z 4 080-099 15 15 15 27 27 27 F F F
17162024-02-28T22:13:39.992Z 5 100-119 14 14 14 24 24 24 F F F
17172024-02-28T22:13:39.992Z 6 120-139 12 12 12 20 20 20 F F F
17182024-02-28T22:13:39.992Z 7 140-159 14 14 14 26 26 26 F F F
17192024-02-28T22:13:39.992Z 8 160-179 15 15 15 27 27 27 F F F
17202024-02-28T22:13:39.993Z 9 180-199 15 15 15 27 27 27 F F F
17212024-02-28T22:13:39.993Z 10 200-219 14 14 14 24 24 24 F F F
17222024-02-28T22:13:39.993Z 11 220-239 14 14 14 24 24 24 F F F
17232024-02-28T22:13:39.996Z 12 240-259 13 13 13 23 23 23 F F F
17242024-02-28T22:13:39.996Z 13 260-279 13 13 13 23 23 23 F F F
17252024-02-28T22:13:39.996Z 14 280-299 5 5 5 10 10 10 F F F
17262024-02-28T22:13:39.997Z 15 300-319 15 15 15 27 27 27 F F F
17272024-02-28T22:13:39.997Z 16 320-339 12 12 12 20 20 20 F F F
17282024-02-28T22:13:39.997Z 17 340-359 11 11 11 19 19 19 F F F
17292024-02-28T22:13:39.997Z 18 360-379 15 15 15 27 27 27 F F F
17302024-02-28T22:13:39.997Z 19 380-399 10 10 10 18 18 18 F F F
17312024-02-28T22:13:39.997Z 20 400-419 12 12 12 21 21 21 F F F
17322024-02-28T22:13:39.997Z 21 420-439 13 13 13 23 23 23 F F F
17332024-02-28T22:13:39.997Z 22 440-459 11 11 11 19 19 19 F F F
17342024-02-28T22:13:39.997Z 23 460-479 14 14 14 26 26 26 F F F
17352024-02-28T22:13:39.997Z 24 480-499 14 14 14 26 26 26 F F F
17362024-02-28T22:13:39.997Z 25 500-519 11 11 11 19 19 19 F F F
17372024-02-28T22:13:39.997Z 26 520-539 11 11 11 19 19 19 F F F
17382024-02-28T22:13:39.997Z 27 540-559 15 15 15 27 27 27 F F F
17392024-02-28T22:13:39.997Z 28 560-579 9 9 9 17 17 17 F F F
17402024-02-28T22:13:39.997Z 29 580-599 14 14 14 25 25 25 F F F
17412024-02-28T22:13:39.997ZMax gen: 15, Max flush: 27
17422024-02-28T22:13:39.997Z
17432024-02-28T22:13:39.997ZBegin loop 8
17442024-02-28T22:13:39.997ZDownstairs to restart: 1
17452024-02-28T22:13:40.013Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.914749312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
17462024-02-28T22:13:40.013Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"Crucible runtime is spawned
17472024-02-28T22:13:40.013Z80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915076163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
17482024-02-28T22:13:40.021Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915112332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
17492024-02-28T22:13:40.021Z{"msg":"Crucible 3256dfbb-0454-4f82-a591-6d6b809d84fb has session id: 648ddf12-b66a-4d17-b215-1a84ac6b0f45","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915155231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17502024-02-28T22:13:40.021Z{"msg":"Upstairs opts: Upstairs UUID: 3256dfbb-0454-4f82-a591-6d6b809d84fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915211599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17512024-02-28T22:13:40.021Z{"msg":"Crucible stats registered with UUID: 3256dfbb-0454-4f82-a591-6d6b809d84fb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915258088Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17522024-02-28T22:13:40.021Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915292407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
17532024-02-28T22:13:40.021Z{"msg":"3256dfbb-0454-4f82-a591-6d6b809d84fb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915309226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17542024-02-28T22:13:40.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915333836Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17552024-02-28T22:13:40.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915352555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17562024-02-28T22:13:40.021Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915371185Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17572024-02-28T22:13:40.021Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915401074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17582024-02-28T22:13:40.022Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915427693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17592024-02-28T22:13:40.022Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915452143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17602024-02-28T22:13:40.022Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915487702Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17612024-02-28T22:13:40.022Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.91552561Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17622024-02-28T22:13:40.022Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915566319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17632024-02-28T22:13:40.022Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915608508Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17642024-02-28T22:13:40.022Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915641007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17652024-02-28T22:13:40.022Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.915680166Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17662024-02-28T22:13:40.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916407676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17672024-02-28T22:13:40.022Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916445745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17682024-02-28T22:13:40.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916462415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17692024-02-28T22:13:40.022Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916510533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17702024-02-28T22:13:40.022Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916533723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17712024-02-28T22:13:40.023Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.916565542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17722024-02-28T22:13:40.023Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917066568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17732024-02-28T22:13:40.023Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917104047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17742024-02-28T22:13:40.023Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917149336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17752024-02-28T22:13:40.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917667311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17762024-02-28T22:13:40.023Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.91770606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17772024-02-28T22:13:40.023Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.91773228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17782024-02-28T22:13:40.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917760079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17792024-02-28T22:13:40.023Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917785308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17802024-02-28T22:13:40.023Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917814847Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17812024-02-28T22:13:40.023Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917837697Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17822024-02-28T22:13:40.023Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917870776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17832024-02-28T22:13:40.024Z{"msg":"[0]R flush_numbers[0..12]: [23, 24, 23, 26, 27, 24, 20, 26, 27, 27, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917896955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17842024-02-28T22:13:40.024ZWait for a query_work_queue command to finish before sending IO
17852024-02-28T22:13:40.024Z{"msg":"[0]R generation[0..12]: [13, 14, 13, 14, 15, 14, 12, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917930184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17862024-02-28T22:13:40.024Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917956983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17872024-02-28T22:13:40.024Z{"msg":"[1]R flush_numbers[0..12]: [23, 24, 23, 26, 27, 24, 20, 26, 27, 27, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.917989802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17882024-02-28T22:13:40.024Z{"msg":"[1]R generation[0..12]: [13, 14, 13, 14, 15, 14, 12, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918014892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17892024-02-28T22:13:40.024Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918046721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17902024-02-28T22:13:40.024Z{"msg":"[2]R flush_numbers[0..12]: [23, 24, 23, 26, 27, 24, 20, 26, 27, 27, 24, 24]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.9180786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17912024-02-28T22:13:40.024Z{"msg":"[2]R generation[0..12]: [13, 14, 13, 14, 15, 14, 12, 14, 15, 15, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.91810383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17922024-02-28T22:13:40.024Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918134929Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17932024-02-28T22:13:40.024Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918158968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17942024-02-28T22:13:40.024Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918187827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17952024-02-28T22:13:40.025Z{"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918211387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17962024-02-28T22:13:40.025Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918240506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"":"downstairs","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17972024-02-28T22:13:40.025Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918265375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17982024-02-28T22:13:40.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918293024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"0","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
17992024-02-28T22:13:40.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918324273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"1","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
18002024-02-28T22:13:40.025Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918353843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"client":"2","":"downstairs","session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18012024-02-28T22:13:40.025Z}
18022024-02-28T22:13:40.025Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918391071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
18032024-02-28T22:13:40.025Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918415181Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
18042024-02-28T22:13:40.025Z{"msg":"3256dfbb-0454-4f82-a591-6d6b809d84fb is now active with session: 648ddf12-b66a-4d17-b215-1a84ac6b0f45","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.91843878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
18052024-02-28T22:13:40.025Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918468449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199,"session_id":"648ddf12-b66a-4d17-b215-1a84ac6b0f45"}
18062024-02-28T22:13:40.025Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:38.918490989Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
18072024-02-28T22:13:40.026ZLoading write count information from file "/tmp/test_repair_verify.data"
18082024-02-28T22:13:40.026ZRun Repair workload
18092024-02-28T22:13:40.030Z01/30 Write block 205 len 28672 data: 3 3 3 3 3 3 3
18102024-02-28T22:13:40.030Z02/30 Write block 146 len 16384 data: 2 2 2 2
18112024-02-28T22:13:40.030Z03/30 Read block 258 len 4096
18122024-02-28T22:13:40.030Z04/30 Read block 576 len 24576
18132024-02-28T22:13:40.030Z05/30 Write block 27 len 20480 data: 2 2 2 3 3
18142024-02-28T22:13:40.030Z06/30 Read block 167 len 32768
18152024-02-28T22:13:40.030Z07/30 Read block 192 len 20480
18162024-02-28T22:13:40.031Z08/30 Write block 164 len 16384 data: 2 3 3 3
18172024-02-28T22:13:40.031Z09/30 Read block 394 len 40960
18182024-02-28T22:13:40.031Z10/30 Read block 212 len 28672
18192024-02-28T22:13:40.031Z11/30 Read block 288 len 24576
18202024-02-28T22:13:40.031Z12/30 Read block 139 len 20480
18212024-02-28T22:13:40.031Z13/30 Write block 444 len 40960 data: 2 2 2 2 2 2 2 4 4 4
18222024-02-28T22:13:40.031Z14/30 Write block 138 len 8192 data: 2 2
18232024-02-28T22:13:40.035Z15/30 Read block 533 len 24576
18242024-02-28T22:13:40.035Z16/30 Write block 354 len 40960 data: 3 4 5 4 3 3 3 3 3 3
18252024-02-28T22:13:40.035Z17/30 Read block 151 len 20480
18262024-02-28T22:13:40.035Z18/30 Flush
18272024-02-28T22:13:40.039Z19/30 Write block 338 len 36864 data: 3 2 2 2 2 2 2 2 2
18282024-02-28T22:13:40.039Z20/30 Write block 503 len 36864 data: 4 4 3 3 3 3 2 2 2
18292024-02-28T22:13:40.039Z21/30 Read block 12 len 40960
18302024-02-28T22:13:40.054Z22/30 Flush
18312024-02-28T22:13:40.054Z23/30 Write block 513 len 12288 data: 2 2 2
18322024-02-28T22:13:40.058Z24/30 Read block 360 len 28672
18332024-02-28T22:13:40.059Z25/30 Write block 16 len 36864 data: 3 3 3 4 4 3 2 2 2
18342024-02-28T22:13:40.059Z26/30 Write block 425 len 20480 data: 3 2 2 2 3
18352024-02-28T22:13:40.059Z27/30 Write block 220 len 36864 data: 2 2 2 2 2 2 3 4 5
18362024-02-28T22:13:40.059Z28/30 Read block 283 len 16384
18372024-02-28T22:13:40.059Z29/30 Read block 181 len 24576
18382024-02-28T22:13:40.059Z30/30 Read block 166 len 40960
18392024-02-28T22:13:40.059Z----------------------------------------------------------------
18402024-02-28T22:13:40.059Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
18412024-02-28T22:13:40.059ZStates: Active Active Active
18422024-02-28T22:13:40.059ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18432024-02-28T22:13:40.059Z 1 Acked 1000 Write 7 Done Done Done false
18442024-02-28T22:13:40.059Z 2 Acked 1001 Write 4 Done Done Done false
18452024-02-28T22:13:40.060Z 3 Acked 1002 Read 1 Done Done Done false
18462024-02-28T22:13:40.060Z 4 Acked 1003 Read 6 Done Sent Done false
18472024-02-28T22:13:40.060Z 5 Acked 1004 Write 5 Done Sent Done false
18482024-02-28T22:13:40.060Z 6 Acked 1005 Read 8 Done Sent Done false
18492024-02-28T22:13:40.060Z 7 Acked 1006 Read 5 Done Sent Done false
18502024-02-28T22:13:40.060Z 8 Acked 1007 Write 4 Done Sent Done false
18512024-02-28T22:13:40.060Z 9 Acked 1008 Read 10 Done Sent Done false
18522024-02-28T22:13:40.060Z 10 Acked 1009 Read 7 Done Sent Done false
18532024-02-28T22:13:40.060Z 11 Acked 1010 Read 6 Done Sent Done false
18542024-02-28T22:13:40.060Z 12 Acked 1011 Read 5 Done Sent Done false
18552024-02-28T22:13:40.060Z 13 Acked 1012 Write 10 Done Sent Done false
18562024-02-28T22:13:40.060Z 14 Acked 1013 Write 2 Done Sent Done false
18572024-02-28T22:13:40.060Z 15 Acked 1014 Read 6 Done Sent Done false
18582024-02-28T22:13:40.061Z 16 Acked 1015 Write 10 Done Sent Done false
18592024-02-28T22:13:40.061Z 17 Acked 1016 Read 5 Done Sent Done false
18602024-02-28T22:13:40.061Z 18 Acked 1017 Flush 0 Done Sent Done false
18612024-02-28T22:13:40.061Z 19 Acked 1018 Write 9 Done Sent Done false
18622024-02-28T22:13:40.061Z 20 Acked 1019 Write 9 Done Sent Done false
18632024-02-28T22:13:40.061Z 21 Acked 1020 Read 10 Done Sent Done false
18642024-02-28T22:13:40.061Z 22 Acked 1021 Flush 0 Done Sent Done false
18652024-02-28T22:13:40.061Z 23 Acked 1022 Write 3 Done Sent Done false
18662024-02-28T22:13:40.061Z 24 Acked 1023 Read 7 Done Sent Done false
18672024-02-28T22:13:40.061Z 25 Acked 1024 Write 9 Done Sent Done false
18682024-02-28T22:13:40.061Z 26 Acked 1025 Write 5 Done Sent Done false
18692024-02-28T22:13:40.061Z 27 Acked 1026 Write 9 Done Sent Done false
18702024-02-28T22:13:40.062Z 28 Acked 1027 Read 4 Done Sent Done false
18712024-02-28T22:13:40.062Z 29 Acked 1028 Read 6 Done Sent Done false
18722024-02-28T22:13:40.062Z 30 Acked 1029 Read 10 Sent Sent Done false
18732024-02-28T22:13:40.062Z STATES DS:0 DS:1 DS:2 TOTAL
18742024-02-28T22:13:40.062Z New 0 0 0 0
18752024-02-28T22:13:40.062Z Sent 1 27 0 28
18762024-02-28T22:13:40.062Z Done 29 3 30 62
18772024-02-28T22:13:40.062Z Skipped 0 0 0 0
18782024-02-28T22:13:40.062Z Error 0 0 0 0
18792024-02-28T22:13:40.062ZLast Flush: 1021 0 1021
18802024-02-28T22:13:40.062ZDownstairs last five completed:
18812024-02-28T22:13:40.062ZUpstairs last five completed: 30 29 28 27 26
18822024-02-28T22:13:40.062Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:38.95147532Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1199}
18832024-02-28T22:13:40.063ZWrote out file "/tmp/test_repair_verify.data"
18842024-02-28T22:13:40.063Z
18852024-02-28T22:13:40.063ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18862024-02-28T22:13:40.073ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18872024-02-28T22:13:40.073Z 0 000-019 13 13 13 23 23 23 T F T <---
18882024-02-28T22:13:40.073Z 1 020-039 16 14 16 28 24 28 T F T <---
18892024-02-28T22:13:40.073Z 2 040-059 13 13 13 23 23 23 F F F
18902024-02-28T22:13:40.073Z 3 060-079 14 14 14 26 26 26 F F F
18912024-02-28T22:13:40.073Z 4 080-099 15 15 15 27 27 27 F F F
18922024-02-28T22:13:40.073Z 5 100-119 14 14 14 24 24 24 F F F
18932024-02-28T22:13:40.073Z 6 120-139 16 12 16 28 20 28 F F F <---
18942024-02-28T22:13:40.073Z 7 140-159 16 14 16 28 26 28 F T F <---
18952024-02-28T22:13:40.074Z 8 160-179 16 15 16 28 27 28 F F F <---
18962024-02-28T22:13:40.074Z 9 180-199 15 15 15 27 27 27 F F F
18972024-02-28T22:13:40.074Z 10 200-219 16 14 16 28 24 28 F T F <---
18982024-02-28T22:13:40.074Z 11 220-239 14 14 14 24 24 24 T F T <---
18992024-02-28T22:13:40.074Z 12 240-259 13 13 13 23 23 23 F F F
19002024-02-28T22:13:40.074Z 13 260-279 13 13 13 23 23 23 F F F
19012024-02-28T22:13:40.079Z 14 280-299 5 5 5 10 10 10 F F F
19022024-02-28T22:13:40.079Z 15 300-319 15 15 15 27 27 27 F F F
19032024-02-28T22:13:40.079Z 16 320-339 16 12 16 29 20 29 F F F <---
19042024-02-28T22:13:40.079Z 17 340-359 16 11 16 29 19 29 F F F <---
19052024-02-28T22:13:40.079Z 18 360-379 16 15 16 28 27 28 F F F <---
19062024-02-28T22:13:40.079Z 19 380-399 10 10 10 18 18 18 F F F
19072024-02-28T22:13:40.080Z 20 400-419 12 12 12 21 21 21 F F F
19082024-02-28T22:13:40.080Z 21 420-439 13 13 13 23 23 23 T F T <---
19092024-02-28T22:13:40.080Z 22 440-459 16 11 16 28 19 28 F F F <---
19102024-02-28T22:13:40.080Z 23 460-479 14 14 14 26 26 26 F F F
19112024-02-28T22:13:40.080Z 24 480-499 14 14 14 26 26 26 F F F
19122024-02-28T22:13:40.080Z 25 500-519 16 11 16 29 19 29 T F T <---
19132024-02-28T22:13:40.080Z 26 520-539 11 11 11 19 19 19 F F F
19142024-02-28T22:13:40.080Z 27 540-559 15 15 15 27 27 27 F F F
19152024-02-28T22:13:40.080Z 28 560-579 9 9 9 17 17 17 F F F
19162024-02-28T22:13:40.080Z 29 580-599 14 14 14 25 25 25 F F F
19172024-02-28T22:13:40.080ZMax gen: 16, Max flush: 29
19182024-02-28T22:13:40.080ZError: Difference in extent metadata found!
19192024-02-28T22:13:40.080ZA Difference in extent metadata is expected here
19202024-02-28T22:13:40.080ZOn loop 8
19212024-02-28T22:13:40.080Z
19222024-02-28T22:13:40.085ZVerifying data now
19232024-02-28T22:13:46.394ZNo change after verify
19242024-02-28T22:13:46.394ZLoop: 8 Downstairs dump after verify (and repair):
19252024-02-28T22:13:46.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19262024-02-28T22:13:46.407Z 0 000-019 17 17 17 30 30 30 F F F
19272024-02-28T22:13:46.407Z 1 020-039 17 17 17 30 30 30 F F F
19282024-02-28T22:13:46.407Z 2 040-059 13 13 13 23 23 23 F F F
19292024-02-28T22:13:46.407Z 3 060-079 14 14 14 26 26 26 F F F
19302024-02-28T22:13:46.407Z 4 080-099 15 15 15 27 27 27 F F F
19312024-02-28T22:13:46.412Z 5 100-119 14 14 14 24 24 24 F F F
19322024-02-28T22:13:46.412Z 6 120-139 16 16 16 28 28 28 F F F
19332024-02-28T22:13:46.412Z 7 140-159 16 16 16 28 28 28 F F F
19342024-02-28T22:13:46.412Z 8 160-179 16 16 16 28 28 28 F F F
19352024-02-28T22:13:46.412Z 9 180-199 15 15 15 27 27 27 F F F
19362024-02-28T22:13:46.412Z 10 200-219 16 16 16 28 28 28 F F F
19372024-02-28T22:13:46.413Z 11 220-239 17 17 17 30 30 30 F F F
19382024-02-28T22:13:46.413Z 12 240-259 13 13 13 23 23 23 F F F
19392024-02-28T22:13:46.413Z 13 260-279 13 13 13 23 23 23 F F F
19402024-02-28T22:13:46.413Z 14 280-299 5 5 5 10 10 10 F F F
19412024-02-28T22:13:46.413Z 15 300-319 15 15 15 27 27 27 F F F
19422024-02-28T22:13:46.413Z 16 320-339 16 16 16 29 29 29 F F F
19432024-02-28T22:13:46.413Z 17 340-359 16 16 16 29 29 29 F F F
19442024-02-28T22:13:46.413Z 18 360-379 16 16 16 28 28 28 F F F
19452024-02-28T22:13:46.413Z 19 380-399 10 10 10 18 18 18 F F F
19462024-02-28T22:13:46.413Z 20 400-419 12 12 12 21 21 21 F F F
19472024-02-28T22:13:46.413Z 21 420-439 17 17 17 30 30 30 F F F
19482024-02-28T22:13:46.413Z 22 440-459 16 16 16 28 28 28 F F F
19492024-02-28T22:13:46.414Z 23 460-479 14 14 14 26 26 26 F F F
19502024-02-28T22:13:46.414Z 24 480-499 14 14 14 26 26 26 F F F
19512024-02-28T22:13:46.414Z 25 500-519 17 17 17 30 30 30 F F F
19522024-02-28T22:13:46.414Z 26 520-539 11 11 11 19 19 19 F F F
19532024-02-28T22:13:46.414Z 27 540-559 15 15 15 27 27 27 F F F
19542024-02-28T22:13:46.414Z 28 560-579 9 9 9 17 17 17 F F F
19552024-02-28T22:13:46.414Z 29 580-599 14 14 14 25 25 25 F F F
19562024-02-28T22:13:46.414ZMax gen: 17, Max flush: 30
19572024-02-28T22:13:46.414Z
19582024-02-28T22:13:46.414ZBegin loop 9
19592024-02-28T22:13:46.414ZDownstairs to restart: 1
19602024-02-28T22:13:46.432Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.330707059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
19612024-02-28T22:13:46.441Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33101416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
19622024-02-28T22:13:46.442ZCrucible runtime is spawned
19632024-02-28T22:13:46.442Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331069609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
19642024-02-28T22:13:46.442Z{"msg":"Crucible 4aa125c4-7ddc-4bfa-a8d3-946247f61340 has session id: ef263eb0-cbfa-4fb6-9241-df3d660ded21","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331098478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19652024-02-28T22:13:46.442Z{"msg":"Upstairs opts: Upstairs UUID: 4aa125c4-7ddc-4bfa-a8d3-946247f61340, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331146457Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19662024-02-28T22:13:46.442Z{"msg":"Crucible stats registered with UUID: 4aa125c4-7ddc-4bfa-a8d3-946247f61340","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331166266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19672024-02-28T22:13:46.442Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331182816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
19682024-02-28T22:13:46.442Z{"msg":"4aa125c4-7ddc-4bfa-a8d3-946247f61340 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331207175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19692024-02-28T22:13:46.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331223944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19702024-02-28T22:13:46.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331251974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19712024-02-28T22:13:46.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331269613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19722024-02-28T22:13:46.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331293093Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19732024-02-28T22:13:46.442Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331332821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19742024-02-28T22:13:46.442Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331358871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19752024-02-28T22:13:46.442Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33139241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19762024-02-28T22:13:46.442Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331433159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19772024-02-28T22:13:46.442Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331468368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19782024-02-28T22:13:46.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331493467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19792024-02-28T22:13:46.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331527546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19802024-02-28T22:13:46.443Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.331660643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19812024-02-28T22:13:46.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332243016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19822024-02-28T22:13:46.443Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332290585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19832024-02-28T22:13:46.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332315494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19842024-02-28T22:13:46.443Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332350493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19852024-02-28T22:13:46.443Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332730913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19862024-02-28T22:13:46.443Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332771132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19872024-02-28T22:13:46.443Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.332949137Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19882024-02-28T22:13:46.443Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333002926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19892024-02-28T22:13:46.443Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333436323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19902024-02-28T22:13:46.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333475033Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19912024-02-28T22:13:46.443Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333491682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19922024-02-28T22:13:46.443Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333514081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19932024-02-28T22:13:46.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333545431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19942024-02-28T22:13:46.443Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33356118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19952024-02-28T22:13:46.443Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33358437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19962024-02-28T22:13:46.443Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33391029Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19972024-02-28T22:13:46.444Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333951619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19982024-02-28T22:13:46.444Z{"msg":"[0]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 28, 28, 28, 27, 28, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.333979029Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
19992024-02-28T22:13:46.444Z{"msg":"[0]R generation[0..12]: [17, 17, 13, 14, 15, 14, 16, 16, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334003698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20002024-02-28T22:13:46.444Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334019318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20012024-02-28T22:13:46.444Z{"msg":"[1]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 28, 28, 28, 27, 28, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334034727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20022024-02-28T22:13:46.444Z{"msg":"[1]R generation[0..12]: [17, 17, 13, 14, 15, 14, 16, 16, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334048977Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20032024-02-28T22:13:46.444Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334070286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20042024-02-28T22:13:46.444Z{"msg":"[2]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 28, 28, 28, 27, 28, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334084546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20052024-02-28T22:13:46.444Z{"msg":"[2]R generation[0..12]: [17, 17, 13, 14, 15, 14, 16, 16, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334098375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20062024-02-28T22:13:46.444Z{Wait for a query_work_queue command to finish before sending IO
20072024-02-28T22:13:46.444Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334124185Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20082024-02-28T22:13:46.444Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334138304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20092024-02-28T22:13:46.444Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334152384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20102024-02-28T22:13:46.444Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334165343Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20112024-02-28T22:13:46.444Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334179073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"":"downstairs","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20122024-02-28T22:13:46.444Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334192713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20132024-02-28T22:13:46.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334217832Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"0","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20142024-02-28T22:13:46.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334242921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"1","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20152024-02-28T22:13:46.444Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334272331Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"client":"2","":"downstairs","session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20162024-02-28T22:13:46.444Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.33429758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20172024-02-28T22:13:46.444Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334319479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20182024-02-28T22:13:46.444Z{"msg":"4aa125c4-7ddc-4bfa-a8d3-946247f61340 is now active with session: ef263eb0-cbfa-4fb6-9241-df3d660ded21","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334341349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20192024-02-28T22:13:46.445Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334362348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207,"session_id":"ef263eb0-cbfa-4fb6-9241-df3d660ded21"}
20202024-02-28T22:13:46.445ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20212024-02-28T22:13:46.445Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:45.334398127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
20222024-02-28T22:13:46.445ZLoading write count information from file "/tmp/test_repair_verify.data"
20232024-02-28T22:13:46.445ZRun Repair workload
20242024-02-28T22:13:46.445Z01/30 Write block 363 len 12288 data: 4 2 2
20252024-02-28T22:13:46.445Z02/30 Read block 67 len 8192
20262024-02-28T22:13:46.449Z03/30 Read block 492 len 12288
20272024-02-28T22:13:46.450Z04/30 Read block 228 len 36864
20282024-02-28T22:13:46.450Z05/30 Write block 502 len 20480 data: 4 5 5 4 4
20292024-02-28T22:13:46.450Z06/30 Write block 138 len 36864 data: 3 3 2 2 2 2 2 2 3
20302024-02-28T22:13:46.450Z07/30 Write block 470 len 24576 data: 2 3 3 3 5 6
20312024-02-28T22:13:46.450Z08/30 Read block 271 len 12288
20322024-02-28T22:13:46.450Z09/30 Read block 506 len 32768
20332024-02-28T22:13:46.450Z10/30 Read block 582 len 28672
20342024-02-28T22:13:46.450Z11/30 Read block 265 len 16384
20352024-02-28T22:13:46.450Z12/30 Write block 562 len 20480 data: 2 2 2 2 2
20362024-02-28T22:13:46.450Z13/30 Read block 191 len 20480
20372024-02-28T22:13:46.450Z14/30 Flush
20382024-02-28T22:13:46.454Z15/30 Write block 143 len 32768 data: 3 3 3 4 3 3 3 2
20392024-02-28T22:13:46.454Z16/30 Write block 480 len 24576 data: 3 2 2 2 2 3
20402024-02-28T22:13:46.454Z17/30 Read block 106 len 36864
20412024-02-28T22:13:46.454Z18/30 Write block 529 len 24576 data: 2 2 2 2 3 3
20422024-02-28T22:13:46.455Z19/30 Write block 290 len 16384 data: 2 2 2 2
20432024-02-28T22:13:46.458Z20/30 Flush
20442024-02-28T22:13:46.488Z21/30 Flush
20452024-02-28T22:13:46.491Z22/30 Write block 247 len 8192 data: 4 4
20462024-02-28T22:13:46.491Z23/30 Read block 187 len 12288
20472024-02-28T22:13:46.492Z24/30 Write block 131 len 28672 data: 4 4 4 3 3 3 3
20482024-02-28T22:13:46.492Z25/30 Write block 524 len 28672 data: 3 2 2 2 2 3 3
20492024-02-28T22:13:46.495Z26/30 Flush
20502024-02-28T22:13:46.499Z27/30 Write block 322 len 4096 data: 3
20512024-02-28T22:13:46.499Z28/30 Read block 17 len 12288
20522024-02-28T22:13:46.499Z29/30 Read block 52 len 24576
20532024-02-28T22:13:46.499Z30/30 Read block 149 len 12288
20542024-02-28T22:13:46.503Z----------------------------------------------------------------
20552024-02-28T22:13:46.503Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20562024-02-28T22:13:46.503ZStates: Active Active Active
20572024-02-28T22:13:46.504ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20582024-02-28T22:13:46.504Z 1 Acked 1000 Write 3 Done Done Done false
20592024-02-28T22:13:46.504Z 2 Acked 1001 Read 2 Done Done Done false
20602024-02-28T22:13:46.504Z 3 Acked 1002 Read 3 Done Done Done false
20612024-02-28T22:13:46.504Z 4 Acked 1003 Read 9 Done Done Done false
20622024-02-28T22:13:46.504Z 5 Acked 1004 Write 5 Done Sent Done false
20632024-02-28T22:13:46.504Z 6 Acked 1005 Write 9 Done Sent Done false
20642024-02-28T22:13:46.504Z 7 Acked 1006 Write 6 Done Sent Done false
20652024-02-28T22:13:46.504Z 8 Acked 1007 Read 3 Done Sent Done false
20662024-02-28T22:13:46.504Z 9 Acked 1008 Read 8 Done Sent Done false
20672024-02-28T22:13:46.504Z 10 Acked 1009 Read 7 Done Sent Done false
20682024-02-28T22:13:46.504Z 11 Acked 1010 Read 4 Done Sent Done false
20692024-02-28T22:13:46.504Z 12 Acked 1011 Write 5 Done Sent Done false
20702024-02-28T22:13:46.504Z 13 Acked 1012 Read 5 Done Sent Done false
20712024-02-28T22:13:46.504Z 14 Acked 1013 Flush 0 Done Sent Done false
20722024-02-28T22:13:46.504Z 15 Acked 1014 Write 8 Done Sent Done false
20732024-02-28T22:13:46.504Z 16 Acked 1015 Write 6 Done Sent Done false
20742024-02-28T22:13:46.504Z 17 Acked 1016 Read 9 Done Sent Done false
20752024-02-28T22:13:46.504Z 18 Acked 1017 Write 6 Done Sent Done false
20762024-02-28T22:13:46.504Z 19 Acked 1018 Write 4 Done Sent Done false
20772024-02-28T22:13:46.504Z 20 Acked 1019 Flush 0 Done Sent Done false
20782024-02-28T22:13:46.504Z 21 Acked 1020 Flush 0 Done Sent Done false
20792024-02-28T22:13:46.504Z 22 Acked 1021 Write 2 Done Sent Done false
20802024-02-28T22:13:46.505Z 23 Acked 1022 Read 3 Done Sent Done false
20812024-02-28T22:13:46.505Z 24 Acked 1023 Write 7 Done Sent Done false
20822024-02-28T22:13:46.505Z 25 Acked 1024 Write 7 Done Sent Done false
20832024-02-28T22:13:46.505Z 26 Acked 1025 Flush 0 Done Sent Done false
20842024-02-28T22:13:46.505Z 27 Acked 1026 Write 1 Done Sent Done false
20852024-02-28T22:13:46.505Z 28 Acked 1027 Read 3 Done Sent Done false
20862024-02-28T22:13:46.505Z 29 Acked 1028 Read 6 Done Sent Done false
20872024-02-28T22:13:46.505Z 30 Acked 1029 Read 3 Done Sent Done false
20882024-02-28T22:13:46.505Z STATES DS:0 DS:1 DS:2 TOTAL
20892024-02-28T22:13:46.505Z New 0 0 0 0
20902024-02-28T22:13:46.505Z Sent 0 26 0 26
20912024-02-28T22:13:46.505Z Done 30 4 30 64
20922024-02-28T22:13:46.505Z Skipped 0 0 0 0
20932024-02-28T22:13:46.505Z Error 0 0 0 0
20942024-02-28T22:13:46.505ZLast Flush: 1025 0 1025
20952024-02-28T22:13:46.505ZDownstairs last five completed:
20962024-02-28T22:13:46.505ZUpstairs last five completed: 30 29 28 27 26
20972024-02-28T22:13:46.505Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:45.400798408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1207}
20982024-02-28T22:13:46.505ZWrote out file "/tmp/test_repair_verify.data"
20992024-02-28T22:13:46.509Z
21002024-02-28T22:13:46.509ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21012024-02-28T22:13:46.522ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21022024-02-28T22:13:46.522Z 0 000-019 17 17 17 30 30 30 F F F
21032024-02-28T22:13:46.523Z 1 020-039 17 17 17 30 30 30 F F F
21042024-02-28T22:13:46.523Z 2 040-059 13 13 13 23 23 23 F F F
21052024-02-28T22:13:46.523Z 3 060-079 14 14 14 26 26 26 F F F
21062024-02-28T22:13:46.523Z 4 080-099 15 15 15 27 27 27 F F F
21072024-02-28T22:13:46.523Z 5 100-119 14 14 14 24 24 24 F F F
21082024-02-28T22:13:46.523Z 6 120-139 18 16 18 34 28 34 F F F <---
21092024-02-28T22:13:46.523Z 7 140-159 18 16 18 32 28 32 F F F <---
21102024-02-28T22:13:46.527Z 8 160-179 16 16 16 28 28 28 F F F
21112024-02-28T22:13:46.527Z 9 180-199 15 15 15 27 27 27 F F F
21122024-02-28T22:13:46.527Z 10 200-219 16 16 16 28 28 28 F F F
21132024-02-28T22:13:46.527Z 11 220-239 17 17 17 30 30 30 F F F
21142024-02-28T22:13:46.527Z 12 240-259 18 13 18 34 23 34 F F F <---
21152024-02-28T22:13:46.527Z 13 260-279 13 13 13 23 23 23 F F F
21162024-02-28T22:13:46.527Z 14 280-299 18 5 18 32 10 32 F F F <---
21172024-02-28T22:13:46.527Z 15 300-319 15 15 15 27 27 27 F F F
21182024-02-28T22:13:46.528Z 16 320-339 16 16 16 29 29 29 T F T <---
21192024-02-28T22:13:46.528Z 17 340-359 16 16 16 29 29 29 F F F
21202024-02-28T22:13:46.528Z 18 360-379 18 16 18 31 28 31 F T F <---
21212024-02-28T22:13:46.528Z 19 380-399 10 10 10 18 18 18 F F F
21222024-02-28T22:13:46.528Z 20 400-419 12 12 12 21 21 21 F F F
21232024-02-28T22:13:46.528Z 21 420-439 17 17 17 30 30 30 F F F
21242024-02-28T22:13:46.528Z 22 440-459 16 16 16 28 28 28 F F F
21252024-02-28T22:13:46.528Z 23 460-479 18 14 18 31 26 31 F F F <---
21262024-02-28T22:13:46.528Z 24 480-499 18 14 18 32 26 32 F F F <---
21272024-02-28T22:13:46.528Z 25 500-519 18 17 18 31 30 31 F F F <---
21282024-02-28T22:13:46.528Z 26 520-539 18 11 18 34 19 34 F F F <---
21292024-02-28T22:13:46.528Z 27 540-559 15 15 15 27 27 27 F F F
21302024-02-28T22:13:46.529Z 28 560-579 18 9 18 31 17 31 F F F <---
21312024-02-28T22:13:46.529Z 29 580-599 14 14 14 25 25 25 F F F
21322024-02-28T22:13:46.529ZMax gen: 18, Max flush: 34
21332024-02-28T22:13:46.529ZError: Difference in extent metadata found!
21342024-02-28T22:13:46.529ZA Difference in extent metadata is expected here
21352024-02-28T22:13:46.529ZOn loop 9
21362024-02-28T22:13:46.529Z
21372024-02-28T22:13:46.532ZVerifying data now
21382024-02-28T22:13:52.498ZNo change after verify
21392024-02-28T22:13:52.498ZLoop: 9 Downstairs dump after verify (and repair):
21402024-02-28T22:13:52.514ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21412024-02-28T22:13:52.514Z 0 000-019 17 17 17 30 30 30 F F F
21422024-02-28T22:13:52.514Z 1 020-039 17 17 17 30 30 30 F F F
21432024-02-28T22:13:52.514Z 2 040-059 13 13 13 23 23 23 F F F
21442024-02-28T22:13:52.514Z 3 060-079 14 14 14 26 26 26 F F F
21452024-02-28T22:13:52.514Z 4 080-099 15 15 15 27 27 27 F F F
21462024-02-28T22:13:52.514Z 5 100-119 14 14 14 24 24 24 F F F
21472024-02-28T22:13:52.514Z 6 120-139 18 18 18 34 34 34 F F F
21482024-02-28T22:13:52.514Z 7 140-159 18 18 18 32 32 32 F F F
21492024-02-28T22:13:52.515Z 8 160-179 16 16 16 28 28 28 F F F
21502024-02-28T22:13:52.515Z 9 180-199 15 15 15 27 27 27 F F F
21512024-02-28T22:13:52.515Z 10 200-219 16 16 16 28 28 28 F F F
21522024-02-28T22:13:52.518Z 11 220-239 17 17 17 30 30 30 F F F
21532024-02-28T22:13:52.518Z 12 240-259 18 18 18 34 34 34 F F F
21542024-02-28T22:13:52.518Z 13 260-279 13 13 13 23 23 23 F F F
21552024-02-28T22:13:52.518Z 14 280-299 18 18 18 32 32 32 F F F
21562024-02-28T22:13:52.518Z 15 300-319 15 15 15 27 27 27 F F F
21572024-02-28T22:13:52.518Z 16 320-339 19 19 19 35 35 35 F F F
21582024-02-28T22:13:52.518Z 17 340-359 16 16 16 29 29 29 F F F
21592024-02-28T22:13:52.518Z 18 360-379 18 18 18 31 31 31 F F F
21602024-02-28T22:13:52.518Z 19 380-399 10 10 10 18 18 18 F F F
21612024-02-28T22:13:52.518Z 20 400-419 12 12 12 21 21 21 F F F
21622024-02-28T22:13:52.518Z 21 420-439 17 17 17 30 30 30 F F F
21632024-02-28T22:13:52.518Z 22 440-459 16 16 16 28 28 28 F F F
21642024-02-28T22:13:52.519Z 23 460-479 18 18 18 31 31 31 F F F
21652024-02-28T22:13:52.519Z 24 480-499 18 18 18 32 32 32 F F F
21662024-02-28T22:13:52.519Z 25 500-519 18 18 18 31 31 31 F F F
21672024-02-28T22:13:52.519Z 26 520-539 18 18 18 34 34 34 F F F
21682024-02-28T22:13:52.519Z 27 540-559 15 15 15 27 27 27 F F F
21692024-02-28T22:13:52.519Z 28 560-579 18 18 18 31 31 31 F F F
21702024-02-28T22:13:52.519Z 29 580-599 14 14 14 25 25 25 F F F
21712024-02-28T22:13:52.519ZMax gen: 19, Max flush: 35
21722024-02-28T22:13:52.519Z
21732024-02-28T22:13:52.519ZBegin loop 10
21742024-02-28T22:13:52.519ZDownstairs to restart: 1
21752024-02-28T22:13:52.532Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433166052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
21762024-02-28T22:13:52.532Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433441344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
21772024-02-28T22:13:52.538Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433462404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
21782024-02-28T22:13:52.538Z{"msg":"Crucible 1ad9242c-cf0d-4cc5-ba50-b1832650f9f7 has session id: 1c2828df-32e0-4dad-ad68-c6eef85b286e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433493763Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21792024-02-28T22:13:52.538Z{"msg":"Upstairs opts: Upstairs UUID: 1ad9242c-cf0d-4cc5-ba50-b1832650f9f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false"Crucible runtime is spawned
21802024-02-28T22:13:52.539Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433548142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21812024-02-28T22:13:52.539Z{"msg":"Crucible stats registered with UUID: 1ad9242c-cf0d-4cc5-ba50-b1832650f9f7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433585741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21822024-02-28T22:13:52.539Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.43360265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
21832024-02-28T22:13:52.539Z{"msg":"1ad9242c-cf0d-4cc5-ba50-b1832650f9f7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433724527Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21842024-02-28T22:13:52.539Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433752676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21852024-02-28T22:13:52.539Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433769075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21862024-02-28T22:13:52.539Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433783335Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21872024-02-28T22:13:52.539Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433813714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21882024-02-28T22:13:52.539Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433859823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21892024-02-28T22:13:52.539Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433884592Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21902024-02-28T22:13:52.539Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433909581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21912024-02-28T22:13:52.539Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.433929801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21922024-02-28T22:13:52.539Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.43396321Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21932024-02-28T22:13:52.539Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434018549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21942024-02-28T22:13:52.539Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434267872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21952024-02-28T22:13:52.539Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.43431821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21962024-02-28T22:13:52.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434742358Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21972024-02-28T22:13:52.539Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434781208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21982024-02-28T22:13:52.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434797477Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
21992024-02-28T22:13:52.539Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434814267Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22002024-02-28T22:13:52.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434927633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22012024-02-28T22:13:52.540Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.434978692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22022024-02-28T22:13:52.540Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435355602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22032024-02-28T22:13:52.540Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435397591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22042024-02-28T22:13:52.540Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435679923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22052024-02-28T22:13:52.540Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435886577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22062024-02-28T22:13:52.540Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435927996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22072024-02-28T22:13:52.540Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435955175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22082024-02-28T22:13:52.540Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.435975125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22092024-02-28T22:13:52.540Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436006664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22102024-02-28T22:13:52.540Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436022463Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22112024-02-28T22:13:52.540Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436175109Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22122024-02-28T22:13:52.540Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436208128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22132024-02-28T22:13:52.540Z{"msg":"[0]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 34, 32, 28, 27, 28, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436237397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22142024-02-28T22:13:52.540Z{"msg":"[0]R generation[0..12]: [17, 17, 13, 14, 15, 14, 18, 18, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436268647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22152024-02-28T22:13:52.540Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436285236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22162024-02-28T22:13:52.540Z{"msg":"[1]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 34, 32, 28, 27, 28, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436307035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22172024-02-28T22:13:52.540Z{"msg":"[1]R generation[0..12]: [17, 17, 13, 14, 15, 14, 18, 18, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436321365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22182024-02-28T22:13:52.540Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436346114Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22192024-02-28T22:13:52.540Z{"msg":"[2]R flush_numbers[0..12]: [30, 30, 23, 26, 27, 24, 34, 32, 28, 27, 28, 30]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
22202024-02-28T22:13:52.540Z,"time":"2024-02-28T22:13:51.436387103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22212024-02-28T22:13:52.540Z{"msg":"[2]R generation[0..12]: [17, 17, 13, 14, 15, 14, 18, 18, 16, 15, 16, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436421582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22222024-02-28T22:13:52.540Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436460921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22232024-02-28T22:13:52.540Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436477381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22242024-02-28T22:13:52.540Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.43650164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22252024-02-28T22:13:52.541Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22262024-02-28T22:13:52.541Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436549309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"":"downstairs","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22272024-02-28T22:13:52.541Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436597207Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22282024-02-28T22:13:52.541Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436628087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"0","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22292024-02-28T22:13:52.541Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436642526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"1","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22302024-02-28T22:13:52.541Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436673815Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"client":"2","":"downstairs","session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22312024-02-28T22:13:52.541Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436709874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22322024-02-28T22:13:52.541Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436736174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22332024-02-28T22:13:52.541Z{"msg":"1ad9242c-cf0d-4cc5-ba50-b1832650f9f7 is now active with session: 1c2828df-32e0-4dad-ad68-c6eef85b286e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436751513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22342024-02-28T22:13:52.541Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436783262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215,"session_id":"1c2828df-32e0-4dad-ad68-c6eef85b286e"}
22352024-02-28T22:13:52.541Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:51.436817831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
22362024-02-28T22:13:52.544ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22372024-02-28T22:13:52.544ZLoading write count information from file "/tmp/test_repair_verify.data"
22382024-02-28T22:13:52.545ZRun Repair workload
22392024-02-28T22:13:52.545Z01/30 Write block 460 len 24576 data: 2 2 2 3 3 3
22402024-02-28T22:13:52.545Z02/30 Write block 121 len 32768 data: 5 4 4 4 4 3 3 3
22412024-02-28T22:13:52.545Z03/30 Read block 209 len 8192
22422024-02-28T22:13:52.545Z04/30 Read block 427 len 12288
22432024-02-28T22:13:52.545Z05/30 Read block 42 len 12288
22442024-02-28T22:13:52.545Z06/30 Read block 18 len 40960
22452024-02-28T22:13:52.545Z07/30 Write block 432 len 4096 data: 4
22462024-02-28T22:13:52.545Z08/30 Read block 184 len 8192
22472024-02-28T22:13:52.545Z09/30 Read block 92 len 8192
22482024-02-28T22:13:52.545Z10/30 Write block 201 len 8192 data: 2 2
22492024-02-28T22:13:52.545Z11/30 Write block 368 len 20480 data: 3 3 3 4 4
22502024-02-28T22:13:52.545Z12/30 Read block 191 len 12288
22512024-02-28T22:13:52.556Z13/30 Write block 147 len 20480 data: 4 4 4 3 2
22522024-02-28T22:13:52.556Z14/30 Write block 167 len 16384 data: 4 3 3 3
22532024-02-28T22:13:52.556Z15/30 Write block 410 len 4096 data: 4
22542024-02-28T22:13:52.556Z16/30 Read block 63 len 36864
22552024-02-28T22:13:52.556Z17/30 Write block 268 len 20480 data: 2 2 2 2 2
22562024-02-28T22:13:52.556Z18/30 Flush
22572024-02-28T22:13:52.561Z19/30 Write block 536 len 8192 data: 2 2
22582024-02-28T22:13:52.561Z20/30 Read block 547 len 16384
22592024-02-28T22:13:52.561Z21/30 Write block 173 len 32768 data: 4 4 3 3 2 2 2 2
22602024-02-28T22:13:52.561Z22/30 Write block 199 len 4096 data: 2
22612024-02-28T22:13:52.561Z23/30 Read block 510 len 32768
22622024-02-28T22:13:52.561Z24/30 Write block 80 len 4096 data: 2
22632024-02-28T22:13:52.561Z25/30 Flush
22642024-02-28T22:13:52.561Z26/30 Write block 561 len 36864 data: 3 3 3 3 3 3 3 3 2
22652024-02-28T22:13:52.567Z27/30 Write block 3 len 12288 data: 3 3 3
22662024-02-28T22:13:52.567Z28/30 Read block 310 len 40960
22672024-02-28T22:13:52.567Z29/30 Write block 382 len 28672 data: 3 3 4 4 4 4 4
22682024-02-28T22:13:52.567Z30/30 Write block 567 len 36864 data: 4 4 3 2 2 2 2 2 2
22692024-02-28T22:13:52.567Z----------------------------------------------------------------
22702024-02-28T22:13:52.567Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22712024-02-28T22:13:52.567ZStates: Active Active Active
22722024-02-28T22:13:52.567ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22732024-02-28T22:13:52.567Z 1 Acked 1000 Write 6 Done Sent Done false
22742024-02-28T22:13:52.567Z 2 Acked 1001 Write 8 Done Sent Done false
22752024-02-28T22:13:52.567Z 3 Acked 1002 Read 2 Done Sent Done false
22762024-02-28T22:13:52.567Z 4 Acked 1003 Read 3 Done Sent Done false
22772024-02-28T22:13:52.567Z 5 Acked 1004 Read 3 Done Sent Done false
22782024-02-28T22:13:52.567Z 6 Acked 1005 Read 10 Done Sent Done false
22792024-02-28T22:13:52.567Z 7 Acked 1006 Write 1 Done Sent Done false
22802024-02-28T22:13:52.567Z 8 Acked 1007 Read 2 Done Sent Done false
22812024-02-28T22:13:52.567Z 9 Acked 1008 Read 2 Done Sent Done false
22822024-02-28T22:13:52.567Z 10 Acked 1009 Write 2 Done Sent Done false
22832024-02-28T22:13:52.567Z 11 Acked 1010 Write 5 Done Sent Done false
22842024-02-28T22:13:52.567Z 12 Acked 1011 Read 3 Done Sent Done false
22852024-02-28T22:13:52.568Z 13 Acked 1012 Write 5 Done Sent Done false
22862024-02-28T22:13:52.568Z 14 Acked 1013 Write 4 Done Sent Done false
22872024-02-28T22:13:52.568Z 15 Acked 1014 Write 1 Done Sent Done false
22882024-02-28T22:13:52.568Z 16 Acked 1015 Read 9 Done Sent Done false
22892024-02-28T22:13:52.568Z 17 Acked 1016 Write 5 Done Sent Done false
22902024-02-28T22:13:52.568Z 18 Acked 1017 Flush 0 Done Sent Done false
22912024-02-28T22:13:52.568Z 19 Acked 1018 Write 2 Done Sent Done false
22922024-02-28T22:13:52.568Z 20 Acked 1019 Read 4 Done Sent Done false
22932024-02-28T22:13:52.568Z 21 Acked 1020 Write 8 Done Sent Done false
22942024-02-28T22:13:52.568Z 22 Acked 1021 Write 1 Done Sent Done false
22952024-02-28T22:13:52.568Z 23 Acked 1022 Read 8 Done Sent Done false
22962024-02-28T22:13:52.568Z 24 Acked 1023 Write 1 Done Sent Done false
22972024-02-28T22:13:52.568Z 25 Acked 1024 Flush 0 Done Sent Done false
22982024-02-28T22:13:52.568Z 26 Acked 1025 Write 9 Done Sent Done false
22992024-02-28T22:13:52.568Z 27 Acked 1026 Write 3 Done Sent Done false
23002024-02-28T22:13:52.568Z 28 Acked 1027 Read 10 Done Sent Done false
23012024-02-28T22:13:52.568Z 29 Acked 1028 Write 7 Done Sent Sent false
23022024-02-28T22:13:52.568Z 30 Acked 1029 Write 9 Sent Sent Sent false
23032024-02-28T22:13:52.568Z STATES DS:0 DS:1 DS:2 TOTAL
23042024-02-28T22:13:52.568Z New 0 0 0 0
23052024-02-28T22:13:52.568Z Sent 1 30 2 33
23062024-02-28T22:13:52.568Z Done 29 0 28 57
23072024-02-28T22:13:52.568Z Skipped 0 0 0 0
23082024-02-28T22:13:52.569Z Error 0 0 0 0
23092024-02-28T22:13:52.569ZLast Flush: 1024 0 1024
23102024-02-28T22:13:52.569ZDownstairs last five completed:
23112024-02-28T22:13:52.569ZUpstairs last five completed: 30 29 28 27 26
23122024-02-28T22:13:52.569Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:51.46518766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1215}
23132024-02-28T22:13:52.569ZWrote out file "/tmp/test_repair_verify.data"
23142024-02-28T22:13:52.582Z
23152024-02-28T22:13:52.582ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23162024-02-28T22:13:52.587ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23172024-02-28T22:13:52.587Z 0 000-019 17 17 17 30 30 30 T F T <---
23182024-02-28T22:13:52.587Z 1 020-039 17 17 17 30 30 30 F F F
23192024-02-28T22:13:52.587Z 2 040-059 13 13 13 23 23 23 F F F
23202024-02-28T22:13:52.587Z 3 060-079 14 14 14 26 26 26 F F F
23212024-02-28T22:13:52.588Z 4 080-099 20 15 20 37 27 37 F F F <---
23222024-02-28T22:13:52.588Z 5 100-119 14 14 14 24 24 24 F F F
23232024-02-28T22:13:52.588Z 6 120-139 20 18 20 36 34 36 F F F <---
23242024-02-28T22:13:52.588Z 7 140-159 20 18 20 36 32 36 F F F <---
23252024-02-28T22:13:52.588Z 8 160-179 20 16 20 37 28 37 F F F <---
23262024-02-28T22:13:52.593Z 9 180-199 20 15 20 37 27 37 F F F <---
23272024-02-28T22:13:52.594Z 10 200-219 20 16 20 36 28 36 F F F <---
23282024-02-28T22:13:52.594Z 11 220-239 17 17 17 30 30 30 F F F
23292024-02-28T22:13:52.594Z 12 240-259 18 18 18 34 34 34 F F F
23302024-02-28T22:13:52.594Z 13 260-279 20 13 20 36 23 36 F F F <---
23312024-02-28T22:13:52.594Z 14 280-299 18 18 18 32 32 32 F F F
23322024-02-28T22:13:52.594Z 15 300-319 15 15 15 27 27 27 F F F
23332024-02-28T22:13:52.594Z 16 320-339 19 19 19 35 35 35 F F F
23342024-02-28T22:13:52.594Z 17 340-359 16 16 16 29 29 29 F F F
23352024-02-28T22:13:52.594Z 18 360-379 20 18 20 36 31 36 F F F <---
23362024-02-28T22:13:52.594Z 19 380-399 10 10 10 18 18 18 T F T <---
23372024-02-28T22:13:52.594Z 20 400-419 20 12 20 36 21 36 F F F <---
23382024-02-28T22:13:52.594Z 21 420-439 20 17 20 36 30 36 F F F <---
23392024-02-28T22:13:52.594Z 22 440-459 16 16 16 28 28 28 F F F
23402024-02-28T22:13:52.594Z 23 460-479 20 18 20 36 31 36 F F F <---
23412024-02-28T22:13:52.594Z 24 480-499 18 18 18 32 32 32 F F F
23422024-02-28T22:13:52.594Z 25 500-519 18 18 18 31 31 31 F F F
23432024-02-28T22:13:52.594Z 26 520-539 20 18 20 37 34 37 F F F <---
23442024-02-28T22:13:52.594Z 27 540-559 15 15 15 27 27 27 F F F
23452024-02-28T22:13:52.594Z 28 560-579 18 18 18 31 31 31 T F T <---
23462024-02-28T22:13:52.594Z 29 580-599 14 14 14 25 25 25 F F F
23472024-02-28T22:13:52.594ZMax gen: 20, Max flush: 37
23482024-02-28T22:13:52.594ZError: Difference in extent metadata found!
23492024-02-28T22:13:52.594ZA Difference in extent metadata is expected here
23502024-02-28T22:13:52.595ZOn loop 10
23512024-02-28T22:13:52.595Z
23522024-02-28T22:13:52.602ZVerifying data now
23532024-02-28T22:13:58.896ZNo change after verify
23542024-02-28T22:13:58.897ZLoop: 10 Downstairs dump after verify (and repair):
23552024-02-28T22:13:58.910ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23562024-02-28T22:13:58.910Z 0 000-019 21 21 21 38 38 38 F F F
23572024-02-28T22:13:58.910Z 1 020-039 17 17 17 30 30 30 F F F
23582024-02-28T22:13:58.911Z 2 040-059 13 13 13 23 23 23 F F F
23592024-02-28T22:13:58.911Z 3 060-079 14 14 14 26 26 26 F F F
23602024-02-28T22:13:58.911Z 4 080-099 20 20 20 37 37 37 F F F
23612024-02-28T22:13:58.911Z 5 100-119 14 14 14 24 24 24 F F F
23622024-02-28T22:13:58.911Z 6 120-139 20 20 20 36 36 36 F F F
23632024-02-28T22:13:58.911Z 7 140-159 20 20 20 36 36 36 F F F
23642024-02-28T22:13:58.911Z 8 160-179 20 20 20 37 37 37 F F F
23652024-02-28T22:13:58.911Z 9 180-199 20 20 20 37 37 37 F F F
23662024-02-28T22:13:58.911Z 10 200-219 20 20 20 36 36 36 F F F
23672024-02-28T22:13:58.915Z 11 220-239 17 17 17 30 30 30 F F F
23682024-02-28T22:13:58.915Z 12 240-259 18 18 18 34 34 34 F F F
23692024-02-28T22:13:58.915Z 13 260-279 20 20 20 36 36 36 F F F
23702024-02-28T22:13:58.915Z 14 280-299 18 18 18 32 32 32 F F F
23712024-02-28T22:13:58.915Z 15 300-319 15 15 15 27 27 27 F F F
23722024-02-28T22:13:58.915Z 16 320-339 19 19 19 35 35 35 F F F
23732024-02-28T22:13:58.915Z 17 340-359 16 16 16 29 29 29 F F F
23742024-02-28T22:13:58.915Z 18 360-379 20 20 20 36 36 36 F F F
23752024-02-28T22:13:58.915Z 19 380-399 21 21 21 38 38 38 F F F
23762024-02-28T22:13:58.915Z 20 400-419 20 20 20 36 36 36 F F F
23772024-02-28T22:13:58.915Z 21 420-439 20 20 20 36 36 36 F F F
23782024-02-28T22:13:58.915Z 22 440-459 16 16 16 28 28 28 F F F
23792024-02-28T22:13:58.915Z 23 460-479 20 20 20 36 36 36 F F F
23802024-02-28T22:13:58.915Z 24 480-499 18 18 18 32 32 32 F F F
23812024-02-28T22:13:58.915Z 25 500-519 18 18 18 31 31 31 F F F
23822024-02-28T22:13:58.916Z 26 520-539 20 20 20 37 37 37 F F F
23832024-02-28T22:13:58.916Z 27 540-559 15 15 15 27 27 27 F F F
23842024-02-28T22:13:58.916Z 28 560-579 21 21 21 38 38 38 F F F
23852024-02-28T22:13:58.916Z 29 580-599 14 14 14 25 25 25 F F F
23862024-02-28T22:13:58.916ZMax gen: 21, Max flush: 38
23872024-02-28T22:13:58.916Z
23882024-02-28T22:13:58.916ZBegin loop 11
23892024-02-28T22:13:58.916ZDownstairs to restart: 0
23902024-02-28T22:13:58.933Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833429861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
23912024-02-28T22:13:58.933Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833680534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
23922024-02-28T22:13:58.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833719683Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
23932024-02-28T22:13:58.942ZCrucible runtime is spawned
23942024-02-28T22:13:58.942Z{"msg":"Crucible fcd3235b-54da-46f9-93a9-b08ed93ea342 has session id: 76a109db-ea47-4dac-a39c-fc7332e3ff39","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833781602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
23952024-02-28T22:13:58.942Z{"msg":"Upstairs opts: Upstairs UUID: fcd3235b-54da-46f9-93a9-b08ed93ea342, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.83382758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
23962024-02-28T22:13:58.942Z{"msg":"Crucible stats registered with UUID: fcd3235b-54da-46f9-93a9-b08ed93ea342","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833867179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
23972024-02-28T22:13:58.942Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833899048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
23982024-02-28T22:13:58.942Z{"msg":"fcd3235b-54da-46f9-93a9-b08ed93ea342 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833929287Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
23992024-02-28T22:13:58.942Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833960337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24002024-02-28T22:13:58.942Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.833987746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24012024-02-28T22:13:58.942Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834021265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24022024-02-28T22:13:58.942Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834047074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24032024-02-28T22:13:58.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834083323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24042024-02-28T22:13:58.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834108933Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24052024-02-28T22:13:58.943Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.83418792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24062024-02-28T22:13:58.943Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834223739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24072024-02-28T22:13:58.943Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834261878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24082024-02-28T22:13:58.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834286758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24092024-02-28T22:13:58.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834312127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24102024-02-28T22:13:58.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.834348786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24112024-02-28T22:13:58.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835055256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24122024-02-28T22:13:58.943Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835097995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24132024-02-28T22:13:58.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835129925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24142024-02-28T22:13:58.943Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835152804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24152024-02-28T22:13:58.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835170243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24162024-02-28T22:13:58.943Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835183593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24172024-02-28T22:13:58.944Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835707298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24182024-02-28T22:13:58.944Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835745188Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24192024-02-28T22:13:58.944Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.835892853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24202024-02-28T22:13:58.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836222184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24212024-02-28T22:13:58.944Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836276443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24222024-02-28T22:13:58.944Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836302902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24232024-02-28T22:13:58.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836318822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24242024-02-28T22:13:58.944Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836342991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24252024-02-28T22:13:58.944Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836357481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24262024-02-28T22:13:58.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.8363901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24272024-02-28T22:13:58.944Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836409949Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24282024-02-28T22:13:58.944Z{"msg":"[0]R flush_numbers[0..12]: [38, 30, 23, 26, 37, 24, 36, 36, 37, 37, 36, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836490127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24292024-02-28T22:13:58.944Z{"msg":"[0]R generation[0..12]: [21, 17, 13, 14, 20, 14, 20, 20, 20, 20, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836538866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24302024-02-28T22:13:58.944Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836577594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24312024-02-28T22:13:58.944Z{"msg":"[1]R flush_numbers[0..12]: [38, 30, 23, 26, 37, 24, 36, 36, 37, 37, 36, 30]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
24322024-02-28T22:13:58.944Z2024-02-28T22:13:57.836614533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24332024-02-28T22:13:58.944Z{"msg":"[1]R generation[0..12]: [21, 17, 13, 14, 20, 14, 20, 20, 20, 20, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836656312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24342024-02-28T22:13:58.944Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836698481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24352024-02-28T22:13:58.944Z{"msg":"[2]R flush_numbers[0..12]: [38, 30, 23, 26, 37, 24, 36, 36, 37, 37, 36, 30]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.83673786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24362024-02-28T22:13:58.944Z{"msg":"[2]R generation[0..12]: [21, 17, 13, 14, 20, 14, 20, 20, 20, 20, 20, 17]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836766699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24372024-02-28T22:13:58.944Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836797499Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24382024-02-28T22:13:58.945Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836812368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24392024-02-28T22:13:58.945Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836826968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24402024-02-28T22:13:58.945Z{"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836840437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24412024-02-28T22:13:58.945Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836854487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"":"downstairs","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24422024-02-28T22:13:58.945Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836883516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24432024-02-28T22:13:58.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836908045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"0","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24442024-02-28T22:13:58.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836936615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"1","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24452024-02-28T22:13:58.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836957594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"client":"2","":"downstairs","session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24462024-02-28T22:13:58.945Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.836989253Z","hostname":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24472024-02-28T22:13:58.945Z"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24482024-02-28T22:13:58.945Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.837016702Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24492024-02-28T22:13:58.945Z{"msg":"fcd3235b-54da-46f9-93a9-b08ed93ea342 is now active with session: 76a109db-ea47-4dac-a39c-fc7332e3ff39","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.837040232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24502024-02-28T22:13:58.945Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.837066251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223,"session_id":"76a109db-ea47-4dac-a39c-fc7332e3ff39"}
24512024-02-28T22:13:58.945Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:13:57.83708882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
24522024-02-28T22:13:58.945ZLoading write count information from file "/tmp/test_repair_verify.data"
24532024-02-28T22:13:58.945ZRun Repair workload
24542024-02-28T22:13:58.949Z01/30 Write block 7 len 28672 data: 3 3 2 2 2 2 2
24552024-02-28T22:13:58.949Z02/30 Write block 594 len 16384 data: 4 4 3 3
24562024-02-28T22:13:58.949Z03/30 Read block 314 len 16384
24572024-02-28T22:13:58.949Z04/30 Write block 591 len 32768 data: 5 5 5 5 5 4 4 3
24582024-02-28T22:13:58.949Z05/30 Read block 425 len 32768
24592024-02-28T22:13:58.950Z06/30 Read block 433 len 28672
24602024-02-28T22:13:58.950Z07/30 Read block 295 len 12288
24612024-02-28T22:13:58.950Z08/30 Read block 0 len 24576
24622024-02-28T22:13:58.950Z09/30 Write block 507 len 40960 data: 4 4 3 3 3 2 3 3 3 2
24632024-02-28T22:13:58.950Z10/30 Write block 223 len 36864 data: 3 3 3 4 5 6 6 6 6
24642024-02-28T22:13:58.950Z11/30 Read block 377 len 12288
24652024-02-28T22:13:58.950Z12/30 Write block 339 len 40960 data: 3 3 3 3 3 3 3 3 2 3
24662024-02-28T22:13:58.950Z13/30 Write block 127 len 28672 data: 4 4 3 3 5 5 5
24672024-02-28T22:13:58.954Z14/30 Read block 124 len 16384
24682024-02-28T22:13:58.954Z15/30 Flush
24692024-02-28T22:13:58.957Z16/30 Write block 140 len 40960 data: 3 3 3 4 4 4 5 5 5 5
24702024-02-28T22:13:58.957Z17/30 Read block 192 len 20480
24712024-02-28T22:13:58.957Z18/30 Write block 348 len 12288 data: 4 2 2
24722024-02-28T22:13:58.957Z19/30 Read block 61 len 28672
24732024-02-28T22:13:58.961Z20/30 Read block 15 len 40960
24742024-02-28T22:13:58.961Z21/30 Flush
24752024-02-28T22:13:58.986Z22/30 Write block 303 len 20480 data: 4 5 4 4 4
24762024-02-28T22:13:58.990Z23/30 Write block 321 len 4096 data: 2
24772024-02-28T22:13:58.990Z24/30 Write block 122 len 36864 data: 5 5 5 5 4 5 5 4 4
24782024-02-28T22:13:58.990Z25/30 Read block 274 len 12288
24792024-02-28T22:13:58.990Z26/30 Flush
24802024-02-28T22:13:58.993Z27/30 Write block 311 len 24576 data: 2 2 3 3 3 3
24812024-02-28T22:13:58.997Z28/30 Write block 327 len 32768 data: 3 3 3 3 3 3 3 4
24822024-02-28T22:13:58.997Z29/30 Read block 205 len 24576
24832024-02-28T22:13:58.997Z30/30 Read block 551 len 28672
24842024-02-28T22:13:58.997Z----------------------------------------------------------------
24852024-02-28T22:13:58.997Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24862024-02-28T22:13:58.997ZStates: Active Active Active
24872024-02-28T22:13:58.997ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24882024-02-28T22:13:58.997Z 1 Acked 1000 Write 7 Done Done Done false
24892024-02-28T22:13:58.997Z 2 Acked 1001 Write 4 Done Done Done false
24902024-02-28T22:13:58.998Z 3 Acked 1002 Read 4 Done Done Done false
24912024-02-28T22:13:58.998Z 4 Acked 1003 Write 8 Done Done Done false
24922024-02-28T22:13:58.998Z 5 Acked 1004 Read 8 Done Done Done false
24932024-02-28T22:13:58.998Z 6 Acked 1005 Read 7 Done Done Done false
24942024-02-28T22:13:58.998Z 7 Acked 1006 Read 3 Done Done Done false
24952024-02-28T22:13:58.998Z 8 Acked 1007 Read 6 Sent Done Done false
24962024-02-28T22:13:58.998Z 9 Acked 1008 Write 10 Sent Done Done false
24972024-02-28T22:13:58.998Z 10 Acked 1009 Write 9 Sent Done Done false
24982024-02-28T22:13:58.998Z 11 Acked 1010 Read 3 Sent Done Done false
24992024-02-28T22:13:58.998Z 12 Acked 1011 Write 10 Sent Done Done false
25002024-02-28T22:13:58.998Z 13 Acked 1012 Write 7 Sent Done Done false
25012024-02-28T22:13:58.998Z 14 Acked 1013 Read 4 Sent Done Done false
25022024-02-28T22:13:58.998Z 15 Acked 1014 Flush 0 Sent Done Done false
25032024-02-28T22:13:58.998Z 16 Acked 1015 Write 10 Sent Done Done false
25042024-02-28T22:13:58.998Z 17 Acked 1016 Read 5 Sent Done Done false
25052024-02-28T22:13:58.998Z 18 Acked 1017 Write 3 Sent Done Done false
25062024-02-28T22:13:58.998Z 19 Acked 1018 Read 7 Sent Done Done false
25072024-02-28T22:13:58.998Z 20 Acked 1019 Read 10 Sent Done Done false
25082024-02-28T22:13:58.998Z 21 Acked 1020 Flush 0 Sent Done Done false
25092024-02-28T22:13:58.998Z 22 Acked 1021 Write 5 Sent Done Done false
25102024-02-28T22:13:58.999Z 23 Acked 1022 Write 1 Sent Done Done false
25112024-02-28T22:13:58.999Z 24 Acked 1023 Write 9 Sent Done Done false
25122024-02-28T22:13:58.999Z 25 Acked 1024 Read 3 Sent Done Done false
25132024-02-28T22:13:58.999Z 26 Acked 1025 Flush 0 Sent Done Done false
25142024-02-28T22:13:58.999Z 27 Acked 1026 Write 6 Sent Done Done false
25152024-02-28T22:13:58.999Z 28 Acked 1027 Write 8 Sent Done Done false
25162024-02-28T22:13:58.999Z 29 Acked 1028 Read 6 Sent Done Done false
25172024-02-28T22:13:58.999Z 30 Acked 1029 Read 7 Sent Done Done false
25182024-02-28T22:13:58.999Z STATES DS:0 DS:1 DS:2 TOTAL
25192024-02-28T22:13:58.999Z New 0 0 0 0
25202024-02-28T22:13:58.999Z Sent 23 0 0 23
25212024-02-28T22:13:58.999Z Done 7 30 30 67
25222024-02-28T22:13:58.999Z Skipped 0 0 0 0
25232024-02-28T22:13:58.999Z Error 0 0 0 0
25242024-02-28T22:13:58.999ZLast Flush: 0 1025 1025
25252024-02-28T22:13:58.999ZDownstairs last five completed:
25262024-02-28T22:13:58.999ZUpstairs last five completed: 30 29 28 27 26
25272024-02-28T22:13:58.999Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:13:57.89700953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1223}
25282024-02-28T22:13:58.999ZWrote out file "/tmp/test_repair_verify.data"
25292024-02-28T22:13:59.004Z
25302024-02-28T22:13:59.007ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25312024-02-28T22:13:59.021ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25322024-02-28T22:13:59.022Z 0 000-019 21 22 22 38 39 39 T F F <---
25332024-02-28T22:13:59.022Z 1 020-039 17 17 17 30 30 30 F F F
25342024-02-28T22:13:59.022Z 2 040-059 13 13 13 23 23 23 F F F
25352024-02-28T22:13:59.022Z 3 060-079 14 14 14 26 26 26 F F F
25362024-02-28T22:13:59.022Z 4 080-099 20 20 20 37 37 37 F F F
25372024-02-28T22:13:59.030Z 5 100-119 14 14 14 24 24 24 F F F
25382024-02-28T22:13:59.030Z 6 120-139 20 22 22 36 41 41 F F F <---
25392024-02-28T22:13:59.030Z 7 140-159 20 22 22 36 40 40 F F F <---
25402024-02-28T22:13:59.031Z 8 160-179 20 20 20 37 37 37 F F F
25412024-02-28T22:13:59.031Z 9 180-199 20 20 20 37 37 37 F F F
25422024-02-28T22:13:59.031Z 10 200-219 20 20 20 36 36 36 F F F
25432024-02-28T22:13:59.031Z 11 220-239 17 22 22 30 39 39 F F F <---
25442024-02-28T22:13:59.031Z 12 240-259 18 18 18 34 34 34 F F F
25452024-02-28T22:13:59.031Z 13 260-279 20 20 20 36 36 36 F F F
25462024-02-28T22:13:59.031Z 14 280-299 18 18 18 32 32 32 F F F
25472024-02-28T22:13:59.031Z 15 300-319 15 22 22 27 41 41 F T T <---
25482024-02-28T22:13:59.031Z 16 320-339 19 22 22 35 41 41 F T T <---
25492024-02-28T22:13:59.031Z 17 340-359 16 22 22 29 40 40 F F F <---
25502024-02-28T22:13:59.031Z 18 360-379 20 20 20 36 36 36 F F F
25512024-02-28T22:13:59.031Z 19 380-399 21 21 21 38 38 38 F F F
25522024-02-28T22:13:59.031Z 20 400-419 20 20 20 36 36 36 F F F
25532024-02-28T22:13:59.031Z 21 420-439 20 20 20 36 36 36 F F F
25542024-02-28T22:13:59.031Z 22 440-459 16 16 16 28 28 28 F F F
25552024-02-28T22:13:59.031Z 23 460-479 20 20 20 36 36 36 F F F
25562024-02-28T22:13:59.031Z 24 480-499 18 18 18 32 32 32 F F F
25572024-02-28T22:13:59.031Z 25 500-519 18 22 22 31 39 39 F F F <---
25582024-02-28T22:13:59.031Z 26 520-539 20 20 20 37 37 37 F F F
25592024-02-28T22:13:59.031Z 27 540-559 15 15 15 27 27 27 F F F
25602024-02-28T22:13:59.032Z 28 560-579 21 21 21 38 38 38 F F F
25612024-02-28T22:13:59.032Z 29 580-599 14 22 22 25 39 39 T F F <---
25622024-02-28T22:13:59.032ZMax gen: 22, Max flush: 41
25632024-02-28T22:13:59.032ZError: Difference in extent metadata found!
25642024-02-28T22:13:59.032ZA Difference in extent metadata is expected here
25652024-02-28T22:13:59.032ZOn loop 11
25662024-02-28T22:13:59.032Z
25672024-02-28T22:13:59.032ZVerifying data now
25682024-02-28T22:14:03.835ZNo change after verify
25692024-02-28T22:14:03.835ZLoop: 11 Downstairs dump after verify (and repair):
25702024-02-28T22:14:03.849ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25712024-02-28T22:14:03.849Z 0 000-019 22 22 22 39 39 39 F F F
25722024-02-28T22:14:03.849Z 1 020-039 17 17 17 30 30 30 F F F
25732024-02-28T22:14:03.849Z 2 040-059 13 13 13 23 23 23 F F F
25742024-02-28T22:14:03.849Z 3 060-079 14 14 14 26 26 26 F F F
25752024-02-28T22:14:03.849Z 4 080-099 20 20 20 37 37 37 F F F
25762024-02-28T22:14:03.850Z 5 100-119 14 14 14 24 24 24 F F F
25772024-02-28T22:14:03.850Z 6 120-139 22 22 22 41 41 41 F F F
25782024-02-28T22:14:03.850Z 7 140-159 22 22 22 40 40 40 F F F
25792024-02-28T22:14:03.850Z 8 160-179 20 20 20 37 37 37 F F F
25802024-02-28T22:14:03.850Z 9 180-199 20 20 20 37 37 37 F F F
25812024-02-28T22:14:03.853Z 10 200-219 20 20 20 36 36 36 F F F
25822024-02-28T22:14:03.853Z 11 220-239 22 22 22 39 39 39 F F F
25832024-02-28T22:14:03.853Z 12 240-259 18 18 18 34 34 34 F F F
25842024-02-28T22:14:03.853Z 13 260-279 20 20 20 36 36 36 F F F
25852024-02-28T22:14:03.853Z 14 280-299 18 18 18 32 32 32 F F F
25862024-02-28T22:14:03.853Z 15 300-319 23 23 23 42 42 42 F F F
25872024-02-28T22:14:03.854Z 16 320-339 23 23 23 42 42 42 F F F
25882024-02-28T22:14:03.854Z 17 340-359 22 22 22 40 40 40 F F F
25892024-02-28T22:14:03.854Z 18 360-379 20 20 20 36 36 36 F F F
25902024-02-28T22:14:03.854Z 19 380-399 21 21 21 38 38 38 F F F
25912024-02-28T22:14:03.854Z 20 400-419 20 20 20 36 36 36 F F F
25922024-02-28T22:14:03.854Z 21 420-439 20 20 20 36 36 36 F F F
25932024-02-28T22:14:03.854Z 22 440-459 16 16 16 28 28 28 F F F
25942024-02-28T22:14:03.854Z 23 460-479 20 20 20 36 36 36 F F F
25952024-02-28T22:14:03.854Z 24 480-499 18 18 18 32 32 32 F F F
25962024-02-28T22:14:03.854Z 25 500-519 22 22 22 39 39 39 F F F
25972024-02-28T22:14:03.854Z 26 520-539 20 20 20 37 37 37 F F F
25982024-02-28T22:14:03.854Z 27 540-559 15 15 15 27 27 27 F F F
25992024-02-28T22:14:03.854Z 28 560-579 21 21 21 38 38 38 F F F
26002024-02-28T22:14:03.854Z 29 580-599 22 22 22 39 39 39 F F F
26012024-02-28T22:14:03.854ZMax gen: 23, Max flush: 42
26022024-02-28T22:14:03.854Z
26032024-02-28T22:14:03.854ZBegin loop 12
26042024-02-28T22:14:03.854ZDownstairs to restart: 0
26052024-02-28T22:14:03.871Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771271069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
26062024-02-28T22:14:03.872Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
26072024-02-28T22:14:03.872Z,"time":"2024-02-28T22:14:02.77157953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
26082024-02-28T22:14:03.879Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771615969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
26092024-02-28T22:14:03.879Z{"msg":"Crucible 3a2842c4-f238-41b8-8ed9-525744d3b1df has session id: 36e11e69-5937-4ac0-a38f-c95707c80b54","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771656628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26102024-02-28T22:14:03.879Z{"msg":"Upstairs opts: Upstairs UUID: 3a2842c4-f238-41b8-8ed9-525744d3b1df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771701987Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26112024-02-28T22:14:03.879Z{"msg":"Crucible stats registered with UUID: 3a2842c4-f238-41b8-8ed9-525744d3b1df","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771750626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26122024-02-28T22:14:03.879Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771782965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
26132024-02-28T22:14:03.879Z{"msg":"3a2842c4-f238-41b8-8ed9-525744d3b1df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771811614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26142024-02-28T22:14:03.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771866452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26152024-02-28T22:14:03.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771899421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26162024-02-28T22:14:03.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77193776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26172024-02-28T22:14:03.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77196678Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26182024-02-28T22:14:03.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.771995259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26192024-02-28T22:14:03.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772118925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26202024-02-28T22:14:03.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772178444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26212024-02-28T22:14:03.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772228872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26222024-02-28T22:14:03.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772255252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26232024-02-28T22:14:03.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772287361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26242024-02-28T22:14:03.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77230404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26252024-02-28T22:14:03.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772335549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26262024-02-28T22:14:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772786507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26272024-02-28T22:14:03.880Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772830956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26282024-02-28T22:14:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772874504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26292024-02-28T22:14:03.880Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.772969082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26302024-02-28T22:14:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77303509Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26312024-02-28T22:14:03.880Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.773106638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26322024-02-28T22:14:03.880Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.773457139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26332024-02-28T22:14:03.880Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.773550276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26342024-02-28T22:14:03.880Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.773596045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26352024-02-28T22:14:03.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774050492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26362024-02-28T22:14:03.880Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774099551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26372024-02-28T22:14:03.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77411827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26382024-02-28T22:14:03.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774150669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26392024-02-28T22:14:03.880Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774170679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26402024-02-28T22:14:03.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774201268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26412024-02-28T22:14:03.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774216327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26422024-02-28T22:14:03.880Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774247847Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26432024-02-28T22:14:03.880Z{"msg":"[0]R flush_numbers[0..12]: [39, 30, 23, 26, 37, 24, 41, 40, 37, 37, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774268166Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26442024-02-28T22:14:03.880Z{"msg":"[0]R generation[0..12]: [22, 17, 13, 14, 20, 14, 22, 22, 20, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774298145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26452024-02-28T22:14:03.880Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774314915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26462024-02-28T22:14:03.880Z{"msg":"[1]R flush_numbers[0..12]: [39, 30, 23, 26, 37, 24, 41, 40, 37, 37, 36, 39]","v":0,"nameWait for a query_work_queue command to finish before sending IO
26472024-02-28T22:14:03.881Z":"crucible","level":30,"time":"2024-02-28T22:14:02.774338064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26482024-02-28T22:14:03.881Z{"msg":"[1]R generation[0..12]: [22, 17, 13, 14, 20, 14, 22, 22, 20, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774369993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26492024-02-28T22:14:03.881Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774385953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26502024-02-28T22:14:03.881Z{"msg":"[2]R flush_numbers[0..12]: [39, 30, 23, 26, 37, 24, 41, 40, 37, 37, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774408432Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26512024-02-28T22:14:03.881Z{"msg":"[2]R generation[0..12]: [22, 17, 13, 14, 20, 14, 22, 22, 20, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774435241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26522024-02-28T22:14:03.881Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774450321Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26532024-02-28T22:14:03.881Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774470351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26542024-02-28T22:14:03.881Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.77449338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26552024-02-28T22:14:03.881Z{"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774521469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26562024-02-28T22:14:03.881Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774537379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"":"downstairs","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26572024-02-28T22:14:03.881Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774557988Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26582024-02-28T22:14:03.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774585287Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"0","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26592024-02-28T22:14:03.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774599997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"1","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26602024-02-28T22:14:03.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774619827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"client":"2","":"downstairs","session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26612024-02-28T22:14:03.881Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774661055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26622024-02-28T22:14:03.881Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774675775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26632024-02-28T22:14:03.881Z{"msg":"3a2842c4-f238-41b8-8ed9-525744d3b1df is now active with session: 36e11e69-5937-4ac0-a38f-c95707c80b54","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774694554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26642024-02-28T22:14:03.881Z{"msg":"Set Active after no reconciliation","v":0,"nameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26652024-02-28T22:14:03.881Z":"crucible","level":30,"time":"2024-02-28T22:14:02.774724653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231,"session_id":"36e11e69-5937-4ac0-a38f-c95707c80b54"}
26662024-02-28T22:14:03.881Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:02.774738843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
26672024-02-28T22:14:03.882ZLoading write count information from file "/tmp/test_repair_verify.data"
26682024-02-28T22:14:03.882ZRun Repair workload
26692024-02-28T22:14:03.882Z01/30 Write block 403 len 40960 data: 3 3 3 3 4 4 4 5 3 3
26702024-02-28T22:14:03.886Z02/30 Write block 247 len 32768 data: 5 5 4 3 4 3 3 2
26712024-02-28T22:14:03.886Z03/30 Read block 191 len 36864
26722024-02-28T22:14:03.886Z04/30 Read block 98 len 8192
26732024-02-28T22:14:03.886Z05/30 Read block 111 len 4096
26742024-02-28T22:14:03.886Z06/30 Read block 319 len 20480
26752024-02-28T22:14:03.886Z07/30 Read block 362 len 4096
26762024-02-28T22:14:03.887Z08/30 Read block 214 len 20480
26772024-02-28T22:14:03.887Z09/30 Read block 364 len 8192
26782024-02-28T22:14:03.887Z10/30 Read block 396 len 4096
26792024-02-28T22:14:03.887Z11/30 Read block 269 len 24576
26802024-02-28T22:14:03.887Z12/30 Flush
26812024-02-28T22:14:03.891Z13/30 Flush
26822024-02-28T22:14:03.891Z14/30 Write block 509 len 8192 data: 4 4
26832024-02-28T22:14:03.891Z15/30 Read block 19 len 40960
26842024-02-28T22:14:03.891Z16/30 Write block 541 len 36864 data: 2 3 3 3 4 5 5 2 3
26852024-02-28T22:14:03.891Z17/30 Write block 3 len 32768 data: 4 4 4 3 4 4 3 3
26862024-02-28T22:14:03.891Z18/30 Flush
26872024-02-28T22:14:03.894Z19/30 Write block 48 len 16384 data: 3 4 3 3
26882024-02-28T22:14:03.894Z20/30 Write block 164 len 28672 data: 3 4 4 5 4 4 4
26892024-02-28T22:14:03.894Z21/30 Read block 529 len 32768
26902024-02-28T22:14:03.898Z22/30 Write block 372 len 32768 data: 5 4 4 4 3 3 3 3
26912024-02-28T22:14:03.898Z23/30 Read block 431 len 24576
26922024-02-28T22:14:03.898Z24/30 Read block 356 len 28672
26932024-02-28T22:14:03.901Z25/30 Flush
26942024-02-28T22:14:03.904Z26/30 Write block 446 len 8192 data: 3 3
26952024-02-28T22:14:03.904Z27/30 Read block 48 len 32768
26962024-02-28T22:14:03.904Z28/30 Write block 155 len 20480 data: 3 3 3 3 3
26972024-02-28T22:14:03.904Z29/30 Write block 563 len 32768 data: 4 4 4 4 5 5 4 3
26982024-02-28T22:14:03.904Z30/30 Read block 17 len 4096
26992024-02-28T22:14:03.909Z----------------------------------------------------------------
27002024-02-28T22:14:03.909Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
27012024-02-28T22:14:03.909ZStates: Active Active Active
27022024-02-28T22:14:03.909ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27032024-02-28T22:14:03.909Z 1 Acked 1000 Write 10 Done Done Done false
27042024-02-28T22:14:03.909Z 2 Acked 1001 Write 8 Done Done Done false
27052024-02-28T22:14:03.909Z 3 Acked 1002 Read 9 Done Done Done false
27062024-02-28T22:14:03.909Z 4 Acked 1003 Read 2 Done Done Done false
27072024-02-28T22:14:03.909Z 5 Acked 1004 Read 1 Done Done Done false
27082024-02-28T22:14:03.909Z 6 Acked 1005 Read 5 Done Done Done false
27092024-02-28T22:14:03.909Z 7 Acked 1006 Read 1 Sent Done Done false
27102024-02-28T22:14:03.909Z 8 Acked 1007 Read 5 Sent Done Done false
27112024-02-28T22:14:03.909Z 9 Acked 1008 Read 2 Sent Done Done false
27122024-02-28T22:14:03.909Z 10 Acked 1009 Read 1 Sent Done Done false
27132024-02-28T22:14:03.909Z 11 Acked 1010 Read 6 Sent Done Done false
27142024-02-28T22:14:03.909Z 12 Acked 1011 Flush 0 Sent Done Done false
27152024-02-28T22:14:03.909Z 13 Acked 1012 Flush 0 Sent Done Done false
27162024-02-28T22:14:03.910Z 14 Acked 1013 Write 2 Sent Done Done false
27172024-02-28T22:14:03.910Z 15 Acked 1014 Read 10 Sent Done Done false
27182024-02-28T22:14:03.910Z 16 Acked 1015 Write 9 Sent Done Done false
27192024-02-28T22:14:03.910Z 17 Acked 1016 Write 8 Sent Done Done false
27202024-02-28T22:14:03.910Z 18 Acked 1017 Flush 0 Sent Done Done false
27212024-02-28T22:14:03.910Z 19 Acked 1018 Write 4 Sent Done Done false
27222024-02-28T22:14:03.910Z 20 Acked 1019 Write 7 Sent Done Done false
27232024-02-28T22:14:03.910Z 21 Acked 1020 Read 8 Sent Done Done false
27242024-02-28T22:14:03.910Z 22 Acked 1021 Write 8 Sent Done Done false
27252024-02-28T22:14:03.910Z 23 Acked 1022 Read 6 Sent Done Done false
27262024-02-28T22:14:03.910Z 24 Acked 1023 Read 7 Sent Done Done false
27272024-02-28T22:14:03.910Z 25 Acked 1024 Flush 0 Sent Done Done false
27282024-02-28T22:14:03.910Z 26 Acked 1025 Write 2 Sent Done Done false
27292024-02-28T22:14:03.910Z 27 Acked 1026 Read 8 Sent Done Done false
27302024-02-28T22:14:03.910Z 28 Acked 1027 Write 5 Sent Done Done false
27312024-02-28T22:14:03.910Z 29 Acked 1028 Write 8 Sent Done Done false
27322024-02-28T22:14:03.910Z 30 Acked 1029 Read 1 Sent Done Done false
27332024-02-28T22:14:03.910Z STATES DS:0 DS:1 DS:2 TOTAL
27342024-02-28T22:14:03.910Z New 0 0 0 0
27352024-02-28T22:14:03.910Z Sent 24 0 0 24
27362024-02-28T22:14:03.910Z Done 6 30 30 66
27372024-02-28T22:14:03.910Z Skipped 0 0 0 0
27382024-02-28T22:14:03.910Z Error 0 0 0 0
27392024-02-28T22:14:03.911ZLast Flush: 0 1024 1024
27402024-02-28T22:14:03.911ZDownstairs last five completed:
27412024-02-28T22:14:03.911ZUpstairs last five completed: 30 29 28 27 26
27422024-02-28T22:14:03.911Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:02.806013482Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1231}
27432024-02-28T22:14:03.911ZWrote out file "/tmp/test_repair_verify.data"
27442024-02-28T22:14:03.915Z
27452024-02-28T22:14:03.915ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27462024-02-28T22:14:03.928ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27472024-02-28T22:14:03.928Z 0 000-019 22 24 24 39 45 45 F F F <---
27482024-02-28T22:14:03.928Z 1 020-039 17 17 17 30 30 30 F F F
27492024-02-28T22:14:03.928Z 2 040-059 13 24 24 23 46 46 F F F <---
27502024-02-28T22:14:03.928Z 3 060-079 14 14 14 26 26 26 F F F
27512024-02-28T22:14:03.928Z 4 080-099 20 20 20 37 37 37 F F F
27522024-02-28T22:14:03.928Z 5 100-119 14 14 14 24 24 24 F F F
27532024-02-28T22:14:03.929Z 6 120-139 22 22 22 41 41 41 F F F
27542024-02-28T22:14:03.929Z 7 140-159 22 22 22 40 40 40 F T T <---
27552024-02-28T22:14:03.929Z 8 160-179 20 24 24 37 46 46 F F F <---
27562024-02-28T22:14:03.929Z 9 180-199 20 20 20 37 37 37 F F F
27572024-02-28T22:14:03.929Z 10 200-219 20 20 20 36 36 36 F F F
27582024-02-28T22:14:03.929Z 11 220-239 22 22 22 39 39 39 F F F
27592024-02-28T22:14:03.929Z 12 240-259 18 24 24 34 43 43 T F F <---
27602024-02-28T22:14:03.933Z 13 260-279 20 20 20 36 36 36 F F F
27612024-02-28T22:14:03.933Z 14 280-299 18 18 18 32 32 32 F F F
27622024-02-28T22:14:03.933Z 15 300-319 23 23 23 42 42 42 F F F
27632024-02-28T22:14:03.933Z 16 320-339 23 23 23 42 42 42 F F F
27642024-02-28T22:14:03.933Z 17 340-359 22 22 22 40 40 40 F F F
27652024-02-28T22:14:03.933Z 18 360-379 20 24 24 36 46 46 F F F <---
27662024-02-28T22:14:03.933Z 19 380-399 21 21 21 38 38 38 F F F
27672024-02-28T22:14:03.933Z 20 400-419 20 24 24 36 43 43 T F F <---
27682024-02-28T22:14:03.933Z 21 420-439 20 20 20 36 36 36 F F F
27692024-02-28T22:14:03.933Z 22 440-459 16 16 16 28 28 28 F T T <---
27702024-02-28T22:14:03.933Z 23 460-479 20 20 20 36 36 36 F F F
27712024-02-28T22:14:03.933Z 24 480-499 18 18 18 32 32 32 F F F
27722024-02-28T22:14:03.933Z 25 500-519 22 24 24 39 45 45 F F F <---
27732024-02-28T22:14:03.933Z 26 520-539 20 20 20 37 37 37 F F F
27742024-02-28T22:14:03.933Z 27 540-559 15 24 24 27 45 45 F F F <---
27752024-02-28T22:14:03.933Z 28 560-579 21 21 21 38 38 38 F T T <---
27762024-02-28T22:14:03.934Z 29 580-599 22 22 22 39 39 39 F F F
27772024-02-28T22:14:03.934ZMax gen: 24, Max flush: 46
27782024-02-28T22:14:03.934ZError: Difference in extent metadata found!
27792024-02-28T22:14:03.934ZA Difference in extent metadata is expected here
27802024-02-28T22:14:03.934ZOn loop 12
27812024-02-28T22:14:03.934Z
27822024-02-28T22:14:03.937ZVerifying data now
27832024-02-28T22:14:09.946ZNo change after verify
27842024-02-28T22:14:09.946ZLoop: 12 Downstairs dump after verify (and repair):
27852024-02-28T22:14:09.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27862024-02-28T22:14:09.960Z 0 000-019 24 24 24 45 45 45 F F F
27872024-02-28T22:14:09.960Z 1 020-039 17 17 17 30 30 30 F F F
27882024-02-28T22:14:09.960Z 2 040-059 24 24 24 46 46 46 F F F
27892024-02-28T22:14:09.960Z 3 060-079 14 14 14 26 26 26 F F F
27902024-02-28T22:14:09.960Z 4 080-099 20 20 20 37 37 37 F F F
27912024-02-28T22:14:09.960Z 5 100-119 14 14 14 24 24 24 F F F
27922024-02-28T22:14:09.961Z 6 120-139 22 22 22 41 41 41 F F F
27932024-02-28T22:14:09.961Z 7 140-159 25 25 25 47 47 47 F F F
27942024-02-28T22:14:09.961Z 8 160-179 24 24 24 46 46 46 F F F
27952024-02-28T22:14:09.961Z 9 180-199 20 20 20 37 37 37 F F F
27962024-02-28T22:14:09.961Z 10 200-219 20 20 20 36 36 36 F F F
27972024-02-28T22:14:09.961Z 11 220-239 22 22 22 39 39 39 F F F
27982024-02-28T22:14:09.961Z 12 240-259 24 24 24 43 43 43 F F F
27992024-02-28T22:14:09.961Z 13 260-279 20 20 20 36 36 36 F F F
28002024-02-28T22:14:09.961Z 14 280-299 18 18 18 32 32 32 F F F
28012024-02-28T22:14:09.961Z 15 300-319 23 23 23 42 42 42 F F F
28022024-02-28T22:14:09.976Z 16 320-339 23 23 23 42 42 42 F F F
28032024-02-28T22:14:09.976Z 17 340-359 22 22 22 40 40 40 F F F
28042024-02-28T22:14:09.977Z 18 360-379 24 24 24 46 46 46 F F F
28052024-02-28T22:14:09.977Z 19 380-399 21 21 21 38 38 38 F F F
28062024-02-28T22:14:09.977Z 20 400-419 24 24 24 43 43 43 F F F
28072024-02-28T22:14:09.977Z 21 420-439 20 20 20 36 36 36 F F F
28082024-02-28T22:14:09.977Z 22 440-459 25 25 25 47 47 47 F F F
28092024-02-28T22:14:09.977Z 23 460-479 20 20 20 36 36 36 F F F
28102024-02-28T22:14:09.977Z 24 480-499 18 18 18 32 32 32 F F F
28112024-02-28T22:14:09.977Z 25 500-519 24 24 24 45 45 45 F F F
28122024-02-28T22:14:09.977Z 26 520-539 20 20 20 37 37 37 F F F
28132024-02-28T22:14:09.977Z 27 540-559 24 24 24 45 45 45 F F F
28142024-02-28T22:14:09.977Z 28 560-579 25 25 25 47 47 47 F F F
28152024-02-28T22:14:09.978Z 29 580-599 22 22 22 39 39 39 F F F
28162024-02-28T22:14:09.978ZMax gen: 25, Max flush: 47
28172024-02-28T22:14:09.978Z
28182024-02-28T22:14:09.978ZBegin loop 13
28192024-02-28T22:14:09.978ZDownstairs to restart: 1
28202024-02-28T22:14:09.989Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.880935793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
28212024-02-28T22:14:09.989Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\"Crucible runtime is spawned
28222024-02-28T22:14:09.989Z,\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881228225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
28232024-02-28T22:14:09.989Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881269184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
28242024-02-28T22:14:09.989Z{"msg":"Crucible 3c0edf72-e284-40a0-a21c-9f50fcc7ee6a has session id: 705e5c91-ea49-4182-98e5-e58c7c343454","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881288894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28252024-02-28T22:14:09.989Z{"msg":"Upstairs opts: Upstairs UUID: 3c0edf72-e284-40a0-a21c-9f50fcc7ee6a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881336542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28262024-02-28T22:14:09.989Z{"msg":"Crucible stats registered with UUID: 3c0edf72-e284-40a0-a21c-9f50fcc7ee6a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881374812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28272024-02-28T22:14:09.990Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881446279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
28282024-02-28T22:14:09.990Z{"msg":"3c0edf72-e284-40a0-a21c-9f50fcc7ee6a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881483909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28292024-02-28T22:14:09.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881508238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28302024-02-28T22:14:09.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881539487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28312024-02-28T22:14:09.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881561376Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28322024-02-28T22:14:09.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881599045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28332024-02-28T22:14:09.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881632274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28342024-02-28T22:14:09.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881666713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28352024-02-28T22:14:09.990Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881693253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28362024-02-28T22:14:09.990Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881851588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28372024-02-28T22:14:09.991Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881883187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28382024-02-28T22:14:09.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881907027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28392024-02-28T22:14:09.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881941716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28402024-02-28T22:14:09.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.881967915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28412024-02-28T22:14:09.991Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.882620687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28422024-02-28T22:14:09.991Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.882692865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28432024-02-28T22:14:09.991Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.882746644Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28442024-02-28T22:14:09.991Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.882788262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28452024-02-28T22:14:09.992Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.882827261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28462024-02-28T22:14:09.992Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.8828759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28472024-02-28T22:14:09.992Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883289669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28482024-02-28T22:14:09.992Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883449084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28492024-02-28T22:14:09.992Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883734716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28502024-02-28T22:14:09.992Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883782945Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28512024-02-28T22:14:09.992Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883808425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28522024-02-28T22:14:09.992Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883832564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28532024-02-28T22:14:09.992Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.883943721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28542024-02-28T22:14:09.993Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.88397416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28552024-02-28T22:14:09.993Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.88399052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28562024-02-28T22:14:09.993Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884207614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28572024-02-28T22:14:09.993Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884239893Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28582024-02-28T22:14:09.993Z{"msg":"[0]R flush_numbers[0..12]: [45, 30, 46, 26, 37, 24, 41, 47, 46, 37, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.8843358Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28592024-02-28T22:14:09.993Z{"msg":"[0]R generation[0..12]: [24, 17, 24, 14, 20, 14, 22, 25, 24, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884362339Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28602024-02-28T22:14:09.993Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884377709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28612024-02-28T22:14:09.993Z{"msg":"[1]R flush_numbers[0..12]: [45, 30, 46, 26, 37, 24, 41, 47, 46, 37, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884393158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28622024-02-28T22:14:09.993Z{"msg":"[1]R generation[0..12]: [24, 17, 24, 14, 20, 14, 22, 25, 24, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884407378Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28632024-02-28T22:14:09.993Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884421608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28642024-02-28T22:14:09.993Z{"msg":"[2]R flush_numbers[0..12]: [45, 30, 46, 26, 37, 24, 41, 47, 46, 37, 36, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884443917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28652024-02-28T22:14:09.994Z{"msg":"[2]R generation[0..12]: [24, 17, 24, 14, 20, 14, 22, 25, 24, 20, 20, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884458156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28662024-02-28T22:14:09.994Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
28672024-02-28T22:14:09.994Zcrucible","level":30,"time":"2024-02-28T22:14:08.884477586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28682024-02-28T22:14:09.994Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884490926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28692024-02-28T22:14:09.994Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884505025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28702024-02-28T22:14:09.994Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884517905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28712024-02-28T22:14:09.994Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884538944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"":"downstairs","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28722024-02-28T22:14:09.994Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884553884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28732024-02-28T22:14:09.994Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884566193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"0","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28742024-02-28T22:14:09.994Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884579563Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"1","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28752024-02-28T22:14:09.994Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884592873Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"client":"2","":"downstairs","session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28762024-02-28T22:14:09.995Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884613962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28772024-02-28T22:14:09.995Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884626812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28782024-02-28T22:14:09.995Z{"msg":"3c0edf72-e284-40a0-a21c-9f50fcc7ee6a is now active with session: 705e5c91-ea49-4182-98e5-e58c7c343454","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884640002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28792024-02-28T22:14:09.995Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884652871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239,"session_id":"705e5c91-ea49-4182-98e5-e58c7c343454"}
28802024-02-28T22:14:09.995Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:08.884673131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
28812024-02-28T22:14:10.000ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28822024-02-28T22:14:10.000ZLoading write count information from file "/tmp/test_repair_verify.data"
28832024-02-28T22:14:10.001ZRun Repair workload
28842024-02-28T22:14:10.001Z01/30 Write block 550 len 24576 data: 3 3 3 3 3 3
28852024-02-28T22:14:10.001Z02/30 Write block 566 len 28672 data: 5 6 6 5 4 3 3
28862024-02-28T22:14:10.001Z03/30 Write block 575 len 28672 data: 3 2 2 2 2 2 2
28872024-02-28T22:14:10.001Z04/30 Read block 496 len 36864
28882024-02-28T22:14:10.001Z05/30 Write block 484 len 20480 data: 3 4 3 3 3
28892024-02-28T22:14:10.001Z06/30 Read block 551 len 28672
28902024-02-28T22:14:10.001Z07/30 Write block 445 len 24576 data: 3 4 4 3 3 3
28912024-02-28T22:14:10.001Z08/30 Write block 418 len 36864 data: 2 2 3 3 3 3 3 4 3
28922024-02-28T22:14:10.002Z09/30 Write block 59 len 40960 data: 3 3 3 4 5 5 5 5 6 5
28932024-02-28T22:14:10.002Z10/30 Write block 544 len 32768 data: 4 5 6 6 3 4 4 4
28942024-02-28T22:14:10.002Z11/30 Write block 246 len 36864 data: 3 6 6 5 4 5 4 4 3
28952024-02-28T22:14:10.002Z12/30 Read block 360 len 28672
28962024-02-28T22:14:10.002Z13/30 Write block 390 len 28672 data: 4 4 3 3 3 4 4
28972024-02-28T22:14:10.002Z14/30 Write block 202 len 16384 data: 3 2 2 4
28982024-02-28T22:14:10.002Z15/30 Read block 408 len 32768
28992024-02-28T22:14:10.006Z16/30 Read block 235 len 8192
29002024-02-28T22:14:10.006Z17/30 Read block 395 len 8192
29012024-02-28T22:14:10.007Z18/30 Read block 595 len 20480
29022024-02-28T22:14:10.007Z19/30 Write block 161 len 36864 data: 2 2 2 4 5 5 6 5 5
29032024-02-28T22:14:10.007Z20/30 Write block 307 len 32768 data: 5 4 3 2 3 3 4 4
29042024-02-28T22:14:10.007Z21/30 Write block 585 len 8192 data: 2 2
29052024-02-28T22:14:10.007Z22/30 Write block 524 len 20480 data: 4 3 3 3 3
29062024-02-28T22:14:10.007Z23/30 Write block 335 len 32768 data: 4 3 3 4 4 4 4 4
29072024-02-28T22:14:10.007Z24/30 Write block 13 len 28672 data: 3 3 3 4 4 4 5
29082024-02-28T22:14:10.014Z25/30 Read block 227 len 36864
29092024-02-28T22:14:10.014Z26/30 Read block 121 len 12288
29102024-02-28T22:14:10.014Z27/30 Write block 489 len 40960 data: 3 3 3 3 3 2 4 4 4 4
29112024-02-28T22:14:10.014Z28/30 Read block 507 len 20480
29122024-02-28T22:14:10.015Z29/30 Read block 165 len 24576
29132024-02-28T22:14:10.015Z30/30 Read block 585 len 20480
29142024-02-28T22:14:10.015Z----------------------------------------------------------------
29152024-02-28T22:14:10.015Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29162024-02-28T22:14:10.015ZStates: Active Active Active
29172024-02-28T22:14:10.015ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29182024-02-28T22:14:10.016Z 1 Acked 1000 Write 6 Done Done Done false
29192024-02-28T22:14:10.016Z 2 Acked 1001 Write 7 Done Done Done false
29202024-02-28T22:14:10.016Z 3 Acked 1002 Write 7 Done Done Done false
29212024-02-28T22:14:10.016Z 4 Acked 1003 Read 9 Done Done Done false
29222024-02-28T22:14:10.016Z 5 Acked 1004 Write 5 Done Sent Done false
29232024-02-28T22:14:10.016Z 6 Acked 1005 Read 7 Done Sent Done false
29242024-02-28T22:14:10.016Z 7 Acked 1006 Write 6 Done Sent Done false
29252024-02-28T22:14:10.016Z 8 Acked 1007 Write 9 Done Sent Done false
29262024-02-28T22:14:10.016Z 9 Acked 1008 Write 10 Done Sent Done false
29272024-02-28T22:14:10.016Z 10 Acked 1009 Write 8 Done Sent Done false
29282024-02-28T22:14:10.016Z 11 Acked 1010 Write 9 Done Sent Done false
29292024-02-28T22:14:10.017Z 12 Acked 1011 Read 7 Done Sent Done false
29302024-02-28T22:14:10.017Z 13 Acked 1012 Write 7 Done Sent Done false
29312024-02-28T22:14:10.017Z 14 Acked 1013 Write 4 Done Sent Done false
29322024-02-28T22:14:10.017Z 15 Acked 1014 Read 8 Done Sent Done false
29332024-02-28T22:14:10.017Z 16 Acked 1015 Read 2 Done Sent Done false
29342024-02-28T22:14:10.017Z 17 Acked 1016 Read 2 Done Sent Done false
29352024-02-28T22:14:10.017Z 18 Acked 1017 Read 5 Done Sent Done false
29362024-02-28T22:14:10.017Z 19 Acked 1018 Write 9 Done Sent Done false
29372024-02-28T22:14:10.017Z 20 Acked 1019 Write 8 Done Sent Done false
29382024-02-28T22:14:10.017Z 21 Acked 1020 Write 2 Done Sent Done false
29392024-02-28T22:14:10.018Z 22 Acked 1021 Write 5 Done Sent Done false
29402024-02-28T22:14:10.018Z 23 Acked 1022 Write 8 Done Sent Done false
29412024-02-28T22:14:10.018Z 24 Acked 1023 Write 7 Done Sent Done false
29422024-02-28T22:14:10.018Z 25 Acked 1024 Read 9 Done Sent Done false
29432024-02-28T22:14:10.018Z 26 Acked 1025 Read 3 Done Sent Done false
29442024-02-28T22:14:10.018Z 27 Acked 1026 Write 10 Done Sent Done false
29452024-02-28T22:14:10.018Z 28 Acked 1027 Read 5 Done Sent Done false
29462024-02-28T22:14:10.018Z 29 Acked 1028 Read 6 Done Sent Done false
29472024-02-28T22:14:10.018Z 30 Acked 1029 Read 5 Done Sent Done false
29482024-02-28T22:14:10.019Z STATES DS:0 DS:1 DS:2 TOTAL
29492024-02-28T22:14:10.019Z New 0 0 0 0
29502024-02-28T22:14:10.019Z Sent 0 26 0 26
29512024-02-28T22:14:10.019Z Done 30 4 30 64
29522024-02-28T22:14:10.019Z Skipped 0 0 0 0
29532024-02-28T22:14:10.019Z Error 0 0 0 0
29542024-02-28T22:14:10.019ZLast Flush: 0 0 0
29552024-02-28T22:14:10.019ZDownstairs last five completed:
29562024-02-28T22:14:10.019ZUpstairs last five completed: 30 29 28 27 26
29572024-02-28T22:14:10.019Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:08.912942142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1239}
29582024-02-28T22:14:10.019ZWrote out file "/tmp/test_repair_verify.data"
29592024-02-28T22:14:10.024Z
29602024-02-28T22:14:10.024ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29612024-02-28T22:14:10.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29622024-02-28T22:14:10.049Z 0 000-019 24 24 24 45 45 45 T F T <---
29632024-02-28T22:14:10.049Z 1 020-039 17 17 17 30 30 30 F F F
29642024-02-28T22:14:10.053Z 2 040-059 24 24 24 46 46 46 T F T <---
29652024-02-28T22:14:10.054Z 3 060-079 14 14 14 26 26 26 T F T <---
29662024-02-28T22:14:10.054Z 4 080-099 20 20 20 37 37 37 F F F
29672024-02-28T22:14:10.054Z 5 100-119 14 14 14 24 24 24 F F F
29682024-02-28T22:14:10.054Z 6 120-139 22 22 22 41 41 41 F F F
29692024-02-28T22:14:10.054Z 7 140-159 25 25 25 47 47 47 F F F
29702024-02-28T22:14:10.054Z 8 160-179 24 24 24 46 46 46 T F T <---
29712024-02-28T22:14:10.054Z 9 180-199 20 20 20 37 37 37 F F F
29722024-02-28T22:14:10.054Z 10 200-219 20 20 20 36 36 36 T F T <---
29732024-02-28T22:14:10.054Z 11 220-239 22 22 22 39 39 39 F F F
29742024-02-28T22:14:10.054Z 12 240-259 24 24 24 43 43 43 T F T <---
29752024-02-28T22:14:10.054Z 13 260-279 20 20 20 36 36 36 F F F
29762024-02-28T22:14:10.054Z 14 280-299 18 18 18 32 32 32 F F F
29772024-02-28T22:14:10.054Z 15 300-319 23 23 23 42 42 42 T F T <---
29782024-02-28T22:14:10.054Z 16 320-339 23 23 23 42 42 42 T F T <---
29792024-02-28T22:14:10.054Z 17 340-359 22 22 22 40 40 40 T F T <---
29802024-02-28T22:14:10.054Z 18 360-379 24 24 24 46 46 46 F F F
29812024-02-28T22:14:10.055Z 19 380-399 21 21 21 38 38 38 T F T <---
29822024-02-28T22:14:10.055Z 20 400-419 24 24 24 43 43 43 T F T <---
29832024-02-28T22:14:10.055Z 21 420-439 20 20 20 36 36 36 T F T <---
29842024-02-28T22:14:10.055Z 22 440-459 25 25 25 47 47 47 T F T <---
29852024-02-28T22:14:10.055Z 23 460-479 20 20 20 36 36 36 F F F
29862024-02-28T22:14:10.055Z 24 480-499 18 18 18 32 32 32 T F T <---
29872024-02-28T22:14:10.055Z 25 500-519 24 24 24 45 45 45 F F F
29882024-02-28T22:14:10.055Z 26 520-539 20 20 20 37 37 37 T F T <---
29892024-02-28T22:14:10.055Z 27 540-559 24 24 24 45 45 45 T T T
29902024-02-28T22:14:10.055Z 28 560-579 25 25 25 47 47 47 T T T
29912024-02-28T22:14:10.055Z 29 580-599 22 22 22 39 39 39 T T T
29922024-02-28T22:14:10.055ZMax gen: 25, Max flush: 47
29932024-02-28T22:14:10.055ZError: Difference in extent metadata found!
29942024-02-28T22:14:10.055ZA Difference in extent metadata is expected here
29952024-02-28T22:14:10.055ZOn loop 13
29962024-02-28T22:14:10.055Z
29972024-02-28T22:14:10.055ZVerifying data now
29982024-02-28T22:14:16.992ZNo change after verify
29992024-02-28T22:14:16.992ZLoop: 13 Downstairs dump after verify (and repair):
30002024-02-28T22:14:17.014ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30012024-02-28T22:14:17.014Z 0 000-019 26 26 26 48 48 48 F F F
30022024-02-28T22:14:17.014Z 1 020-039 17 17 17 30 30 30 F F F
30032024-02-28T22:14:17.014Z 2 040-059 26 26 26 48 48 48 F F F
30042024-02-28T22:14:17.014Z 3 060-079 26 26 26 48 48 48 F F F
30052024-02-28T22:14:17.014Z 4 080-099 20 20 20 37 37 37 F F F
30062024-02-28T22:14:17.014Z 5 100-119 14 14 14 24 24 24 F F F
30072024-02-28T22:14:17.014Z 6 120-139 22 22 22 41 41 41 F F F
30082024-02-28T22:14:17.014Z 7 140-159 25 25 25 47 47 47 F F F
30092024-02-28T22:14:17.014Z 8 160-179 26 26 26 48 48 48 F F F
30102024-02-28T22:14:17.014Z 9 180-199 20 20 20 37 37 37 F F F
30112024-02-28T22:14:17.015Z 10 200-219 26 26 26 48 48 48 F F F
30122024-02-28T22:14:17.015Z 11 220-239 22 22 22 39 39 39 F F F
30132024-02-28T22:14:17.015Z 12 240-259 26 26 26 48 48 48 F F F
30142024-02-28T22:14:17.015Z 13 260-279 20 20 20 36 36 36 F F F
30152024-02-28T22:14:17.024Z 14 280-299 18 18 18 32 32 32 F F F
30162024-02-28T22:14:17.025Z 15 300-319 26 26 26 48 48 48 F F F
30172024-02-28T22:14:17.025Z 16 320-339 26 26 26 48 48 48 F F F
30182024-02-28T22:14:17.025Z 17 340-359 26 26 26 48 48 48 F F F
30192024-02-28T22:14:17.025Z 18 360-379 24 24 24 46 46 46 F F F
30202024-02-28T22:14:17.025Z 19 380-399 26 26 26 48 48 48 F F F
30212024-02-28T22:14:17.025Z 20 400-419 26 26 26 48 48 48 F F F
30222024-02-28T22:14:17.025Z 21 420-439 26 26 26 48 48 48 F F F
30232024-02-28T22:14:17.025Z 22 440-459 26 26 26 48 48 48 F F F
30242024-02-28T22:14:17.025Z 23 460-479 20 20 20 36 36 36 F F F
30252024-02-28T22:14:17.025Z 24 480-499 26 26 26 48 48 48 F F F
30262024-02-28T22:14:17.025Z 25 500-519 24 24 24 45 45 45 F F F
30272024-02-28T22:14:17.025Z 26 520-539 26 26 26 48 48 48 F F F
30282024-02-28T22:14:17.025Z 27 540-559 26 26 26 48 48 48 F F F
30292024-02-28T22:14:17.026Z 28 560-579 26 26 26 48 48 48 F F F
30302024-02-28T22:14:17.026Z 29 580-599 26 26 26 48 48 48 F F F
30312024-02-28T22:14:17.026ZMax gen: 26, Max flush: 48
30322024-02-28T22:14:17.026Z
30332024-02-28T22:14:17.026ZBegin loop 14
30342024-02-28T22:14:17.026ZDownstairs to restart: 2
30352024-02-28T22:14:17.140Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.928773443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
30362024-02-28T22:14:17.140Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
30372024-02-28T22:14:17.140Z,"time":"2024-02-28T22:14:15.929138013Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
30382024-02-28T22:14:17.140Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929180581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
30392024-02-28T22:14:17.140Z{"msg":"Crucible 641b74f2-73bb-4f19-bcdd-0401748fe022 has session id: bec84374-7ad8-4fca-bffb-f5abe1e0651d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929202671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30402024-02-28T22:14:17.140Z{"msg":"Upstairs opts: Upstairs UUID: 641b74f2-73bb-4f19-bcdd-0401748fe022, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.92922995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30412024-02-28T22:14:17.140Z{"msg":"Crucible stats registered with UUID: 641b74f2-73bb-4f19-bcdd-0401748fe022","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929262729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30422024-02-28T22:14:17.140Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929280478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
30432024-02-28T22:14:17.140Z{"msg":"641b74f2-73bb-4f19-bcdd-0401748fe022 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929317348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30442024-02-28T22:14:17.140Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929333547Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30452024-02-28T22:14:17.140Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929364646Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30462024-02-28T22:14:17.140Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929396225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30472024-02-28T22:14:17.140Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929441274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30482024-02-28T22:14:17.140Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929488163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30492024-02-28T22:14:17.140Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929536871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30502024-02-28T22:14:17.140Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.92959124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30512024-02-28T22:14:17.140Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929640139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30522024-02-28T22:14:17.141Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929673918Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30532024-02-28T22:14:17.141Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929714697Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30542024-02-28T22:14:17.141Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.929872122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30552024-02-28T22:14:17.141Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.92993688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30562024-02-28T22:14:17.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930502455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30572024-02-28T22:14:17.141Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930538834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30582024-02-28T22:14:17.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930554954Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30592024-02-28T22:14:17.141Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930569173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30602024-02-28T22:14:17.141Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930592472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30612024-02-28T22:14:17.141Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.930606882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30622024-02-28T22:14:17.141Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931120218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30632024-02-28T22:14:17.141Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931171866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30642024-02-28T22:14:17.141Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931224245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30652024-02-28T22:14:17.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931716931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30662024-02-28T22:14:17.141Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.93175647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30672024-02-28T22:14:17.141Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931788629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30682024-02-28T22:14:17.141Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931816229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30692024-02-28T22:14:17.141Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931863027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30702024-02-28T22:14:17.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931883017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30712024-02-28T22:14:17.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931903276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30722024-02-28T22:14:17.142Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931917446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30732024-02-28T22:14:17.142Z{"msg":"[0]R flush_numbers[0..12]: [48, 30, 48, 48, 37, 24, 41, 47, 48, 37, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931938135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30742024-02-28T22:14:17.142Z{"msg":"[0]R generation[0..12]: [26, 17, 26, 26, 20, 14, 22, 25, 26, 20, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931952555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30752024-02-28T22:14:17.142Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931967584Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30762024-02-28T22:14:17.142Z{"msg":"[1]R flush_numbers[0..12]: [48, 30, 48, 48, 37, 24, 41, 47, 48, 37, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.931988494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30772024-02-28T22:14:17.142Z{"msg":"[1]R generation[0..12]: [26, 17, 26, 26, 20, 14, 22, 25, 26, 20, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932002883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30782024-02-28T22:14:17.142Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932024293Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30792024-02-28T22:14:17.142Z{"msg":"[2]R flush_numbers[0..12]: [48, 30, 48, 48, 37, 24, 41, 47, 48, 37, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932038793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30802024-02-28T22:14:17.142Z{"msg":"[2]R generation[0..12]: [26, 17, 26, 26, 20, 14, 22, 25, 26, 20, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932059022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30812024-02-28T22:14:17.142Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
30822024-02-28T22:14:17.142Z,"time":"2024-02-28T22:14:15.932073592Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30832024-02-28T22:14:17.142Z{"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932105571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30842024-02-28T22:14:17.142Z{"msg":"Generation requested: 28 >= found:27","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.93212815Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30852024-02-28T22:14:17.142Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.93214204Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30862024-02-28T22:14:17.142Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932163289Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"":"downstairs","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30872024-02-28T22:14:17.142Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932185458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30882024-02-28T22:14:17.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932212908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"0","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30892024-02-28T22:14:17.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932234667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"1","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30902024-02-28T22:14:17.142Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932252857Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"client":"2","":"downstairs","session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30912024-02-28T22:14:17.142Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932275716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30922024-02-28T22:14:17.143Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932288616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30932024-02-28T22:14:17.143Z{"msg":"641b74f2-73bb-4f19-bcdd-0401748fe022 is now active with session: bec84374-7ad8-4fca-bffb-f5abe1e0651d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932312675Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30942024-02-28T22:14:17.143Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932326725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247,"session_id":"bec84374-7ad8-4fca-bffb-f5abe1e0651d"}
30952024-02-28T22:14:17.143Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:15.932346374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
30962024-02-28T22:14:17.143ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30972024-02-28T22:14:17.143ZLoading write count information from file "/tmp/test_repair_verify.data"
30982024-02-28T22:14:17.143ZRun Repair workload
30992024-02-28T22:14:17.143Z01/30 Write block 92 len 4096 data: 3
31002024-02-28T22:14:17.143Z02/30 Read block 499 len 24576
31012024-02-28T22:14:17.143Z03/30 Flush
31022024-02-28T22:14:17.143Z04/30 Write block 12 len 28672 data: 3 4 4 4 5 5 5
31032024-02-28T22:14:17.143Z05/30 Read block 225 len 36864
31042024-02-28T22:14:17.143Z06/30 Read block 266 len 20480
31052024-02-28T22:14:17.143Z07/30 Flush
31062024-02-28T22:14:17.143Z08/30 Write block 525 len 28672 data: 4 4 4 4 4 4 3
31072024-02-28T22:14:17.143Z09/30 Read block 164 len 16384
31082024-02-28T22:14:17.143Z10/30 Read block 81 len 40960
31092024-02-28T22:14:17.143Z11/30 Flush
31102024-02-28T22:14:17.143Z12/30 Flush
31112024-02-28T22:14:17.143Z13/30 Write block 187 len 8192 data: 2 2
31122024-02-28T22:14:17.143Z14/30 Read block 341 len 40960
31132024-02-28T22:14:17.143Z15/30 Write block 583 len 16384 data: 2 2 3 3
31142024-02-28T22:14:17.144Z16/30 Read block 345 len 4096
31152024-02-28T22:14:17.144Z17/30 Write block 292 len 24576 data: 3 3 2 3 3 3
31162024-02-28T22:14:17.144Z18/30 Read block 134 len 12288
31172024-02-28T22:14:17.144Z19/30 Read block 103 len 4096
31182024-02-28T22:14:17.144Z20/30 Write block 438 len 36864 data: 4 2 2 2 2 2 3 4 5
31192024-02-28T22:14:17.144Z21/30 Read block 514 len 4096
31202024-02-28T22:14:17.144Z22/30 Write block 538 len 28672 data: 2 2 2 3 4 4 5
31212024-02-28T22:14:17.144Z23/30 Read block 519 len 32768
31222024-02-28T22:14:17.144Z24/30 Write block 437 len 24576 data: 4 5 3 3 3 3
31232024-02-28T22:14:17.144Z25/30 Read block 198 len 8192
31242024-02-28T22:14:17.144Z26/30 Write block 26 len 32768 data: 2 3 3 3 4 4 3 3
31252024-02-28T22:14:17.144Z27/30 Write block 167 len 4096 data: 7
31262024-02-28T22:14:17.144Z28/30 Write block 121 len 36864 data: 6 6 6 6 6 5 6 6 5
31272024-02-28T22:14:17.144Z29/30 Read block 559 len 4096
31282024-02-28T22:14:17.144Z30/30 Write block 181 len 16384 data: 3 3 3 3
31292024-02-28T22:14:17.144Z----------------------------------------------------------------
31302024-02-28T22:14:17.144Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:27
31312024-02-28T22:14:17.144ZStates: Active Active Active
31322024-02-28T22:14:17.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31332024-02-28T22:14:17.144Z 4 Acked 1003 Write 7 Done Done Done false
31342024-02-28T22:14:17.144Z 5 Acked 1004 Read 9 Done Done Sent false
31352024-02-28T22:14:17.151Z 6 Acked 1005 Read 5 Done Done Sent false
31362024-02-28T22:14:17.151Z 7 Acked 1006 Flush 0 Done Done Sent false
31372024-02-28T22:14:17.151Z 8 Acked 1007 Write 7 Done Done Sent false
31382024-02-28T22:14:17.151Z 9 Acked 1008 Read 4 Done Done Sent false
31392024-02-28T22:14:17.151Z 10 Acked 1009 Read 10 Done Done Sent false
31402024-02-28T22:14:17.151Z 11 Acked 1010 Flush 0 Done Done Sent false
31412024-02-28T22:14:17.151Z 12 Acked 1011 Flush 0 Done Done Sent false
31422024-02-28T22:14:17.151Z 13 Acked 1012 Write 2 Done Done Sent false
31432024-02-28T22:14:17.151Z 14 Acked 1013 Read 10 Done Done Sent false
31442024-02-28T22:14:17.151Z 15 Acked 1014 Write 4 Done Done Sent false
31452024-02-28T22:14:17.151Z 16 Acked 1015 Read 1 Done Done Sent false
31462024-02-28T22:14:17.151Z 17 Acked 1016 Write 6 Done Done Sent false
31472024-02-28T22:14:17.151Z 18 Acked 1017 Read 3 Done Done Sent false
31482024-02-28T22:14:17.151Z 19 Acked 1018 Read 1 Done Done Sent false
31492024-02-28T22:14:17.151Z 20 Acked 1019 Write 9 Done Done Sent false
31502024-02-28T22:14:17.151Z 21 Acked 1020 Read 1 Done Done Sent false
31512024-02-28T22:14:17.151Z 22 Acked 1021 Write 7 Done Done Sent false
31522024-02-28T22:14:17.151Z 23 Acked 1022 Read 8 Done Done Sent false
31532024-02-28T22:14:17.151Z 24 Acked 1023 Write 6 Done Done Sent false
31542024-02-28T22:14:17.151Z 25 Acked 1024 Read 2 Done Done Sent false
31552024-02-28T22:14:17.152Z 26 Acked 1025 Write 8 Done Done Sent false
31562024-02-28T22:14:17.152Z 27 Acked 1026 Write 1 Done Done Sent false
31572024-02-28T22:14:17.152Z 28 Acked 1027 Write 9 Done Done Sent false
31582024-02-28T22:14:17.152Z 29 Acked 1028 Read 1 Done Done Sent false
31592024-02-28T22:14:17.152Z 30 Acked 1029 Write 4 Sent Sent Sent false
31602024-02-28T22:14:17.152Z STATES DS:0 DS:1 DS:2 TOTAL
31612024-02-28T22:14:17.152Z New 0 0 0 0
31622024-02-28T22:14:17.152Z Sent 1 1 26 28
31632024-02-28T22:14:17.152Z Done 26 26 1 53
31642024-02-28T22:14:17.152Z Skipped 0 0 0 0
31652024-02-28T22:14:17.152Z Error 0 0 0 0
31662024-02-28T22:14:17.152ZLast Flush: 1011 1011 1002
31672024-02-28T22:14:17.152ZDownstairs last five completed: 1002 1001 1000
31682024-02-28T22:14:17.152ZUpstairs last five completed: 30 29 28 27 26
31692024-02-28T22:14:17.152Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:15.985815591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1247}
31702024-02-28T22:14:17.152ZWrote out file "/tmp/test_repair_verify.data"
31712024-02-28T22:14:17.152Z
31722024-02-28T22:14:17.153ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31732024-02-28T22:14:17.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31742024-02-28T22:14:17.153Z 0 000-019 28 28 26 50 50 48 F F T <---
31752024-02-28T22:14:17.153Z 1 020-039 17 17 17 30 30 30 T T F <---
31762024-02-28T22:14:17.153Z 2 040-059 26 26 26 48 48 48 F F F
31772024-02-28T22:14:17.153Z 3 060-079 26 26 26 48 48 48 F F F
31782024-02-28T22:14:17.153Z 4 080-099 28 28 28 49 49 49 F F F
31792024-02-28T22:14:17.153Z 5 100-119 14 14 14 24 24 24 F F F
31802024-02-28T22:14:17.153Z 6 120-139 22 22 22 41 41 41 T T F <---
31812024-02-28T22:14:17.153Z 7 140-159 25 25 25 47 47 47 F F F
31822024-02-28T22:14:17.153Z 8 160-179 26 26 26 48 48 48 T T F <---
31832024-02-28T22:14:17.153Z 9 180-199 20 20 20 37 37 37 T T F <---
31842024-02-28T22:14:17.154Z 10 200-219 26 26 26 48 48 48 F F F
31852024-02-28T22:14:17.154Z 11 220-239 22 22 22 39 39 39 F F F
31862024-02-28T22:14:17.154Z 12 240-259 26 26 26 48 48 48 F F F
31872024-02-28T22:14:17.154Z 13 260-279 20 20 20 36 36 36 F F F
31882024-02-28T22:14:17.154Z 14 280-299 18 18 18 32 32 32 T T F <---
31892024-02-28T22:14:17.154Z 15 300-319 26 26 26 48 48 48 F F F
31902024-02-28T22:14:17.154Z 16 320-339 26 26 26 48 48 48 F F F
31912024-02-28T22:14:17.154Z 17 340-359 26 26 26 48 48 48 F F F
31922024-02-28T22:14:17.154Z 18 360-379 24 24 24 46 46 46 F F F
31932024-02-28T22:14:17.154Z 19 380-399 26 26 26 48 48 48 F F F
31942024-02-28T22:14:17.154Z 20 400-419 26 26 26 48 48 48 F F F
31952024-02-28T22:14:17.154Z 21 420-439 26 26 26 48 48 48 T T F <---
31962024-02-28T22:14:17.154Z 22 440-459 26 26 26 48 48 48 T T F <---
31972024-02-28T22:14:17.154Z 23 460-479 20 20 20 36 36 36 F F F
31982024-02-28T22:14:17.154Z 24 480-499 26 26 26 48 48 48 F F F
31992024-02-28T22:14:17.154Z 25 500-519 24 24 24 45 45 45 F F F
32002024-02-28T22:14:17.154Z 26 520-539 28 28 26 51 51 48 T T F <---
32012024-02-28T22:14:17.154Z 27 540-559 26 26 26 48 48 48 T T F <---
32022024-02-28T22:14:17.155Z 28 560-579 26 26 26 48 48 48 F F F
32032024-02-28T22:14:17.155Z 29 580-599 26 26 26 48 48 48 T T F <---
32042024-02-28T22:14:17.155ZMax gen: 28, Max flush: 51
32052024-02-28T22:14:17.155ZError: Difference in extent metadata found!
32062024-02-28T22:14:17.155ZA Difference in extent metadata is expected here
32072024-02-28T22:14:17.155ZOn loop 14
32082024-02-28T22:14:17.155Z
32092024-02-28T22:14:17.155ZVerifying data now
32102024-02-28T22:14:22.380ZNo change after verify
32112024-02-28T22:14:22.380ZLoop: 14 Downstairs dump after verify (and repair):
32122024-02-28T22:14:22.390ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32132024-02-28T22:14:22.391Z 0 000-019 28 28 28 50 50 50 F F F
32142024-02-28T22:14:22.395Z 1 020-039 29 29 29 52 52 52 F F F
32152024-02-28T22:14:22.395Z 2 040-059 26 26 26 48 48 48 F F F
32162024-02-28T22:14:22.395Z 3 060-079 26 26 26 48 48 48 F F F
32172024-02-28T22:14:22.395Z 4 080-099 28 28 28 49 49 49 F F F
32182024-02-28T22:14:22.395Z 5 100-119 14 14 14 24 24 24 F F F
32192024-02-28T22:14:22.395Z 6 120-139 29 29 29 52 52 52 F F F
32202024-02-28T22:14:22.395Z 7 140-159 25 25 25 47 47 47 F F F
32212024-02-28T22:14:22.395Z 8 160-179 29 29 29 52 52 52 F F F
32222024-02-28T22:14:22.395Z 9 180-199 29 29 29 52 52 52 F F F
32232024-02-28T22:14:22.395Z 10 200-219 26 26 26 48 48 48 F F F
32242024-02-28T22:14:22.396Z 11 220-239 22 22 22 39 39 39 F F F
32252024-02-28T22:14:22.396Z 12 240-259 26 26 26 48 48 48 F F F
32262024-02-28T22:14:22.396Z 13 260-279 20 20 20 36 36 36 F F F
32272024-02-28T22:14:22.396Z 14 280-299 29 29 29 52 52 52 F F F
32282024-02-28T22:14:22.396Z 15 300-319 26 26 26 48 48 48 F F F
32292024-02-28T22:14:22.396Z 16 320-339 26 26 26 48 48 48 F F F
32302024-02-28T22:14:22.396Z 17 340-359 26 26 26 48 48 48 F F F
32312024-02-28T22:14:22.396Z 18 360-379 24 24 24 46 46 46 F F F
32322024-02-28T22:14:22.396Z 19 380-399 26 26 26 48 48 48 F F F
32332024-02-28T22:14:22.396Z 20 400-419 26 26 26 48 48 48 F F F
32342024-02-28T22:14:22.396Z 21 420-439 29 29 29 52 52 52 F F F
32352024-02-28T22:14:22.396Z 22 440-459 29 29 29 52 52 52 F F F
32362024-02-28T22:14:22.396Z 23 460-479 20 20 20 36 36 36 F F F
32372024-02-28T22:14:22.396Z 24 480-499 26 26 26 48 48 48 F F F
32382024-02-28T22:14:22.396Z 25 500-519 24 24 24 45 45 45 F F F
32392024-02-28T22:14:22.396Z 26 520-539 29 29 29 52 52 52 F F F
32402024-02-28T22:14:22.396Z 27 540-559 29 29 29 52 52 52 F F F
32412024-02-28T22:14:22.396Z 28 560-579 26 26 26 48 48 48 F F F
32422024-02-28T22:14:22.396Z 29 580-599 29 29 29 52 52 52 F F F
32432024-02-28T22:14:22.396ZMax gen: 29, Max flush: 52
32442024-02-28T22:14:22.397Z
32452024-02-28T22:14:22.397ZBegin loop 15
32462024-02-28T22:14:22.397ZDownstairs to restart: 2
32472024-02-28T22:14:22.413Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312026889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
32482024-02-28T22:14:22.413ZCrucible runtime is spawned
32492024-02-28T22:14:22.425Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.31235074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
32502024-02-28T22:14:22.425Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312408929Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
32512024-02-28T22:14:22.425Z{"msg":"Crucible e58ffa19-06d5-4b0d-8fa4-69743341c8f5 has session id: 58124b6b-6f73-4db5-beeb-31bd22e54929","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312440478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32522024-02-28T22:14:22.425Z{"msg":"Upstairs opts: Upstairs UUID: e58ffa19-06d5-4b0d-8fa4-69743341c8f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312480687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32532024-02-28T22:14:22.425Z{"msg":"Crucible stats registered with UUID: e58ffa19-06d5-4b0d-8fa4-69743341c8f5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312498936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32542024-02-28T22:14:22.425Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312530515Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
32552024-02-28T22:14:22.425Z{"msg":"e58ffa19-06d5-4b0d-8fa4-69743341c8f5 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312561194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32562024-02-28T22:14:22.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312588064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32572024-02-28T22:14:22.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312612913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32582024-02-28T22:14:22.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312668982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32592024-02-28T22:14:22.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312708781Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32602024-02-28T22:14:22.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.31273538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32612024-02-28T22:14:22.426Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312758979Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32622024-02-28T22:14:22.426Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312849186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32632024-02-28T22:14:22.426Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.312949484Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32642024-02-28T22:14:22.426Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313025272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32652024-02-28T22:14:22.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313065351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32662024-02-28T22:14:22.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.3130942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32672024-02-28T22:14:22.426Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313110139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32682024-02-28T22:14:22.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313680904Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32692024-02-28T22:14:22.426Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313722822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32702024-02-28T22:14:22.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.31381048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32712024-02-28T22:14:22.426Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313908447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32722024-02-28T22:14:22.426Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313948476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32732024-02-28T22:14:22.426Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.313976086Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32742024-02-28T22:14:22.426Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314318196Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32752024-02-28T22:14:22.426Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314356595Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32762024-02-28T22:14:22.427Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314399514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32772024-02-28T22:14:22.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314823822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32782024-02-28T22:14:22.427Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314861251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32792024-02-28T22:14:22.427Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314877651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32802024-02-28T22:14:22.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.314977098Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32812024-02-28T22:14:22.427Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315014917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32822024-02-28T22:14:22.427Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315031777Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32832024-02-28T22:14:22.427Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315046026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32842024-02-28T22:14:22.427Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315067965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32852024-02-28T22:14:22.427Z{"msg":"[0]R flush_numbers[0..12]: [50, 52, 48, 48, 49, 24, 52, 47, 52, 52, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315082365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32862024-02-28T22:14:22.427Z{"msg":"[0]R generation[0..12]: [28, 29, 26, 26, 28, 14, 29, 25, 29, 29, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315104535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32872024-02-28T22:14:22.427Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315119264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32882024-02-28T22:14:22.427Z{"msg":"[1]R flush_numbers[0..12]: [50, 52, 48, 48, 49, 24, 52, 47, 52, 52, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315140423Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32892024-02-28T22:14:22.427Z{"msg":"[1]R generation[0..12]: [28, 29, 26, 26, 28, 14, 29, 25, 29, 29, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315163323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32902024-02-28T22:14:22.427Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315180222Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32912024-02-28T22:14:22.427Z{"msg":"[2]R flush_numbers[0..12]: [50, 52, 48, 48, 49, 24, 52, 47, 52, 52, 48, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315206782Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32922024-02-28T22:14:22.427ZWait for a query_work_queue command to finish before sending IO
32932024-02-28T22:14:22.427Z{"msg":"[2]R generation[0..12]: [28, 29, 26, 26, 28, 14, 29, 25, 29, 29, 26, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315227011Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32942024-02-28T22:14:22.427Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315243551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32952024-02-28T22:14:22.427Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.31525718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32962024-02-28T22:14:22.427Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315283959Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32972024-02-28T22:14:22.428Z{"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315304759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32982024-02-28T22:14:22.428Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315318538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"":"downstairs","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
32992024-02-28T22:14:22.428Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315332068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33002024-02-28T22:14:22.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315354608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"0","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33012024-02-28T22:14:22.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315378637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"1","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33022024-02-28T22:14:22.428Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315393087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"client":"2","":"downstairs","session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33032024-02-28T22:14:22.428Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315413306Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33042024-02-28T22:14:22.428Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315425995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33052024-02-28T22:14:22.428Z{"msg":"e58ffa19-06d5-4b0d-8fa4-69743341c8f5 is now active with session: 58124b6b-6f73-4db5-beeb-31bd22e54929","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315445805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33062024-02-28T22:14:22.428Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315458855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255,"session_id":"58124b6b-6f73-4db5-beeb-31bd22e54929"}
33072024-02-28T22:14:22.428Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:21.315478744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
33082024-02-28T22:14:22.428ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33092024-02-28T22:14:22.428ZLoading write count information from file "/tmp/test_repair_verify.data"
33102024-02-28T22:14:22.428ZRun Repair workload
33112024-02-28T22:14:22.428Z01/30 Write block 253 len 28672 data: 5 4 2 2 3 3 3
33122024-02-28T22:14:22.428Z02/30 Write block 482 len 16384 data: 3 3 4 5
33132024-02-28T22:14:22.428Z03/30 Read block 35 len 20480
33142024-02-28T22:14:22.428Z04/30 Write block 311 len 16384 data: 4 4 5 5
33152024-02-28T22:14:22.428Z05/30 Write block 420 len 20480 data: 4 4 4 4 4
33162024-02-28T22:14:22.445Z06/30 Write block 11 len 8192 data: 3 4
33172024-02-28T22:14:22.445Z07/30 Write block 499 len 20480 data: 4 5 5 5 6
33182024-02-28T22:14:22.445Z08/30 Read block 576 len 12288
33192024-02-28T22:14:22.445Z09/30 Write block 295 len 24576 data: 4 4 4 2 2 2
33202024-02-28T22:14:22.445Z10/30 Read block 312 len 32768
33212024-02-28T22:14:22.445Z11/30 Flush
33222024-02-28T22:14:22.455Z12/30 Write block 550 len 4096 data: 5
33232024-02-28T22:14:22.459Z13/30 Read block 518 len 12288
33242024-02-28T22:14:22.459Z14/30 Write block 77 len 8192 data: 2 2
33252024-02-28T22:14:22.459Z15/30 Write block 541 len 36864 data: 4 5 5 6 6 7 7 4 5
33262024-02-28T22:14:22.459Z16/30 Flush
33272024-02-28T22:14:22.465Z17/30 Write block 127 len 36864 data: 7 7 6 5 6 6 6 4 4
33282024-02-28T22:14:22.465Z18/30 Read block 193 len 32768
33292024-02-28T22:14:22.465Z19/30 Flush
33302024-02-28T22:14:22.466Z20/30 Write block 118 len 12288 data: 3 3 4
33312024-02-28T22:14:22.466Z21/30 Flush
33322024-02-28T22:14:22.504Z22/30 Flush
33332024-02-28T22:14:22.512Z23/30 Write block 167 len 24576 data: 8 6 6 5 4 4
33342024-02-28T22:14:22.512Z24/30 Read block 527 len 36864
33352024-02-28T22:14:22.512Z25/30 Read block 550 len 24576
33362024-02-28T22:14:22.512Z26/30 Read block 242 len 24576
33372024-02-28T22:14:22.512Z27/30 Write block 525 len 12288 data: 5 5 5
33382024-02-28T22:14:22.512Z28/30 Write block 82 len 36864 data: 2 2 2 2 2 3 3 3 3
33392024-02-28T22:14:22.512Z29/30 Write block 213 len 12288 data: 2 2 2
33402024-02-28T22:14:22.512Z30/30 Write block 325 len 24576 data: 4 3 4 4 4 4
33412024-02-28T22:14:22.539Z----------------------------------------------------------------
33422024-02-28T22:14:22.539Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33432024-02-28T22:14:22.539ZStates: Active Active Active
33442024-02-28T22:14:22.539ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33452024-02-28T22:14:22.539Z 1 Acked 1000 Write 7 Done Done Done false
33462024-02-28T22:14:22.539Z 2 Acked 1001 Write 4 Done Done Done false
33472024-02-28T22:14:22.539Z 3 Acked 1002 Read 5 Done Done Done false
33482024-02-28T22:14:22.539Z 4 Acked 1003 Write 4 Done Done Done false
33492024-02-28T22:14:22.539Z 5 Acked 1004 Write 5 Done Done Sent false
33502024-02-28T22:14:22.539Z 6 Acked 1005 Write 2 Done Done Sent false
33512024-02-28T22:14:22.539Z 7 Acked 1006 Write 5 Done Done Sent false
33522024-02-28T22:14:22.539Z 8 Acked 1007 Read 3 Done Done Sent false
33532024-02-28T22:14:22.539Z 9 Acked 1008 Write 6 Done Done Sent false
33542024-02-28T22:14:22.539Z 10 Acked 1009 Read 8 Done Done Sent false
33552024-02-28T22:14:22.540Z 11 Acked 1010 Flush 0 Done Done Sent false
33562024-02-28T22:14:22.540Z 12 Acked 1011 Write 1 Done Done Sent false
33572024-02-28T22:14:22.540Z 13 Acked 1012 Read 3 Done Done Sent false
33582024-02-28T22:14:22.540Z 14 Acked 1013 Write 2 Done Done Sent false
33592024-02-28T22:14:22.540Z 15 Acked 1014 Write 9 Done Done Sent false
33602024-02-28T22:14:22.540Z 16 Acked 1015 Flush 0 Done Done Sent false
33612024-02-28T22:14:22.540Z 17 Acked 1016 Write 9 Done Done Sent false
33622024-02-28T22:14:22.540Z 18 Acked 1017 Read 8 Done Done Sent false
33632024-02-28T22:14:22.540Z 19 Acked 1018 Flush 0 Done Done Sent false
33642024-02-28T22:14:22.540Z 20 Acked 1019 Write 3 Done Done Sent false
33652024-02-28T22:14:22.540Z 21 Acked 1020 Flush 0 Done Done Sent false
33662024-02-28T22:14:22.540Z 22 Acked 1021 Flush 0 Done Done Sent false
33672024-02-28T22:14:22.540Z 23 Acked 1022 Write 6 Done Done Sent false
33682024-02-28T22:14:22.540Z 24 Acked 1023 Read 9 Done Done Sent false
33692024-02-28T22:14:22.540Z 25 Acked 1024 Read 6 Done Done Sent false
33702024-02-28T22:14:22.540Z 26 Acked 1025 Read 6 Done Done Sent false
33712024-02-28T22:14:22.540Z 27 Acked 1026 Write 3 Done Done Sent false
33722024-02-28T22:14:22.540Z 28 Acked 1027 Write 9 Done Done Sent false
33732024-02-28T22:14:22.540Z 29 Acked 1028 Write 3 Done Done Sent false
33742024-02-28T22:14:22.540Z 30 Acked 1029 Write 6 Sent Sent Sent false
33752024-02-28T22:14:22.541Z STATES DS:0 DS:1 DS:2 TOTAL
33762024-02-28T22:14:22.541Z New 0 0 0 0
33772024-02-28T22:14:22.541Z Sent 1 1 26 28
33782024-02-28T22:14:22.541Z Done 29 29 4 62
33792024-02-28T22:14:22.541Z Skipped 0 0 0 0
33802024-02-28T22:14:22.541Z Error 0 0 0 0
33812024-02-28T22:14:22.541ZLast Flush: 1021 1021 0
33822024-02-28T22:14:22.541ZDownstairs last five completed:
33832024-02-28T22:14:22.541ZUpstairs last five completed: 30 29 28 27 26
33842024-02-28T22:14:22.541Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:21.411481339Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1255}
33852024-02-28T22:14:22.541ZWrote out file "/tmp/test_repair_verify.data"
33862024-02-28T22:14:22.541Z
33872024-02-28T22:14:22.541ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33882024-02-28T22:14:22.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33892024-02-28T22:14:22.541Z 0 000-019 30 30 28 53 53 50 F F F <---
33902024-02-28T22:14:22.541Z 1 020-039 29 29 29 52 52 52 F F F
33912024-02-28T22:14:22.541Z 2 040-059 26 26 26 48 48 48 F F F
33922024-02-28T22:14:22.541Z 3 060-079 30 30 26 54 54 48 F F F <---
33932024-02-28T22:14:22.541Z 4 080-099 28 28 28 49 49 49 T T F <---
33942024-02-28T22:14:22.542Z 5 100-119 30 30 14 56 56 24 F F F <---
33952024-02-28T22:14:22.542Z 6 120-139 30 30 29 56 56 52 F F F <---
33962024-02-28T22:14:22.542Z 7 140-159 25 25 25 47 47 47 F F F
33972024-02-28T22:14:22.542Z 8 160-179 29 29 29 52 52 52 T T F <---
33982024-02-28T22:14:22.542Z 9 180-199 29 29 29 52 52 52 F F F
33992024-02-28T22:14:22.542Z 10 200-219 26 26 26 48 48 48 T T F <---
34002024-02-28T22:14:22.542Z 11 220-239 22 22 22 39 39 39 F F F
34012024-02-28T22:14:22.542Z 12 240-259 30 30 26 53 53 48 F F T <---
34022024-02-28T22:14:22.542Z 13 260-279 20 20 20 36 36 36 F F F
34032024-02-28T22:14:22.542Z 14 280-299 30 30 29 53 53 52 F F F <---
34042024-02-28T22:14:22.542Z 15 300-319 30 30 26 53 53 48 F F T <---
34052024-02-28T22:14:22.542Z 16 320-339 26 26 26 48 48 48 T T F <---
34062024-02-28T22:14:22.542Z 17 340-359 26 26 26 48 48 48 F F F
34072024-02-28T22:14:22.542Z 18 360-379 24 24 24 46 46 46 F F F
34082024-02-28T22:14:22.542Z 19 380-399 26 26 26 48 48 48 F F F
34092024-02-28T22:14:22.542Z 20 400-419 26 26 26 48 48 48 F F F
34102024-02-28T22:14:22.542Z 21 420-439 30 30 29 53 53 52 F F F <---
34112024-02-28T22:14:22.542Z 22 440-459 29 29 29 52 52 52 F F F
34122024-02-28T22:14:22.542Z 23 460-479 20 20 20 36 36 36 F F F
34132024-02-28T22:14:22.542Z 24 480-499 30 30 26 53 53 48 F F T <---
34142024-02-28T22:14:22.543Z 25 500-519 30 30 24 53 53 45 F F F <---
34152024-02-28T22:14:22.543Z 26 520-539 29 29 29 52 52 52 T T F <---
34162024-02-28T22:14:22.543Z 27 540-559 30 30 29 54 54 52 F F F <---
34172024-02-28T22:14:22.543Z 28 560-579 26 26 26 48 48 48 F F F
34182024-02-28T22:14:22.543Z 29 580-599 29 29 29 52 52 52 F F F
34192024-02-28T22:14:22.543ZMax gen: 30, Max flush: 56
34202024-02-28T22:14:22.543ZError: Difference in extent metadata found!
34212024-02-28T22:14:22.548ZA Difference in extent metadata is expected here
34222024-02-28T22:14:22.548ZOn loop 15
34232024-02-28T22:14:22.548Z
34242024-02-28T22:14:22.548ZVerifying data now
34252024-02-28T22:14:29.518ZNo change after verify
34262024-02-28T22:14:29.518ZLoop: 15 Downstairs dump after verify (and repair):
34272024-02-28T22:14:29.519ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34282024-02-28T22:14:29.519Z 0 000-019 30 30 30 53 53 53 F F F
34292024-02-28T22:14:29.519Z 1 020-039 29 29 29 52 52 52 F F F
34302024-02-28T22:14:29.519Z 2 040-059 26 26 26 48 48 48 F F F
34312024-02-28T22:14:29.519Z 3 060-079 30 30 30 54 54 54 F F F
34322024-02-28T22:14:29.519Z 4 080-099 31 31 31 57 57 57 F F F
34332024-02-28T22:14:29.519Z 5 100-119 30 30 30 56 56 56 F F F
34342024-02-28T22:14:29.519Z 6 120-139 30 30 30 56 56 56 F F F
34352024-02-28T22:14:29.519Z 7 140-159 25 25 25 47 47 47 F F F
34362024-02-28T22:14:29.519Z 8 160-179 31 31 31 57 57 57 F F F
34372024-02-28T22:14:29.519Z 9 180-199 29 29 29 52 52 52 F F F
34382024-02-28T22:14:29.519Z 10 200-219 31 31 31 57 57 57 F F F
34392024-02-28T22:14:29.519Z 11 220-239 22 22 22 39 39 39 F F F
34402024-02-28T22:14:29.519Z 12 240-259 30 30 30 53 53 53 F F F
34412024-02-28T22:14:29.519Z 13 260-279 20 20 20 36 36 36 F F F
34422024-02-28T22:14:29.519Z 14 280-299 30 30 30 53 53 53 F F F
34432024-02-28T22:14:29.519Z 15 300-319 30 30 30 53 53 53 F F F
34442024-02-28T22:14:29.519Z 16 320-339 31 31 31 57 57 57 F F F
34452024-02-28T22:14:29.519Z 17 340-359 26 26 26 48 48 48 F F F
34462024-02-28T22:14:29.519Z 18 360-379 24 24 24 46 46 46 F F F
34472024-02-28T22:14:29.519Z 19 380-399 26 26 26 48 48 48 F F F
34482024-02-28T22:14:29.519Z 20 400-419 26 26 26 48 48 48 F F F
34492024-02-28T22:14:29.766Z 21 420-439 30 30 30 53 53 53 F F F
34502024-02-28T22:14:29.766Z 22 440-459 29 29 29 52 52 52 F F F
34512024-02-28T22:14:29.766Z 23 460-479 20 20 20 36 36 36 F F F
34522024-02-28T22:14:29.767Z 24 480-499 30 30 30 53 53 53 F F F
34532024-02-28T22:14:29.767Z 25 500-519 30 30 30 53 53 53 F F F
34542024-02-28T22:14:29.767Z 26 520-539 31 31 31 57 57 57 F F F
34552024-02-28T22:14:29.767Z 27 540-559 30 30 30 54 54 54 F F F
34562024-02-28T22:14:29.767Z 28 560-579 26 26 26 48 48 48 F F F
34572024-02-28T22:14:29.767Z 29 580-599 29 29 29 52 52 52 F F F
34582024-02-28T22:14:29.767ZMax gen: 31, Max flush: 57
34592024-02-28T22:14:29.767Z
34602024-02-28T22:14:29.767ZBegin loop 16
34612024-02-28T22:14:29.767ZDownstairs to restart: 0
34622024-02-28T22:14:29.767Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06033748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
34632024-02-28T22:14:29.767Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060619412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
34642024-02-28T22:14:29.767ZCrucible runtime is spawned
34652024-02-28T22:14:29.767Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060675401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
34662024-02-28T22:14:29.767Z{"msg":"Crucible 6948d018-c6bb-49b9-805a-2decbf63a65f has session id: 06b8c06b-5383-4c3d-8a03-620d8e5ef89a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06070121Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34672024-02-28T22:14:29.767Z{"msg":"Upstairs opts: Upstairs UUID: 6948d018-c6bb-49b9-805a-2decbf63a65f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060746559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34682024-02-28T22:14:29.767Z{"msg":"Crucible stats registered with UUID: 6948d018-c6bb-49b9-805a-2decbf63a65f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060782218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34692024-02-28T22:14:29.767Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060823327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
34702024-02-28T22:14:29.767Z{"msg":"6948d018-c6bb-49b9-805a-2decbf63a65f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060897695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34712024-02-28T22:14:29.767Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.060978213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34722024-02-28T22:14:29.768Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061029001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34732024-02-28T22:14:29.768Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061115839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34742024-02-28T22:14:29.768Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061163347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34752024-02-28T22:14:29.768Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061201226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34762024-02-28T22:14:29.768Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061257135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34772024-02-28T22:14:29.768Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061318183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34782024-02-28T22:14:29.768Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061361572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34792024-02-28T22:14:29.768Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061395801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34802024-02-28T22:14:29.768Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06143127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34812024-02-28T22:14:29.768Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061461659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34822024-02-28T22:14:29.768Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.061477579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34832024-02-28T22:14:29.768Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062005614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34842024-02-28T22:14:29.768Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062043313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34852024-02-28T22:14:29.768Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062059493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34862024-02-28T22:14:29.768Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062074402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34872024-02-28T22:14:29.768Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062089192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34882024-02-28T22:14:29.768Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06214145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34892024-02-28T22:14:29.768Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062585408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34902024-02-28T22:14:29.768Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062643667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34912024-02-28T22:14:29.768Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.062835381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34922024-02-28T22:14:29.768Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063091934Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34932024-02-28T22:14:29.768Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063184142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34942024-02-28T22:14:29.769Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063214851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34952024-02-28T22:14:29.769Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06324122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34962024-02-28T22:14:29.769Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063273039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34972024-02-28T22:14:29.769Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063296169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34982024-02-28T22:14:29.769Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063316808Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
34992024-02-28T22:14:29.769Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063337688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35002024-02-28T22:14:29.769Z{"msg":"[0]R flush_numbers[0..12]: [53, 52, 48, 54, 57, 56, 56, 47, 57, 52, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063352757Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35012024-02-28T22:14:29.769Z{"msg":"[0]R generation[0..12]: [30, 29, 26, 30, 31, 30, 30, 25, 31, 29, 31, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063373576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35022024-02-28T22:14:29.769Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063388996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35032024-02-28T22:14:29.769Z{"msg":"[1]R flush_numbers[0..12]: [53, 52, 48, 54, 57, 56, 56, 47, 57, 52, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063409506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35042024-02-28T22:14:29.769Z{"msg":"[1]R generation[0..12]: [30, 29, 26, 30, 31, 30, 30, 25, 31, 29, 31, 22]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
35052024-02-28T22:14:29.769Zcrucible","level":30,"time":"2024-02-28T22:14:28.063434265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35062024-02-28T22:14:29.769Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063455334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35072024-02-28T22:14:29.769Z{"msg":"[2]R flush_numbers[0..12]: [53, 52, 48, 54, 57, 56, 56, 47, 57, 52, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063475884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35082024-02-28T22:14:29.769Z{"msg":"[2]R generation[0..12]: [30, 29, 26, 30, 31, 30, 30, 25, 31, 29, 31, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063496333Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35092024-02-28T22:14:29.769Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063517132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35102024-02-28T22:14:29.769Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063537742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35112024-02-28T22:14:29.769Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063551711Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35122024-02-28T22:14:29.769Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063571611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35132024-02-28T22:14:29.769Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06359183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"":"downstairs","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35142024-02-28T22:14:29.769Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.06361186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35152024-02-28T22:14:29.769Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063631099Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"0","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35162024-02-28T22:14:29.769Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063651469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"1","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35172024-02-28T22:14:29.770Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063671998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"client":"2","":"downstairs","session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35182024-02-28T22:14:29.770Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063692488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35192024-02-28T22:14:29.770Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063711887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35202024-02-28T22:14:29.770Z{"msg":"6948d018-c6bb-49b9-805a-2decbf63a65f is now active with session: 06b8c06b-5383-4c3d-8a03-620d8e5ef89a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063731827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35212024-02-28T22:14:29.770Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063750706Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263,"session_id":"06b8c06b-5383-4c3d-8a03-620d8e5ef89a"}
35222024-02-28T22:14:29.770Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:28.063769856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
35232024-02-28T22:14:29.770ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35242024-02-28T22:14:29.770ZLoading write count information from file "/tmp/test_repair_verify.data"
35252024-02-28T22:14:29.770ZRun Repair workload
35262024-02-28T22:14:29.770ZError: Difference in extent metadata found!
35272024-02-28T22:14:29.770Z01/30 Write block 380 len 12288 data: 3 3 4
35282024-02-28T22:14:29.770Z02/30 Write block 154 len 40960 data: 3 4 4 4 4 4 2 3 3 3
35292024-02-28T22:14:29.770Z03/30 Write block 565 len 20480 data: 5 6 7 7 6
35302024-02-28T22:14:29.770Z04/30 Write block 529 len 32768 data: 5 5 4 3 4 4 2 3
35312024-02-28T22:14:29.770Z05/30 Write block 70 len 8192 data: 5 5
35322024-02-28T22:14:29.770Z06/30 Read block 481 len 8192
35332024-02-28T22:14:29.770Z07/30 Read block 480 len 24576
35342024-02-28T22:14:29.770Z08/30 Write block 485 len 4096 data: 6
35352024-02-28T22:14:29.770Z09/30 Write block 148 len 20480 data: 6 6 4 3 2
35362024-02-28T22:14:29.770Z10/30 Read block 294 len 4096
35372024-02-28T22:14:29.770Z11/30 Write block 365 len 36864 data: 3 2 2 4 4 4 5 6 5
35382024-02-28T22:14:29.770Z12/30 Read block 535 len 4096
35392024-02-28T22:14:29.771Z13/30 Write block 414 len 12288 data: 3 3 3
35402024-02-28T22:14:29.771Z14/30 Write block 504 len 28672 data: 6 5 5 5 5 5 5
35412024-02-28T22:14:29.771Z15/30 Read block 259 len 36864
35422024-02-28T22:14:29.771Z16/30 Flush
35432024-02-28T22:14:29.771Z17/30 Write block 307 len 24576 data: 6 5 4 3 5 5
35442024-02-28T22:14:29.771Z18/30 Flush
35452024-02-28T22:14:29.771Z19/30 Write block 456 len 40960 data: 4 3 3 3 3 3 3 4 4 4
35462024-02-28T22:14:29.771Z20/30 Read block 37 len 4096
35472024-02-28T22:14:29.771Z21/30 Flush
35482024-02-28T22:14:29.771Z22/30 Write block 255 len 28672 data: 3 3 4 4 4 3 3
35492024-02-28T22:14:29.827Z23/30 Flush
35502024-02-28T22:14:29.827Z24/30 Write block 185 len 36864 data: 2 2 3 3 2 2 2 3 2
35512024-02-28T22:14:29.828Z25/30 Flush
35522024-02-28T22:14:29.828Z26/30 Write block 330 len 4096 data: 5
35532024-02-28T22:14:29.828Z27/30 Read block 176 len 12288
35542024-02-28T22:14:29.828Z28/30 Read block 417 len 4096
35552024-02-28T22:14:29.828Z29/30 Write block 291 len 8192 data: 3 4
35562024-02-28T22:14:29.828Z30/30 Read block 453 len 16384
35572024-02-28T22:14:29.828Z----------------------------------------------------------------
35582024-02-28T22:14:29.828Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35592024-02-28T22:14:29.828ZStates: Active Active Active
35602024-02-28T22:14:29.828ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35612024-02-28T22:14:29.828Z 1 Acked 1000 Write 3 Done Done Done false
35622024-02-28T22:14:29.828Z 2 Acked 1001 Write 10 Sent Done Done false
35632024-02-28T22:14:29.828Z 3 Acked 1002 Write 5 Sent Done Done false
35642024-02-28T22:14:29.828Z 4 Acked 1003 Write 8 Sent Done Done false
35652024-02-28T22:14:29.829Z 5 Acked 1004 Write 2 Sent Done Done false
35662024-02-28T22:14:29.829Z 6 Acked 1005 Read 2 Sent Done Done false
35672024-02-28T22:14:29.829Z 7 Acked 1006 Read 6 Sent Done Done false
35682024-02-28T22:14:29.829Z 8 Acked 1007 Write 1 Sent Done Done false
35692024-02-28T22:14:29.829Z 9 Acked 1008 Write 5 Sent Done Done false
35702024-02-28T22:14:29.829Z 10 Acked 1009 Read 1 Sent Done Done false
35712024-02-28T22:14:29.829Z 11 Acked 1010 Write 9 Sent Done Done false
35722024-02-28T22:14:29.829Z 12 Acked 1011 Read 1 Sent Done Done false
35732024-02-28T22:14:29.829Z 13 Acked 1012 Write 3 Sent Done Done false
35742024-02-28T22:14:29.829Z 14 Acked 1013 Write 7 Sent Done Done false
35752024-02-28T22:14:29.829Z 15 Acked 1014 Read 9 Sent Done Done false
35762024-02-28T22:14:29.829Z 16 Acked 1015 Flush 0 Sent Done Done false
35772024-02-28T22:14:29.829Z 17 Acked 1016 Write 6 Sent Done Done false
35782024-02-28T22:14:29.829Z 18 Acked 1017 Flush 0 Sent Done Done false
35792024-02-28T22:14:29.830Z 19 Acked 1018 Write 10 Sent Done Done false
35802024-02-28T22:14:29.830Z 20 Acked 1019 Read 1 Sent Done Done false
35812024-02-28T22:14:29.830Z 21 Acked 1020 Flush 0 Sent Done Done false
35822024-02-28T22:14:29.830Z 22 Acked 1021 Write 7 Sent Done Done false
35832024-02-28T22:14:29.830Z 23 Acked 1022 Flush 0 Sent Done Done false
35842024-02-28T22:14:29.830Z 24 Acked 1023 Write 9 Sent Done Done false
35852024-02-28T22:14:29.830Z 25 Acked 1024 Flush 0 Sent Done Done false
35862024-02-28T22:14:29.830Z 26 Acked 1025 Write 1 Sent Done Done false
35872024-02-28T22:14:29.830Z 27 Acked 1026 Read 3 Sent Done Done false
35882024-02-28T22:14:29.831Z 28 Acked 1027 Read 1 Sent Done Done false
35892024-02-28T22:14:29.831Z 29 Acked 1028 Write 2 Sent Done Done false
35902024-02-28T22:14:29.831Z 30 Acked 1029 Read 4 Sent Sent Done false
35912024-02-28T22:14:29.831Z STATES DS:0 DS:1 DS:2 TOTAL
35922024-02-28T22:14:29.831Z New 0 0 0 0
35932024-02-28T22:14:29.831Z Sent 29 1 0 30
35942024-02-28T22:14:29.831Z Done 1 29 30 60
35952024-02-28T22:14:29.831Z Skipped 0 0 0 0
35962024-02-28T22:14:29.831Z Error 0 0 0 0
35972024-02-28T22:14:29.831ZLast Flush: 0 1024 1024
35982024-02-28T22:14:29.831ZDownstairs last five completed:
35992024-02-28T22:14:29.831ZUpstairs last five completed: 30 29 28 27 26
36002024-02-28T22:14:29.831Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:28.160017994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1263}
36012024-02-28T22:14:29.831ZWrote out file "/tmp/test_repair_verify.data"
36022024-02-28T22:14:29.831Z
36032024-02-28T22:14:29.831ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36042024-02-28T22:14:29.831ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36052024-02-28T22:14:29.832Z 0 000-019 30 30 30 53 53 53 F F F
36062024-02-28T22:14:29.832Z 1 020-039 29 29 29 52 52 52 F F F
36072024-02-28T22:14:29.832Z 2 040-059 26 26 26 48 48 48 F F F
36082024-02-28T22:14:29.832Z 3 060-079 30 32 32 54 58 58 F F F <---
36092024-02-28T22:14:29.832Z 4 080-099 31 31 31 57 57 57 F F F
36102024-02-28T22:14:29.832Z 5 100-119 30 30 30 56 56 56 F F F
36112024-02-28T22:14:29.832Z 6 120-139 30 30 30 56 56 56 F F F
36122024-02-28T22:14:29.832Z 7 140-159 25 32 32 47 58 58 F F F <---
36132024-02-28T22:14:29.832Z 8 160-179 31 32 32 57 58 58 F F F <---
36142024-02-28T22:14:29.832Z 9 180-199 29 32 32 52 62 62 F F F <---
36152024-02-28T22:14:29.832Z 10 200-219 31 31 31 57 57 57 F F F
36162024-02-28T22:14:29.832Z 11 220-239 22 22 22 39 39 39 F F F
36172024-02-28T22:14:29.832Z 12 240-259 30 32 32 53 61 61 F F F <---
36182024-02-28T22:14:29.832Z 13 260-279 20 32 32 36 61 61 F F F <---
36192024-02-28T22:14:29.832Z 14 280-299 30 30 30 53 53 53 F T T <---
36202024-02-28T22:14:29.832Z 15 300-319 30 32 32 53 59 59 F F F <---
36212024-02-28T22:14:29.832Z 16 320-339 31 31 31 57 57 57 F T T <---
36222024-02-28T22:14:29.832Z 17 340-359 26 26 26 48 48 48 F F F
36232024-02-28T22:14:29.832Z 18 360-379 24 32 32 46 58 58 F F F <---
36242024-02-28T22:14:29.832Z 19 380-399 26 32 32 48 58 58 T F F <---
36252024-02-28T22:14:29.832Z 20 400-419 26 32 32 48 58 58 F F F <---
36262024-02-28T22:14:29.833Z 21 420-439 30 30 30 53 53 53 F F F
36272024-02-28T22:14:29.833Z 22 440-459 29 32 32 52 60 60 F F F <---
36282024-02-28T22:14:29.833Z 23 460-479 20 32 32 36 60 60 F F F <---
36292024-02-28T22:14:29.833Z 24 480-499 30 32 32 53 58 58 F F F <---
36302024-02-28T22:14:29.833Z 25 500-519 30 32 32 53 58 58 F F F <---
36312024-02-28T22:14:29.833Z 26 520-539 31 32 32 57 58 58 F F F <---
36322024-02-28T22:14:29.833Z 27 540-559 30 30 30 54 54 54 F F F
36332024-02-28T22:14:29.833Z 28 560-579 26 32 32 48 58 58 F F F <---
36342024-02-28T22:14:29.833Z 29 580-599 29 29 29 52 52 52 F F F
36352024-02-28T22:14:29.833ZMax gen: 32, Max flush: 62
36362024-02-28T22:14:29.833ZA Difference in extent metadata is expected here
36372024-02-28T22:14:29.833ZOn loop 16
36382024-02-28T22:14:29.833Z
36392024-02-28T22:14:29.833ZVerifying data now
36402024-02-28T22:14:35.727ZNo change after verify
36412024-02-28T22:14:35.728ZLoop: 16 Downstairs dump after verify (and repair):
36422024-02-28T22:14:35.742ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36432024-02-28T22:14:35.742Z 0 000-019 30 30 30 53 53 53 F F F
36442024-02-28T22:14:35.742Z 1 020-039 29 29 29 52 52 52 F F F
36452024-02-28T22:14:35.742Z 2 040-059 26 26 26 48 48 48 F F F
36462024-02-28T22:14:35.742Z 3 060-079 32 32 32 58 58 58 F F F
36472024-02-28T22:14:35.743Z 4 080-099 31 31 31 57 57 57 F F F
36482024-02-28T22:14:35.743Z 5 100-119 30 30 30 56 56 56 F F F
36492024-02-28T22:14:35.743Z 6 120-139 30 30 30 56 56 56 F F F
36502024-02-28T22:14:35.743Z 7 140-159 32 32 32 58 58 58 F F F
36512024-02-28T22:14:35.743Z 8 160-179 32 32 32 58 58 58 F F F
36522024-02-28T22:14:35.743Z 9 180-199 32 32 32 62 62 62 F F F
36532024-02-28T22:14:35.743Z 10 200-219 31 31 31 57 57 57 F F F
36542024-02-28T22:14:35.743Z 11 220-239 22 22 22 39 39 39 F F F
36552024-02-28T22:14:35.743Z 12 240-259 32 32 32 61 61 61 F F F
36562024-02-28T22:14:35.743Z 13 260-279 32 32 32 61 61 61 F F F
36572024-02-28T22:14:35.743Z 14 280-299 33 33 33 63 63 63 F F F
36582024-02-28T22:14:35.743Z 15 300-319 32 32 32 59 59 59 F F F
36592024-02-28T22:14:35.747Z 16 320-339 33 33 33 63 63 63 F F F
36602024-02-28T22:14:35.748Z 17 340-359 26 26 26 48 48 48 F F F
36612024-02-28T22:14:35.748Z 18 360-379 32 32 32 58 58 58 F F F
36622024-02-28T22:14:35.748Z 19 380-399 32 32 32 58 58 58 F F F
36632024-02-28T22:14:35.748Z 20 400-419 32 32 32 58 58 58 F F F
36642024-02-28T22:14:35.748Z 21 420-439 30 30 30 53 53 53 F F F
36652024-02-28T22:14:35.748Z 22 440-459 32 32 32 60 60 60 F F F
36662024-02-28T22:14:35.748Z 23 460-479 32 32 32 60 60 60 F F F
36672024-02-28T22:14:35.748Z 24 480-499 32 32 32 58 58 58 F F F
36682024-02-28T22:14:35.748Z 25 500-519 32 32 32 58 58 58 F F F
36692024-02-28T22:14:35.748Z 26 520-539 32 32 32 58 58 58 F F F
36702024-02-28T22:14:35.748Z 27 540-559 30 30 30 54 54 54 F F F
36712024-02-28T22:14:35.748Z 28 560-579 32 32 32 58 58 58 F F F
36722024-02-28T22:14:35.748Z 29 580-599 29 29 29 52 52 52 F F F
36732024-02-28T22:14:35.748ZMax gen: 33, Max flush: 63
36742024-02-28T22:14:35.748Z
36752024-02-28T22:14:35.748ZBegin loop 17
36762024-02-28T22:14:35.748ZDownstairs to restart: 0
36772024-02-28T22:14:35.769Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.664820043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
36782024-02-28T22:14:35.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665145014Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
36792024-02-28T22:14:35.769Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665176233Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
36802024-02-28T22:14:35.769Z{"msg":"Crucible 257359a5-a09f-4f53-813b-2a68d89e67bb has session id: 8dbd0376-15be-4c87-b71a-b888afaf449b","v":Crucible runtime is spawned
36812024-02-28T22:14:35.775Z0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665226692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36822024-02-28T22:14:35.775Z{"msg":"Upstairs opts: Upstairs UUID: 257359a5-a09f-4f53-813b-2a68d89e67bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.66529069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36832024-02-28T22:14:35.775Z{"msg":"Crucible stats registered with UUID: 257359a5-a09f-4f53-813b-2a68d89e67bb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665328609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36842024-02-28T22:14:35.775Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665346328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
36852024-02-28T22:14:35.775Z{"msg":"257359a5-a09f-4f53-813b-2a68d89e67bb active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665369828Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36862024-02-28T22:14:35.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665404597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36872024-02-28T22:14:35.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665465745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36882024-02-28T22:14:35.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665511294Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36892024-02-28T22:14:35.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665551873Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36902024-02-28T22:14:35.776Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665578582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36912024-02-28T22:14:35.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665612531Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36922024-02-28T22:14:35.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.66564764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36932024-02-28T22:14:35.776Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665690589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36942024-02-28T22:14:35.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665732488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36952024-02-28T22:14:35.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665891233Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36962024-02-28T22:14:35.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665941522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36972024-02-28T22:14:35.776Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.665971441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36982024-02-28T22:14:35.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.666576284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
36992024-02-28T22:14:35.776Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.666620243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37002024-02-28T22:14:35.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.666654652Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37012024-02-28T22:14:35.776Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.666676212Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37022024-02-28T22:14:35.776Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.666721151Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37032024-02-28T22:14:35.776Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.66674575Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37042024-02-28T22:14:35.776Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667351103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37052024-02-28T22:14:35.776Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667397522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37062024-02-28T22:14:35.776Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667430391Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37072024-02-28T22:14:35.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667867549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37082024-02-28T22:14:35.776Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667917778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37092024-02-28T22:14:35.776Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.667947397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37102024-02-28T22:14:35.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668051074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37112024-02-28T22:14:35.776Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668088203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37122024-02-28T22:14:35.776Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668104542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37132024-02-28T22:14:35.776Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668125872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37142024-02-28T22:14:35.777Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668140401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37152024-02-28T22:14:35.777Z{"msg":"[0]R flush_numbers[0..12]: [53, 52, 48, 58, 57, 56, 56, 58, 58, 62, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668161351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37162024-02-28T22:14:35.777Z{"msg":"[0]R generation[0..12]: [30, 29, 26, 32, 31, 30, 30, 32, 32, 32, 31, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.66817597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37172024-02-28T22:14:35.777Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.66819704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37182024-02-28T22:14:35.777Z{"msg":"[1]R flush_numbers[0..12]: [53, 52, 48, 58, 57, 56, 56, 58, 58, 62, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668211179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37192024-02-28T22:14:35.777Z{"msg":"[1]R generation[0..12]: [30, 29, 26, 32, 31, 30, 30, 32, 32, 32, 31, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668231759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37202024-02-28T22:14:35.777Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
37212024-02-28T22:14:35.777Z2024-02-28T22:14:34.668246049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37222024-02-28T22:14:35.777Z{"msg":"[2]R flush_numbers[0..12]: [53, 52, 48, 58, 57, 56, 56, 58, 58, 62, 57, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668277908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37232024-02-28T22:14:35.777Z{"msg":"[2]R generation[0..12]: [30, 29, 26, 32, 31, 30, 30, 32, 32, 32, 31, 22]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668292637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37242024-02-28T22:14:35.777Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668314407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37252024-02-28T22:14:35.777Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668328206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37262024-02-28T22:14:35.777Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668341906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37272024-02-28T22:14:35.777Z{"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668362255Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37282024-02-28T22:14:35.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668375895Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"":"downstairs","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37292024-02-28T22:14:35.777Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668396064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37302024-02-28T22:14:35.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668408544Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"0","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37312024-02-28T22:14:35.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668429283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"1","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37322024-02-28T22:14:35.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668442833Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"client":"2","":"downstairs","session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37332024-02-28T22:14:35.777Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668456103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37342024-02-28T22:14:35.777Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668476132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37352024-02-28T22:14:35.777Z{"msg":"257359a5-a09f-4f53-813b-2a68d89e67bb is now active with session: 8dbd0376-15be-4c87-b71a-b888afaf449b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668489422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37362024-02-28T22:14:35.777Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668507891Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271,"session_id":"8dbd0376-15be-4c87-b71a-b888afaf449b"}
37372024-02-28T22:14:35.778Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:34.668520591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
37382024-02-28T22:14:35.778ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37392024-02-28T22:14:35.778ZLoading write count information from file "/tmp/test_repair_verify.data"
37402024-02-28T22:14:35.778ZRun Repair workload
37412024-02-28T22:14:35.778Z01/30 Write block 585 len 16384 data: 4 4 2 2
37422024-02-28T22:14:35.778Z02/30 Read block 220 len 28672
37432024-02-28T22:14:35.781Z03/30 Read block 524 len 16384
37442024-02-28T22:14:35.781Z04/30 Write block 228 len 8192 data: 7 7
37452024-02-28T22:14:35.781Z05/30 Read block 392 len 32768
37462024-02-28T22:14:35.781Z06/30 Read block 59 len 36864
37472024-02-28T22:14:35.781Z07/30 Write block 434 len 36864 data: 4 4 4 5 6 4 4 4 4
37482024-02-28T22:14:35.782Z08/30 Read block 71 len 12288
37492024-02-28T22:14:35.782Z09/30 Write block 433 len 28672 data: 4 5 5 5 6 7 5
37502024-02-28T22:14:35.782Z10/30 Write block 91 len 36864 data: 3 4 2 2 2 2 2 2 2
37512024-02-28T22:14:35.782Z11/30 Read block 500 len 28672
37522024-02-28T22:14:35.785Z12/30 Read block 459 len 4096
37532024-02-28T22:14:35.785Z13/30 Read block 51 len 20480
37542024-02-28T22:14:35.785Z14/30 Flush
37552024-02-28T22:14:35.788Z15/30 Flush
37562024-02-28T22:14:35.788Z16/30 Write block 335 len 40960 data: 5 4 4 5 5 5 5 5 4 4
37572024-02-28T22:14:35.788Z17/30 Read block 525 len 12288
37582024-02-28T22:14:35.791Z18/30 Read block 58 len 24576
37592024-02-28T22:14:35.791Z19/30 Read block 550 len 36864
37602024-02-28T22:14:35.791Z20/30 Read block 562 len 8192
37612024-02-28T22:14:35.794Z21/30 Read block 480 len 36864
37622024-02-28T22:14:35.795Z22/30 Flush
37632024-02-28T22:14:35.798Z23/30 Write block 295 len 16384 data: 5 5 5 3
37642024-02-28T22:14:35.798Z24/30 Write block 238 len 32768 data: 3 3 3 3 2 2 4 4
37652024-02-28T22:14:35.798Z25/30 Read block 84 len 40960
37662024-02-28T22:14:35.798Z26/30 Write block 212 len 12288 data: 3 3 3
37672024-02-28T22:14:35.801Z27/30 Write block 187 len 36864 data: 4 4 3 3 3 4 3 2 2
37682024-02-28T22:14:35.801Z28/30 Write block 6 len 28672 data: 4 5 5 4 4 4 5
37692024-02-28T22:14:35.801Z29/30 Read block 243 len 36864
37702024-02-28T22:14:35.805Z30/30 Write block 577 len 24576 data: 3 3 3 3 3 2
37712024-02-28T22:14:35.805Z----------------------------------------------------------------
37722024-02-28T22:14:35.805Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
37732024-02-28T22:14:35.805ZStates: Active Active Active
37742024-02-28T22:14:35.805ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37752024-02-28T22:14:35.805Z 1 Acked 1000 Write 4 Done Done Done false
37762024-02-28T22:14:35.805Z 2 Acked 1001 Read 7 Done Done Done false
37772024-02-28T22:14:35.805Z 3 Acked 1002 Read 4 Sent Done Done false
37782024-02-28T22:14:35.805Z 4 Acked 1003 Write 2 Sent Done Done false
37792024-02-28T22:14:35.805Z 5 Acked 1004 Read 8 Sent Done Done false
37802024-02-28T22:14:35.805Z 6 Acked 1005 Read 9 Sent Done Done false
37812024-02-28T22:14:35.806Z 7 Acked 1006 Write 9 Sent Done Done false
37822024-02-28T22:14:35.806Z 8 Acked 1007 Read 3 Sent Done Done false
37832024-02-28T22:14:35.806Z 9 Acked 1008 Write 7 Sent Done Done false
37842024-02-28T22:14:35.806Z 10 Acked 1009 Write 9 Sent Done Done false
37852024-02-28T22:14:35.806Z 11 Acked 1010 Read 7 Sent Done Done false
37862024-02-28T22:14:35.806Z 12 Acked 1011 Read 1 Sent Done Done false
37872024-02-28T22:14:35.806Z 13 Acked 1012 Read 5 Sent Done Done false
37882024-02-28T22:14:35.806Z 14 Acked 1013 Flush 0 Sent Done Done false
37892024-02-28T22:14:35.806Z 15 Acked 1014 Flush 0 Sent Done Done false
37902024-02-28T22:14:35.806Z 16 Acked 1015 Write 10 Sent Done Done false
37912024-02-28T22:14:35.806Z 17 Acked 1016 Read 3 Sent Done Done false
37922024-02-28T22:14:35.806Z 18 Acked 1017 Read 6 Sent Done Done false
37932024-02-28T22:14:35.806Z 19 Acked 1018 Read 9 Sent Done Done false
37942024-02-28T22:14:35.806Z 20 Acked 1019 Read 2 Sent Done Done false
37952024-02-28T22:14:35.806Z 21 Acked 1020 Read 9 Sent Done Done false
37962024-02-28T22:14:35.806Z 22 Acked 1021 Flush 0 Sent Done Done false
37972024-02-28T22:14:35.806Z 23 Acked 1022 Write 4 Sent Done Done false
37982024-02-28T22:14:35.806Z 24 Acked 1023 Write 8 Sent Done Done false
37992024-02-28T22:14:35.806Z 25 Acked 1024 Read 10 Sent Done Done false
38002024-02-28T22:14:35.806Z 26 Acked 1025 Write 3 Sent Done Done false
38012024-02-28T22:14:35.806Z 27 Acked 1026 Write 9 Sent Done Done false
38022024-02-28T22:14:35.806Z 28 Acked 1027 Write 7 Sent Done Done false
38032024-02-28T22:14:35.806Z 29 Acked 1028 Read 9 Sent Done Done false
38042024-02-28T22:14:35.806Z 30 Acked 1029 Write 6 Sent Sent Sent false
38052024-02-28T22:14:35.807Z STATES DS:0 DS:1 DS:2 TOTAL
38062024-02-28T22:14:35.807Z New 0 0 0 0
38072024-02-28T22:14:35.807Z Sent 28 1 1 30
38082024-02-28T22:14:35.807Z Done 2 29 29 60
38092024-02-28T22:14:35.807Z Skipped 0 0 0 0
38102024-02-28T22:14:35.807Z Error 0 0 0 0
38112024-02-28T22:14:35.807ZLast Flush: 0 1021 1021
38122024-02-28T22:14:35.807ZDownstairs last five completed:
38132024-02-28T22:14:35.807ZUpstairs last five completed: 30 29 28 27 26
38142024-02-28T22:14:35.807Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:34.701243369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1271}
38152024-02-28T22:14:35.807ZWrote out file "/tmp/test_repair_verify.data"
38162024-02-28T22:14:35.810Z
38172024-02-28T22:14:35.813ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38182024-02-28T22:14:35.830ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38192024-02-28T22:14:35.830Z 0 000-019 30 30 30 53 53 53 F T T <---
38202024-02-28T22:14:35.830Z 1 020-039 29 29 29 52 52 52 F F F
38212024-02-28T22:14:35.830Z 2 040-059 26 26 26 48 48 48 F F F
38222024-02-28T22:14:35.830Z 3 060-079 32 32 32 58 58 58 F F F
38232024-02-28T22:14:35.830Z 4 080-099 31 34 34 57 64 64 F F F <---
38242024-02-28T22:14:35.830Z 5 100-119 30 30 30 56 56 56 F F F
38252024-02-28T22:14:35.830Z 6 120-139 30 30 30 56 56 56 F F F
38262024-02-28T22:14:35.830Z 7 140-159 32 32 32 58 58 58 F F F
38272024-02-28T22:14:35.830Z 8 160-179 32 32 32 58 58 58 F F F
38282024-02-28T22:14:35.830Z 9 180-199 32 32 32 62 62 62 F T T <---
38292024-02-28T22:14:35.830Z 10 200-219 31 31 31 57 57 57 F T T <---
38302024-02-28T22:14:35.830Z 11 220-239 22 34 34 39 64 64 F T T <---
38312024-02-28T22:14:35.830Z 12 240-259 32 32 32 61 61 61 F T T <---
38322024-02-28T22:14:35.830Z 13 260-279 32 32 32 61 61 61 F F F
38332024-02-28T22:14:35.831Z 14 280-299 33 33 33 63 63 63 F T T <---
38342024-02-28T22:14:35.831Z 15 300-319 32 32 32 59 59 59 F F F
38352024-02-28T22:14:35.831Z 16 320-339 33 34 34 63 66 66 F F F <---
38362024-02-28T22:14:35.831Z 17 340-359 26 34 34 48 66 66 F F F <---
38372024-02-28T22:14:35.831Z 18 360-379 32 32 32 58 58 58 F F F
38382024-02-28T22:14:35.834Z 19 380-399 32 32 32 58 58 58 F F F
38392024-02-28T22:14:35.834Z 20 400-419 32 32 32 58 58 58 F F F
38402024-02-28T22:14:35.834Z 21 420-439 30 34 34 53 64 64 F F F <---
38412024-02-28T22:14:35.834Z 22 440-459 32 34 34 60 64 64 F F F <---
38422024-02-28T22:14:35.834Z 23 460-479 32 32 32 60 60 60 F F F
38432024-02-28T22:14:35.834Z 24 480-499 32 32 32 58 58 58 F F F
38442024-02-28T22:14:35.834Z 25 500-519 32 32 32 58 58 58 F F F
38452024-02-28T22:14:35.835Z 26 520-539 32 32 32 58 58 58 F F F
38462024-02-28T22:14:35.835Z 27 540-559 30 30 30 54 54 54 F F F
38472024-02-28T22:14:35.835Z 28 560-579 32 32 32 58 58 58 F T T <---
38482024-02-28T22:14:35.835Z 29 580-599 29 34 34 52 64 64 T T T <---
38492024-02-28T22:14:35.835ZMax gen: 34, Max flush: 66
38502024-02-28T22:14:35.835ZError: Difference in extent metadata found!
38512024-02-28T22:14:35.835ZA Difference in extent metadata is expected here
38522024-02-28T22:14:35.835ZOn loop 17
38532024-02-28T22:14:35.835Z
38542024-02-28T22:14:35.838ZVerifying data now
38552024-02-28T22:14:42.153ZNo change after verify
38562024-02-28T22:14:42.154ZLoop: 17 Downstairs dump after verify (and repair):
38572024-02-28T22:14:42.168ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38582024-02-28T22:14:42.168Z 0 000-019 35 35 35 67 67 67 F F F
38592024-02-28T22:14:42.168Z 1 020-039 29 29 29 52 52 52 F F F
38602024-02-28T22:14:42.168Z 2 040-059 26 26 26 48 48 48 F F F
38612024-02-28T22:14:42.168Z 3 060-079 32 32 32 58 58 58 F F F
38622024-02-28T22:14:42.168Z 4 080-099 34 34 34 64 64 64 F F F
38632024-02-28T22:14:42.168Z 5 100-119 30 30 30 56 56 56 F F F
38642024-02-28T22:14:42.168Z 6 120-139 30 30 30 56 56 56 F F F
38652024-02-28T22:14:42.168Z 7 140-159 32 32 32 58 58 58 F F F
38662024-02-28T22:14:42.169Z 8 160-179 32 32 32 58 58 58 F F F
38672024-02-28T22:14:42.169Z 9 180-199 35 35 35 67 67 67 F F F
38682024-02-28T22:14:42.169Z 10 200-219 35 35 35 67 67 67 F F F
38692024-02-28T22:14:42.169Z 11 220-239 35 35 35 67 67 67 F F F
38702024-02-28T22:14:42.169Z 12 240-259 35 35 35 67 67 67 F F F
38712024-02-28T22:14:42.169Z 13 260-279 32 32 32 61 61 61 F F F
38722024-02-28T22:14:42.169Z 14 280-299 35 35 35 67 67 67 F F F
38732024-02-28T22:14:42.172Z 15 300-319 32 32 32 59 59 59 F F F
38742024-02-28T22:14:42.172Z 16 320-339 34 34 34 66 66 66 F F F
38752024-02-28T22:14:42.172Z 17 340-359 34 34 34 66 66 66 F F F
38762024-02-28T22:14:42.172Z 18 360-379 32 32 32 58 58 58 F F F
38772024-02-28T22:14:42.173Z 19 380-399 32 32 32 58 58 58 F F F
38782024-02-28T22:14:42.173Z 20 400-419 32 32 32 58 58 58 F F F
38792024-02-28T22:14:42.173Z 21 420-439 34 34 34 64 64 64 F F F
38802024-02-28T22:14:42.173Z 22 440-459 34 34 34 64 64 64 F F F
38812024-02-28T22:14:42.173Z 23 460-479 32 32 32 60 60 60 F F F
38822024-02-28T22:14:42.173Z 24 480-499 32 32 32 58 58 58 F F F
38832024-02-28T22:14:42.173Z 25 500-519 32 32 32 58 58 58 F F F
38842024-02-28T22:14:42.173Z 26 520-539 32 32 32 58 58 58 F F F
38852024-02-28T22:14:42.173Z 27 540-559 30 30 30 54 54 54 F F F
38862024-02-28T22:14:42.173Z 28 560-579 35 35 35 67 67 67 F F F
38872024-02-28T22:14:42.173Z 29 580-599 35 35 35 67 67 67 F F F
38882024-02-28T22:14:42.173ZMax gen: 35, Max flush: 67
38892024-02-28T22:14:42.173Z
38902024-02-28T22:14:42.173ZBegin loop 18
38912024-02-28T22:14:42.173ZDownstairs to restart: 2
38922024-02-28T22:14:42.194Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.091837555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
38932024-02-28T22:14:42.194Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092085589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
38942024-02-28T22:14:42.194Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092109668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
38952024-02-28T22:14:42.200Z{"msg":"Crucible e8b20971-64b9-4744-a44e-454a26ed3e47 has session id: a9b5dbf1-598e-4e55-a785-95c2d7ccb666","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092144107Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
38962024-02-28T22:14:42.200ZCrucible runtime is spawned
38972024-02-28T22:14:42.200Z{"msg":"Upstairs opts: Upstairs UUID: e8b20971-64b9-4744-a44e-454a26ed3e47, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092209575Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
38982024-02-28T22:14:42.200Z{"msg":"Crucible stats registered with UUID: e8b20971-64b9-4744-a44e-454a26ed3e47","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092232105Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
38992024-02-28T22:14:42.200Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092274814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
39002024-02-28T22:14:42.200Z{"msg":"e8b20971-64b9-4744-a44e-454a26ed3e47 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092435749Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39012024-02-28T22:14:42.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092467758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39022024-02-28T22:14:42.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092548786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39032024-02-28T22:14:42.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092591075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39042024-02-28T22:14:42.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092618454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39052024-02-28T22:14:42.200Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092717791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39062024-02-28T22:14:42.200Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.09277682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39072024-02-28T22:14:42.200Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092813299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39082024-02-28T22:14:42.200Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092872887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39092024-02-28T22:14:42.200Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092901396Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39102024-02-28T22:14:42.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.092968414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39112024-02-28T22:14:42.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093015783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39122024-02-28T22:14:42.201Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093053262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39132024-02-28T22:14:42.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093693924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39142024-02-28T22:14:42.201Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093730483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39152024-02-28T22:14:42.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093747003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39162024-02-28T22:14:42.201Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093800261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39172024-02-28T22:14:42.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093881149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39182024-02-28T22:14:42.201Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.093935838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39192024-02-28T22:14:42.201Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094294518Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39202024-02-28T22:14:42.201Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094347026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39212024-02-28T22:14:42.201Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094447934Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39222024-02-28T22:14:42.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094853732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39232024-02-28T22:14:42.201Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094901961Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39242024-02-28T22:14:42.201Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.09494772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39252024-02-28T22:14:42.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.094975369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39262024-02-28T22:14:42.201Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095016428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39272024-02-28T22:14:42.201Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095061397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39282024-02-28T22:14:42.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095081946Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39292024-02-28T22:14:42.201Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095097496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39302024-02-28T22:14:42.201Z{"msg":"[0]R flush_numbers[0..12]: [67, 52, 48, 58, 64, 56, 56, 58, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095120405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39312024-02-28T22:14:42.201Z{"msg":"[0]R generation[0..12]: [35, 29, 26, 32, 34, 30, 30, 32, 32, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095139844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39322024-02-28T22:14:42.201Z{"msg":"Wait for a query_work_queue command to finish before sending IO
39332024-02-28T22:14:42.202Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095178883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39342024-02-28T22:14:42.202Z{"msg":"[1]R flush_numbers[0..12]: [67, 52, 48, 58, 64, 56, 56, 58, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095215142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39352024-02-28T22:14:42.202Z{"msg":"[1]R generation[0..12]: [35, 29, 26, 32, 34, 30, 30, 32, 32, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095239882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39362024-02-28T22:14:42.202Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095262191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39372024-02-28T22:14:42.202Z{"msg":"[2]R flush_numbers[0..12]: [67, 52, 48, 58, 64, 56, 56, 58, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095281171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39382024-02-28T22:14:42.202Z{"msg":"[2]R generation[0..12]: [35, 29, 26, 32, 34, 30, 30, 32, 32, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.09531693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39392024-02-28T22:14:42.202Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095355059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39402024-02-28T22:14:42.202Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095390258Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39412024-02-28T22:14:42.202Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095419127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39422024-02-28T22:14:42.202Z{"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095454736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39432024-02-28T22:14:42.202Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095492225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"":"downstairs","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39442024-02-28T22:14:42.202Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095524574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39452024-02-28T22:14:42.202Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095539093Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"0","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39462024-02-28T22:14:42.202Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095560523Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"1","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39472024-02-28T22:14:42.202Z{"msg":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39482024-02-28T22:14:42.202Z"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095597962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"client":"2","":"downstairs","session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39492024-02-28T22:14:42.202Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095633621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39502024-02-28T22:14:42.202Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.09566701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39512024-02-28T22:14:42.202Z{"msg":"e8b20971-64b9-4744-a44e-454a26ed3e47 is now active with session: a9b5dbf1-598e-4e55-a785-95c2d7ccb666","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095699739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39522024-02-28T22:14:42.202Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095730008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279,"session_id":"a9b5dbf1-598e-4e55-a785-95c2d7ccb666"}
39532024-02-28T22:14:42.202Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:41.095760147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
39542024-02-28T22:14:42.202ZLoading write count information from file "/tmp/test_repair_verify.data"
39552024-02-28T22:14:42.202ZRun Repair workload
39562024-02-28T22:14:42.206Z01/30 Write block 323 len 36864 data: 3 4 5 4 5 5 5 6 4
39572024-02-28T22:14:42.206Z02/30 Write block 157 len 4096 data: 5
39582024-02-28T22:14:42.206Z03/30 Read block 364 len 16384
39592024-02-28T22:14:42.206Z04/30 Read block 374 len 24576
39602024-02-28T22:14:42.206Z05/30 Write block 134 len 28672 data: 5 5 4 4 4 4 4
39612024-02-28T22:14:42.206Z06/30 Write block 276 len 36864 data: 2 2 2 2 2 2 3 3 3
39622024-02-28T22:14:42.206Z07/30 Read block 332 len 40960
39632024-02-28T22:14:42.206Z08/30 Write block 145 len 36864 data: 5 6 6 7 7 5 4 3 2
39642024-02-28T22:14:42.210Z09/30 Write block 567 len 28672 data: 8 8 7 5 4 4 3
39652024-02-28T22:14:42.210Z10/30 Read block 59 len 16384
39662024-02-28T22:14:42.210Z11/30 Read block 254 len 12288
39672024-02-28T22:14:42.210Z12/30 Write block 432 len 20480 data: 5 5 6 6 6
39682024-02-28T22:14:42.210Z13/30 Write block 141 len 40960 data: 4 4 5 5 6 7 7 8 8 6
39692024-02-28T22:14:42.215Z14/30 Read block 19 len 32768
39702024-02-28T22:14:42.215Z15/30 Write block 89 len 32768 data: 4 4 4 5 3 3 3 3
39712024-02-28T22:14:42.218Z16/30 Write block 390 len 36864 data: 5 5 4 4 4 5 5 4 4
39722024-02-28T22:14:42.218Z17/30 Write block 381 len 24576 data: 4 5 4 5 5 5
39732024-02-28T22:14:42.218Z18/30 Read block 75 len 32768
39742024-02-28T22:14:42.218Z19/30 Flush
39752024-02-28T22:14:42.243Z20/30 Flush
39762024-02-28T22:14:42.246Z21/30 Flush
39772024-02-28T22:14:42.246Z22/30 Write block 382 len 20480 data: 6 5 6 6 6
39782024-02-28T22:14:42.246Z23/30 Write block 80 len 4096 data: 3
39792024-02-28T22:14:42.246Z24/30 Write block 54 len 20480 data: 2 2 4 3 3
39802024-02-28T22:14:42.246Z25/30 Write block 338 len 28672 data: 6 6 6 6 6 5 5
39812024-02-28T22:14:42.246Z26/30 Flush
39822024-02-28T22:14:42.250Z27/30 Write block 252 len 20480 data: 5 6 5 4 4
39832024-02-28T22:14:42.253Z28/30 Read block 471 len 4096
39842024-02-28T22:14:42.253Z29/30 Read block 42 len 32768
39852024-02-28T22:14:42.253Z30/30 Write block 323 len 32768 data: 4 5 6 5 6 6 6 7
39862024-02-28T22:14:42.253Z----------------------------------------------------------------
39872024-02-28T22:14:42.253Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39882024-02-28T22:14:42.253ZStates: Active Active Active
39892024-02-28T22:14:42.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39902024-02-28T22:14:42.253Z 1 Acked 1000 Write 9 Done Done Done false
39912024-02-28T22:14:42.254Z 2 Acked 1001 Write 1 Done Done Done false
39922024-02-28T22:14:42.254Z 3 Acked 1002 Read 4 Done Done Done false
39932024-02-28T22:14:42.254Z 4 Acked 1003 Read 6 Done Done Sent false
39942024-02-28T22:14:42.254Z 5 Acked 1004 Write 7 Done Done Sent false
39952024-02-28T22:14:42.254Z 6 Acked 1005 Write 9 Done Done Sent false
39962024-02-28T22:14:42.254Z 7 Acked 1006 Read 10 Done Done Sent false
39972024-02-28T22:14:42.254Z 8 Acked 1007 Write 9 Done Done Sent false
39982024-02-28T22:14:42.254Z 9 Acked 1008 Write 7 Done Done Sent false
39992024-02-28T22:14:42.254Z 10 Acked 1009 Read 4 Done Done Sent false
40002024-02-28T22:14:42.254Z 11 Acked 1010 Read 3 Done Done Sent false
40012024-02-28T22:14:42.254Z 12 Acked 1011 Write 5 Done Done Sent false
40022024-02-28T22:14:42.254Z 13 Acked 1012 Write 10 Done Done Sent false
40032024-02-28T22:14:42.254Z 14 Acked 1013 Read 8 Done Done Sent false
40042024-02-28T22:14:42.254Z 15 Acked 1014 Write 8 Done Done Sent false
40052024-02-28T22:14:42.254Z 16 Acked 1015 Write 9 Done Done Sent false
40062024-02-28T22:14:42.254Z 17 Acked 1016 Write 6 Done Done Sent false
40072024-02-28T22:14:42.254Z 18 Acked 1017 Read 8 Done Done Sent false
40082024-02-28T22:14:42.254Z 19 Acked 1018 Flush 0 Done Done Sent false
40092024-02-28T22:14:42.254Z 20 Acked 1019 Flush 0 Done Done Sent false
40102024-02-28T22:14:42.254Z 21 Acked 1020 Flush 0 Done Done Sent false
40112024-02-28T22:14:42.254Z 22 Acked 1021 Write 5 Done Done Sent false
40122024-02-28T22:14:42.254Z 23 Acked 1022 Write 1 Done Done Sent false
40132024-02-28T22:14:42.254Z 24 Acked 1023 Write 5 Done Done Sent false
40142024-02-28T22:14:42.255Z 25 Acked 1024 Write 7 Done Done Sent false
40152024-02-28T22:14:42.255Z 26 Acked 1025 Flush 0 Done Done Sent false
40162024-02-28T22:14:42.255Z 27 Acked 1026 Write 5 Done Done Sent false
40172024-02-28T22:14:42.255Z 28 Acked 1027 Read 1 Done Done Sent false
40182024-02-28T22:14:42.255Z 29 Acked 1028 Read 8 Done Done Sent false
40192024-02-28T22:14:42.255Z 30 Acked 1029 Write 8 Sent Sent Sent false
40202024-02-28T22:14:42.255Z STATES DS:0 DS:1 DS:2 TOTAL
40212024-02-28T22:14:42.255Z New 0 0 0 0
40222024-02-28T22:14:42.255Z Sent 1 1 27 29
40232024-02-28T22:14:42.255Z Done 29 29 3 61
40242024-02-28T22:14:42.255Z Skipped 0 0 0 0
40252024-02-28T22:14:42.255Z Error 0 0 0 0
40262024-02-28T22:14:42.255ZLast Flush: 1025 1025 0
40272024-02-28T22:14:42.255ZDownstairs last five completed:
40282024-02-28T22:14:42.255ZUpstairs last five completed: 30 29 28 27 26
40292024-02-28T22:14:42.255Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:41.151340476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1279}
40302024-02-28T22:14:42.259ZWrote out file "/tmp/test_repair_verify.data"
40312024-02-28T22:14:42.259Z
40322024-02-28T22:14:42.262ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40332024-02-28T22:14:42.286ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40342024-02-28T22:14:42.286Z 0 000-019 35 35 35 67 67 67 F F F
40352024-02-28T22:14:42.286Z 1 020-039 29 29 29 52 52 52 F F F
40362024-02-28T22:14:42.286Z 2 040-059 36 36 26 71 71 48 F F F <---
40372024-02-28T22:14:42.286Z 3 060-079 32 32 32 58 58 58 F F F
40382024-02-28T22:14:42.286Z 4 080-099 36 36 34 71 71 64 F F F <---
40392024-02-28T22:14:42.286Z 5 100-119 30 30 30 56 56 56 F F F
40402024-02-28T22:14:42.286Z 6 120-139 36 36 30 68 68 56 F F F <---
40412024-02-28T22:14:42.290Z 7 140-159 36 36 32 68 68 58 F F T <---
40422024-02-28T22:14:42.290Z 8 160-179 32 32 32 58 58 58 F F F
40432024-02-28T22:14:42.290Z 9 180-199 35 35 35 67 67 67 F F F
40442024-02-28T22:14:42.290Z 10 200-219 35 35 35 67 67 67 F F F
40452024-02-28T22:14:42.290Z 11 220-239 35 35 35 67 67 67 F F F
40462024-02-28T22:14:42.290Z 12 240-259 35 35 35 67 67 67 T T F <---
40472024-02-28T22:14:42.290Z 13 260-279 36 36 32 68 68 61 F F F <---
40482024-02-28T22:14:42.290Z 14 280-299 36 36 35 68 68 67 F F F <---
40492024-02-28T22:14:42.290Z 15 300-319 32 32 32 59 59 59 F F F
40502024-02-28T22:14:42.290Z 16 320-339 36 36 34 71 71 66 T T T <---
40512024-02-28T22:14:42.290Z 17 340-359 36 36 34 71 71 66 F F F <---
40522024-02-28T22:14:42.290Z 18 360-379 32 32 32 58 58 58 F F F
40532024-02-28T22:14:42.290Z 19 380-399 36 36 32 71 71 58 F F F <---
40542024-02-28T22:14:42.291Z 20 400-419 32 32 32 58 58 58 F F F
40552024-02-28T22:14:42.291Z 21 420-439 36 36 34 68 68 64 F F F <---
40562024-02-28T22:14:42.291Z 22 440-459 34 34 34 64 64 64 F F F
40572024-02-28T22:14:42.291Z 23 460-479 32 32 32 60 60 60 F F F
40582024-02-28T22:14:42.291Z 24 480-499 32 32 32 58 58 58 F F F
40592024-02-28T22:14:42.291Z 25 500-519 32 32 32 58 58 58 F F F
40602024-02-28T22:14:42.291Z 26 520-539 32 32 32 58 58 58 F F F
40612024-02-28T22:14:42.291Z 27 540-559 30 30 30 54 54 54 F F F
40622024-02-28T22:14:42.291Z 28 560-579 36 36 35 68 68 67 F F F <---
40632024-02-28T22:14:42.291Z 29 580-599 35 35 35 67 67 67 F F F
40642024-02-28T22:14:42.291ZMax gen: 36, Max flush: 71
40652024-02-28T22:14:42.291ZError: Difference in extent metadata found!
40662024-02-28T22:14:42.291ZA Difference in extent metadata is expected here
40672024-02-28T22:14:42.291ZOn loop 18
40682024-02-28T22:14:42.291Z
40692024-02-28T22:14:42.291ZVerifying data now
40702024-02-28T22:14:48.131ZNo change after verify
40712024-02-28T22:14:48.131ZLoop: 18 Downstairs dump after verify (and repair):
40722024-02-28T22:14:48.146ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40732024-02-28T22:14:48.146Z 0 000-019 35 35 35 67 67 67 F F F
40742024-02-28T22:14:48.146Z 1 020-039 29 29 29 52 52 52 F F F
40752024-02-28T22:14:48.146Z 2 040-059 36 36 36 71 71 71 F F F
40762024-02-28T22:14:48.147Z 3 060-079 32 32 32 58 58 58 F F F
40772024-02-28T22:14:48.147Z 4 080-099 36 36 36 71 71 71 F F F
40782024-02-28T22:14:48.147Z 5 100-119 30 30 30 56 56 56 F F F
40792024-02-28T22:14:48.147Z 6 120-139 36 36 36 68 68 68 F F F
40802024-02-28T22:14:48.147Z 7 140-159 36 36 36 68 68 68 F F F
40812024-02-28T22:14:48.147Z 8 160-179 32 32 32 58 58 58 F F F
40822024-02-28T22:14:48.147Z 9 180-199 35 35 35 67 67 67 F F F
40832024-02-28T22:14:48.147Z 10 200-219 35 35 35 67 67 67 F F F
40842024-02-28T22:14:48.147Z 11 220-239 35 35 35 67 67 67 F F F
40852024-02-28T22:14:48.150Z 12 240-259 37 37 37 72 72 72 F F F
40862024-02-28T22:14:48.150Z 13 260-279 36 36 36 68 68 68 F F F
40872024-02-28T22:14:48.150Z 14 280-299 36 36 36 68 68 68 F F F
40882024-02-28T22:14:48.150Z 15 300-319 32 32 32 59 59 59 F F F
40892024-02-28T22:14:48.150Z 16 320-339 37 37 37 72 72 72 F F F
40902024-02-28T22:14:48.150Z 17 340-359 36 36 36 71 71 71 F F F
40912024-02-28T22:14:48.150Z 18 360-379 32 32 32 58 58 58 F F F
40922024-02-28T22:14:48.150Z 19 380-399 36 36 36 71 71 71 F F F
40932024-02-28T22:14:48.151Z 20 400-419 32 32 32 58 58 58 F F F
40942024-02-28T22:14:48.151Z 21 420-439 36 36 36 68 68 68 F F F
40952024-02-28T22:14:48.151Z 22 440-459 34 34 34 64 64 64 F F F
40962024-02-28T22:14:48.151Z 23 460-479 32 32 32 60 60 60 F F F
40972024-02-28T22:14:48.151Z 24 480-499 32 32 32 58 58 58 F F F
40982024-02-28T22:14:48.151Z 25 500-519 32 32 32 58 58 58 F F F
40992024-02-28T22:14:48.151Z 26 520-539 32 32 32 58 58 58 F F F
41002024-02-28T22:14:48.151Z 27 540-559 30 30 30 54 54 54 F F F
41012024-02-28T22:14:48.151Z 28 560-579 36 36 36 68 68 68 F F F
41022024-02-28T22:14:48.151Z 29 580-599 35 35 35 67 67 67 F F F
41032024-02-28T22:14:48.151ZMax gen: 37, Max flush: 72
41042024-02-28T22:14:48.151Z
41052024-02-28T22:14:48.151ZBegin loop 19
41062024-02-28T22:14:48.151ZDownstairs to restart: 1
41072024-02-28T22:14:48.168Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06611405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287}
41082024-02-28T22:14:48.168Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066398282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287Crucible runtime is spawned
41092024-02-28T22:14:48.168Z}
41102024-02-28T22:14:48.174Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066445421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287}
41112024-02-28T22:14:48.175Z{"msg":"Crucible 31b9f9fe-112d-4415-85e7-20562401e356 has session id: 4e860090-0f92-4eb8-91b8-c1ca7764efc6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066501889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41122024-02-28T22:14:48.175Z{"msg":"Upstairs opts: Upstairs UUID: 31b9f9fe-112d-4415-85e7-20562401e356, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066543718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41132024-02-28T22:14:48.175Z{"msg":"Crucible stats registered with UUID: 31b9f9fe-112d-4415-85e7-20562401e356","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066585857Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41142024-02-28T22:14:48.175Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066616246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287}
41152024-02-28T22:14:48.175Z{"msg":"31b9f9fe-112d-4415-85e7-20562401e356 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066660385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41162024-02-28T22:14:48.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066687544Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41172024-02-28T22:14:48.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066727603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41182024-02-28T22:14:48.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066760352Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41192024-02-28T22:14:48.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066800301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41202024-02-28T22:14:48.175Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06683232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41212024-02-28T22:14:48.175Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066880229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41222024-02-28T22:14:48.175Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066920067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41232024-02-28T22:14:48.175Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.066971606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41242024-02-28T22:14:48.175Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067016205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41252024-02-28T22:14:48.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067060454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41262024-02-28T22:14:48.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067135542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41272024-02-28T22:14:48.175Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06718819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41282024-02-28T22:14:48.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067618728Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41292024-02-28T22:14:48.175Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067654757Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41302024-02-28T22:14:48.175Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067671647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41312024-02-28T22:14:48.175Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067693586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41322024-02-28T22:14:48.176Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067788364Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41332024-02-28T22:14:48.176Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.067828153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41342024-02-28T22:14:48.176Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06826813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41352024-02-28T22:14:48.176Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.068368688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41362024-02-28T22:14:48.176Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06862494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41372024-02-28T22:14:48.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.068839105Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41382024-02-28T22:14:48.176Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.068879433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41392024-02-28T22:14:48.176Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.068906953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41402024-02-28T22:14:48.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.068965771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41412024-02-28T22:14:48.176Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06900122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41422024-02-28T22:14:48.176Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06901706Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41432024-02-28T22:14:48.176Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069132726Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41442024-02-28T22:14:48.176Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069161166Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41452024-02-28T22:14:48.176Z{"msg":"[0]R flush_numbers[0..12]: [67, 52, 71, 58, 71, 56, 68, 68, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069193505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41462024-02-28T22:14:48.176Z{"msg":"[0]R generation[0..12]: [35, 29, 36, 32, 36, 30, 36, 36, 32, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069214704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41472024-02-28T22:14:48.176Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069230134Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41482024-02-28T22:14:48.176Z{"msg":"[1]R flush_numbers[0..12]: [67, 52, 71, 58, 71, 56, 68, 68, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069250733Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41492024-02-28T22:14:48.176Z{"msg":"[1]R generation[0..12]: [35, 29, 36, 32, 36, 30, 36, 36, 32, 35, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069264813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41502024-02-28T22:14:48.176Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069286362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41512024-02-28T22:14:48.176Z{"msg":"[2]R flush_numbers[0..12]: [67, 52, 71, 58, 71, 56, 68, 68, 58, 67, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069301002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41522024-02-28T22:14:48.176Z{"msg":"[2]R generation[0..12]: [35, 29, 36, 32, 36, 30, 36, 36, 32, 35, 35, 35]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41532024-02-28T22:14:48.176Z,"time":"2024-02-28T22:14:47.069315332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41542024-02-28T22:14:48.177Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069333731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41552024-02-28T22:14:48.177Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.06935472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41562024-02-28T22:14:48.177Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.0693695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41572024-02-28T22:14:48.177Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069396089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41582024-02-28T22:14:48.177Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069417269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"":"downstairs","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41592024-02-28T22:14:48.177Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069452748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41602024-02-28T22:14:48.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069470957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"0","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41612024-02-28T22:14:48.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069484887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"1","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41622024-02-28T22:14:48.177Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069511426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"client":"2","":"downstairs","session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41632024-02-28T22:14:48.177Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069545995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41642024-02-28T22:14:48.177Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069570445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41652024-02-28T22:14:48.177Z{"msg":"31b9f9fe-112d-4415-85e7-20562401e356 is now active with session: 4e860090-0f92-4eb8-91b8-c1ca7764efc6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069602334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41662024-02-28T22:14:48.177Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069625903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287,"session_id":"4e860090-0f92-4eb8-91b8-c1ca7764efc6"}
41672024-02-28T22:14:48.177Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:47.069657862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287}
41682024-02-28T22:14:48.177ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41692024-02-28T22:14:48.181ZLoading write count information from file "/tmp/test_repair_verify.data"
41702024-02-28T22:14:48.181ZRun Repair workload
41712024-02-28T22:14:48.181Z01/30 Write block 455 len 20480 data: 3 5 4 4 4
41722024-02-28T22:14:48.181Z02/30 Read block 442 len 36864
41732024-02-28T22:14:48.181Z03/30 Read block 298 len 24576
41742024-02-28T22:14:48.181Z04/30 Write block 466 len 36864 data: 3 2 2 2 3 4 4 4 6
41752024-02-28T22:14:48.181Z05/30 Write block 130 len 40960 data: 6 7 7 7 6 6 5 5 5 5
41762024-02-28T22:14:48.181Z06/30 Write block 515 len 36864 data: 4 3 2 3 3 3 3 3 3
41772024-02-28T22:14:48.181Z07/30 Read block 49 len 40960
41782024-02-28T22:14:48.181Z08/30 Read block 13 len 8192
41792024-02-28T22:14:48.184Z09/30 Write block 196 len 16384 data: 2 2 2 3
41802024-02-28T22:14:48.184Z10/30 Read block 113 len 36864
41812024-02-28T22:14:48.185Z11/30 Read block 287 len 16384
41822024-02-28T22:14:48.185Z12/30 Read block 464 len 32768
41832024-02-28T22:14:48.185Z13/30 Read block 16 len 20480
41842024-02-28T22:14:48.188Z14/30 Write block 446 len 16384 data: 6 5 4 4
41852024-02-28T22:14:48.188Z15/30 Write block 256 len 16384 data: 5 5 5 5
41862024-02-28T22:14:48.188Z16/30 Read block 75 len 40960
41872024-02-28T22:14:48.188Z17/30 Read block 494 len 16384
41882024-02-28T22:14:48.194Z18/30 Flush
41892024-02-28T22:14:48.194Z19/30 Write block 556 len 28672 data: 3 3 3 3 3 4 4
41902024-02-28T22:14:48.197Z20/30 Read block 471 len 12288
41912024-02-28T22:14:48.197Z21/30 Read block 375 len 12288
41922024-02-28T22:14:48.197Z22/30 Write block 542 len 24576 data: 6 6 7 7 8 8
41932024-02-28T22:14:48.197Z23/30 Write block 413 len 28672 data: 3 4 4 4 2 3 3
41942024-02-28T22:14:48.197Z24/30 Read block 107 len 4096
41952024-02-28T22:14:48.197Z25/30 Write block 75 len 32768 data: 2 2 3 3 2 4 2 3
41962024-02-28T22:14:48.198Z26/30 Read block 386 len 16384
41972024-02-28T22:14:48.201Z27/30 Read block 169 len 20480
41982024-02-28T22:14:48.201Z28/30 Read block 194 len 28672
41992024-02-28T22:14:48.201Z29/30 Read block 247 len 28672
42002024-02-28T22:14:48.201Z30/30 Read block 352 len 20480
42012024-02-28T22:14:48.205Z----------------------------------------------------------------
42022024-02-28T22:14:48.205Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
42032024-02-28T22:14:48.205ZStates: Active Active Active
42042024-02-28T22:14:48.205ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
42052024-02-28T22:14:48.205Z 1 Acked 1000 Write 5 Done Sent Done false
42062024-02-28T22:14:48.205Z 2 Acked 1001 Read 9 Done Sent Done false
42072024-02-28T22:14:48.205Z 3 Acked 1002 Read 6 Done Sent Done false
42082024-02-28T22:14:48.205Z 4 Acked 1003 Write 9 Done Sent Done false
42092024-02-28T22:14:48.205Z 5 Acked 1004 Write 10 Done Sent Done false
42102024-02-28T22:14:48.205Z 6 Acked 1005 Write 9 Done Sent Done false
42112024-02-28T22:14:48.205Z 7 Acked 1006 Read 10 Done Sent Done false
42122024-02-28T22:14:48.205Z 8 Acked 1007 Read 2 Done Sent Done false
42132024-02-28T22:14:48.205Z 9 Acked 1008 Write 4 Done Sent Done false
42142024-02-28T22:14:48.205Z 10 Acked 1009 Read 9 Done Sent Done false
42152024-02-28T22:14:48.205Z 11 Acked 1010 Read 4 Done Sent Done false
42162024-02-28T22:14:48.205Z 12 Acked 1011 Read 8 Done Sent Done false
42172024-02-28T22:14:48.205Z 13 Acked 1012 Read 5 Done Sent Done false
42182024-02-28T22:14:48.206Z 14 Acked 1013 Write 4 Done Sent Done false
42192024-02-28T22:14:48.206Z 15 Acked 1014 Write 4 Done Sent Done false
42202024-02-28T22:14:48.206Z 16 Acked 1015 Read 10 Done Sent Done false
42212024-02-28T22:14:48.206Z 17 Acked 1016 Read 4 Done Sent Done false
42222024-02-28T22:14:48.206Z 18 Acked 1017 Flush 0 Done Sent Done false
42232024-02-28T22:14:48.206Z 19 Acked 1018 Write 7 Done Sent Done false
42242024-02-28T22:14:48.206Z 20 Acked 1019 Read 3 Done Sent Done false
42252024-02-28T22:14:48.206Z 21 Acked 1020 Read 3 Done Sent Done false
42262024-02-28T22:14:48.206Z 22 Acked 1021 Write 6 Done Sent Done false
42272024-02-28T22:14:48.206Z 23 Acked 1022 Write 7 Done Sent Done false
42282024-02-28T22:14:48.206Z 24 Acked 1023 Read 1 Done Sent Done false
42292024-02-28T22:14:48.206Z 25 Acked 1024 Write 8 Done Sent Done false
42302024-02-28T22:14:48.206Z 26 Acked 1025 Read 4 Done Sent Done false
42312024-02-28T22:14:48.206Z 27 Acked 1026 Read 5 Done Sent Done false
42322024-02-28T22:14:48.206Z 28 Acked 1027 Read 7 Done Sent Done false
42332024-02-28T22:14:48.206Z 29 Acked 1028 Read 7 Done Sent Done false
42342024-02-28T22:14:48.206Z 30 Acked 1029 Read 5 Done Sent Done false
42352024-02-28T22:14:48.206Z STATES DS:0 DS:1 DS:2 TOTAL
42362024-02-28T22:14:48.206Z New 0 0 0 0
42372024-02-28T22:14:48.206Z Sent 0 30 0 30
42382024-02-28T22:14:48.206Z Done 30 0 30 60
42392024-02-28T22:14:48.206Z Skipped 0 0 0 0
42402024-02-28T22:14:48.207Z Error 0 0 0 0
42412024-02-28T22:14:48.207ZLast Flush: 1017 0 1017
42422024-02-28T22:14:48.207ZDownstairs last five completed:
42432024-02-28T22:14:48.207ZUpstairs last five completed: 30 29 28 27 26
42442024-02-28T22:14:48.207Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:47.100142242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1287}
42452024-02-28T22:14:48.207ZWrote out file "/tmp/test_repair_verify.data"
42462024-02-28T22:14:48.210Z
42472024-02-28T22:14:48.210ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42482024-02-28T22:14:48.227ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42492024-02-28T22:14:48.227Z 0 000-019 35 35 35 67 67 67 F F F
42502024-02-28T22:14:48.227Z 1 020-039 29 29 29 52 52 52 F F F
42512024-02-28T22:14:48.227Z 2 040-059 36 36 36 71 71 71 F F F
42522024-02-28T22:14:48.227Z 3 060-079 32 32 32 58 58 58 T F T <---
42532024-02-28T22:14:48.227Z 4 080-099 36 36 36 71 71 71 T F T <---
42542024-02-28T22:14:48.227Z 5 100-119 30 30 30 56 56 56 F F F
42552024-02-28T22:14:48.230Z 6 120-139 38 36 38 73 68 73 F F F <---
42562024-02-28T22:14:48.230Z 7 140-159 36 36 36 68 68 68 F F F
42572024-02-28T22:14:48.230Z 8 160-179 32 32 32 58 58 58 F F F
42582024-02-28T22:14:48.230Z 9 180-199 38 35 38 73 67 73 F F F <---
42592024-02-28T22:14:48.230Z 10 200-219 35 35 35 67 67 67 F F F
42602024-02-28T22:14:48.231Z 11 220-239 35 35 35 67 67 67 F F F
42612024-02-28T22:14:48.231Z 12 240-259 38 37 38 73 72 73 F F F <---
42622024-02-28T22:14:48.231Z 13 260-279 36 36 36 68 68 68 F F F
42632024-02-28T22:14:48.231Z 14 280-299 36 36 36 68 68 68 F F F
42642024-02-28T22:14:48.231Z 15 300-319 32 32 32 59 59 59 F F F
42652024-02-28T22:14:48.231Z 16 320-339 37 37 37 72 72 72 F F F
42662024-02-28T22:14:48.231Z 17 340-359 36 36 36 71 71 71 F F F
42672024-02-28T22:14:48.231Z 18 360-379 32 32 32 58 58 58 F F F
42682024-02-28T22:14:48.231Z 19 380-399 36 36 36 71 71 71 F F F
42692024-02-28T22:14:48.231Z 20 400-419 32 32 32 58 58 58 T F T <---
42702024-02-28T22:14:48.231Z 21 420-439 36 36 36 68 68 68 F F F
42712024-02-28T22:14:48.231Z 22 440-459 38 34 38 73 64 73 F F F <---
42722024-02-28T22:14:48.231Z 23 460-479 38 32 38 73 60 73 F F F <---
42732024-02-28T22:14:48.231Z 24 480-499 32 32 32 58 58 58 F F F
42742024-02-28T22:14:48.231Z 25 500-519 38 32 38 73 58 73 F F F <---
42752024-02-28T22:14:48.231Z 26 520-539 38 32 38 73 58 73 F F F <---
42762024-02-28T22:14:48.231Z 27 540-559 30 30 30 54 54 54 T F T <---
42772024-02-28T22:14:48.231Z 28 560-579 36 36 36 68 68 68 T F T <---
42782024-02-28T22:14:48.231Z 29 580-599 35 35 35 67 67 67 F F F
42792024-02-28T22:14:48.231ZMax gen: 38, Max flush: 73
42802024-02-28T22:14:48.231ZError: Difference in extent metadata found!
42812024-02-28T22:14:48.231ZA Difference in extent metadata is expected here
42822024-02-28T22:14:48.231ZOn loop 19
42832024-02-28T22:14:48.232Z
42842024-02-28T22:14:48.235ZVerifying data now
42852024-02-28T22:14:52.837ZNo change after verify
42862024-02-28T22:14:52.837ZLoop: 19 Downstairs dump after verify (and repair):
42872024-02-28T22:14:52.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42882024-02-28T22:14:52.851Z 0 000-019 35 35 35 67 67 67 F F F
42892024-02-28T22:14:52.851Z 1 020-039 29 29 29 52 52 52 F F F
42902024-02-28T22:14:52.851Z 2 040-059 36 36 36 71 71 71 F F F
42912024-02-28T22:14:52.851Z 3 060-079 39 39 39 74 74 74 F F F
42922024-02-28T22:14:52.851Z 4 080-099 39 39 39 74 74 74 F F F
42932024-02-28T22:14:52.851Z 5 100-119 30 30 30 56 56 56 F F F
42942024-02-28T22:14:52.851Z 6 120-139 38 38 38 73 73 73 F F F
42952024-02-28T22:14:52.851Z 7 140-159 36 36 36 68 68 68 F F F
42962024-02-28T22:14:52.851Z 8 160-179 32 32 32 58 58 58 F F F
42972024-02-28T22:14:52.851Z 9 180-199 38 38 38 73 73 73 F F F
42982024-02-28T22:14:52.855Z 10 200-219 35 35 35 67 67 67 F F F
42992024-02-28T22:14:52.855Z 11 220-239 35 35 35 67 67 67 F F F
43002024-02-28T22:14:52.855Z 12 240-259 38 38 38 73 73 73 F F F
43012024-02-28T22:14:52.855Z 13 260-279 36 36 36 68 68 68 F F F
43022024-02-28T22:14:52.855Z 14 280-299 36 36 36 68 68 68 F F F
43032024-02-28T22:14:52.855Z 15 300-319 32 32 32 59 59 59 F F F
43042024-02-28T22:14:52.855Z 16 320-339 37 37 37 72 72 72 F F F
43052024-02-28T22:14:52.855Z 17 340-359 36 36 36 71 71 71 F F F
43062024-02-28T22:14:52.855Z 18 360-379 32 32 32 58 58 58 F F F
43072024-02-28T22:14:52.855Z 19 380-399 36 36 36 71 71 71 F F F
43082024-02-28T22:14:52.855Z 20 400-419 39 39 39 74 74 74 F F F
43092024-02-28T22:14:52.855Z 21 420-439 36 36 36 68 68 68 F F F
43102024-02-28T22:14:52.855Z 22 440-459 38 38 38 73 73 73 F F F
43112024-02-28T22:14:52.855Z 23 460-479 38 38 38 73 73 73 F F F
43122024-02-28T22:14:52.855Z 24 480-499 32 32 32 58 58 58 F F F
43132024-02-28T22:14:52.855Z 25 500-519 38 38 38 73 73 73 F F F
43142024-02-28T22:14:52.855Z 26 520-539 38 38 38 73 73 73 F F F
43152024-02-28T22:14:52.856Z 27 540-559 39 39 39 74 74 74 F F F
43162024-02-28T22:14:52.856Z 28 560-579 39 39 39 74 74 74 F F F
43172024-02-28T22:14:52.856Z 29 580-599 35 35 35 67 67 67 F F F
43182024-02-28T22:14:52.856ZMax gen: 39, Max flush: 74
43192024-02-28T22:14:52.856Z
43202024-02-28T22:14:52.856ZBegin loop 20
43212024-02-28T22:14:52.856ZDownstairs to restart: 0
43222024-02-28T22:14:52.873Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77081756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
43232024-02-28T22:14:52.873Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771082283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
43242024-02-28T22:14:52.878Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771122062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
43252024-02-28T22:14:52.879Z{"msg":"Crucible b8c679f8-ec50-4b9a-8758-7ea377109fb8 has session id: 0c360e75-ea63-41f7-a0dc-1ddb6a29a74d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771147311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43262024-02-28T22:14:52.879Z{"msg":"Upstairs opts: Upstairs UUID: b8c679f8-ec50-4b9a-8758-7ea377109fb8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77118438Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43272024-02-28T22:14:52.879ZCrucible runtime is spawned
43282024-02-28T22:14:52.879Z{"msg":"Crucible stats registered with UUID: b8c679f8-ec50-4b9a-8758-7ea377109fb8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771224379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43292024-02-28T22:14:52.879Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771295067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
43302024-02-28T22:14:52.879Z{"msg":"b8c679f8-ec50-4b9a-8758-7ea377109fb8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771475822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43312024-02-28T22:14:52.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771516341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43322024-02-28T22:14:52.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771581049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43332024-02-28T22:14:52.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771649067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43342024-02-28T22:14:52.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771718205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43352024-02-28T22:14:52.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771751604Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43362024-02-28T22:14:52.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771780964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43372024-02-28T22:14:52.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771808623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43382024-02-28T22:14:52.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77190473Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43392024-02-28T22:14:52.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.771966008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43402024-02-28T22:14:52.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772021467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43412024-02-28T22:14:52.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772053216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43422024-02-28T22:14:52.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772073725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43432024-02-28T22:14:52.879Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772565782Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43442024-02-28T22:14:52.879Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772602451Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43452024-02-28T22:14:52.879Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77261854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43462024-02-28T22:14:52.879Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77264632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43472024-02-28T22:14:52.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772664819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43482024-02-28T22:14:52.880Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.772697158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43492024-02-28T22:14:52.880Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773115107Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43502024-02-28T22:14:52.880Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773156095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43512024-02-28T22:14:52.880Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773396069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43522024-02-28T22:14:52.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773594874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43532024-02-28T22:14:52.880Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773632353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43542024-02-28T22:14:52.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773650462Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43552024-02-28T22:14:52.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773664421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43562024-02-28T22:14:52.880Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773678621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43572024-02-28T22:14:52.880Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77370008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43582024-02-28T22:14:52.880Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773859186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43592024-02-28T22:14:52.880Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773905645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43602024-02-28T22:14:52.880Z{"msg":"[0]R flush_numbers[0..12]: [67, 52, 71, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773932804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43612024-02-28T22:14:52.880Z{"msg":"[0]R generation[0..12]: [35, 29, 36, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773964663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43622024-02-28T22:14:52.880Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.773985823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43632024-02-28T22:14:52.880Z{"msg":"[1]R flush_numbers[0..12]: [67, 52, 71, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774015342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43642024-02-28T22:14:52.880Z{"msg":"[1]R generation[0..12]: [35, 29, 36, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774030971Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43652024-02-28T22:14:52.880Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77406349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43662024-02-28T22:14:52.880Z{"msg":"[2]R flush_numbers[0..12]: [67, 52, 71, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77408208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
43672024-02-28T22:14:52.880Z"pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43682024-02-28T22:14:52.880Z{"msg":"[2]R generation[0..12]: [35, 29, 36, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774113029Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43692024-02-28T22:14:52.880Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774131619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43702024-02-28T22:14:52.881Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774161048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43712024-02-28T22:14:52.881Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774176218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43722024-02-28T22:14:52.881Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774212577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43732024-02-28T22:14:52.881Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774245945Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"":"downstairs","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43742024-02-28T22:14:52.881Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774265115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43752024-02-28T22:14:52.881Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774283755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"0","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43762024-02-28T22:14:52.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774324823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"1","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43772024-02-28T22:14:52.884Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774352013Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"client":"2","":"downstairs","session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43782024-02-28T22:14:52.885Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774385632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43792024-02-28T22:14:52.885Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774407151Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43802024-02-28T22:14:52.885Z{"msg":"b8c679f8-ec50-4b9a-8758-7ea377109fb8 is now active with session: 0c360e75-ea63-41f7-a0dc-1ddb6a29a74d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77442754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43812024-02-28T22:14:52.885Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.77444642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295,"session_id":"0c360e75-ea63-41f7-a0dc-1ddb6a29a74d"}
43822024-02-28T22:14:52.885Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:51.774480159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
43832024-02-28T22:14:52.885ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43842024-02-28T22:14:52.885ZLoading write count information from file "/tmp/test_repair_verify.data"
43852024-02-28T22:14:52.885ZRun Repair workload
43862024-02-28T22:14:52.885Z01/30 Write block 534 len 40960 data: 5 3 4 3 3 3 3 5 7 7
43872024-02-28T22:14:52.885Z02/30 Read block 17 len 8192
43882024-02-28T22:14:52.885Z03/30 Write block 36 len 32768 data: 3 4 4 3 3 3 2 2
43892024-02-28T22:14:52.885Z04/30 Read block 94 len 20480
43902024-02-28T22:14:52.885Z05/30 Read block 462 len 40960
43912024-02-28T22:14:52.885Z06/30 Read block 224 len 20480
43922024-02-28T22:14:52.888Z07/30 Write block 349 len 24576 data: 3 3 2 3 3 4
43932024-02-28T22:14:52.889Z08/30 Read block 27 len 12288
43942024-02-28T22:14:52.889Z09/30 Read block 83 len 4096
43952024-02-28T22:14:52.889Z10/30 Read block 555 len 32768
43962024-02-28T22:14:52.889Z11/30 Flush
43972024-02-28T22:14:52.892Z12/30 Write block 482 len 12288 data: 4 4 5
43982024-02-28T22:14:52.892Z13/30 Read block 547 len 12288
43992024-02-28T22:14:52.895Z14/30 Write block 389 len 4096 data: 4
44002024-02-28T22:14:52.895Z15/30 Read block 115 len 16384
44012024-02-28T22:14:52.895Z16/30 Read block 331 len 4096
44022024-02-28T22:14:52.895Z17/30 Read block 312 len 36864
44032024-02-28T22:14:52.898Z18/30 Read block 18 len 24576
44042024-02-28T22:14:52.898Z19/30 Write block 6 len 32768 data: 5 6 6 5 5 5 6 5
44052024-02-28T22:14:52.898Z20/30 Flush
44062024-02-28T22:14:52.903Z21/30 Write block 345 len 40960 data: 4 4 3 5 4 4 3 4 4 5
44072024-02-28T22:14:52.903Z22/30 Write block 358 len 4096 data: 4
44082024-02-28T22:14:52.906Z23/30 Read block 333 len 24576
44092024-02-28T22:14:52.906Z24/30 Write block 353 len 24576 data: 5 6 5 6 5 5
44102024-02-28T22:14:52.906Z25/30 Read block 153 len 4096
44112024-02-28T22:14:52.906Z26/30 Read block 200 len 20480
44122024-02-28T22:14:52.906Z27/30 Read block 120 len 12288
44132024-02-28T22:14:52.906Z28/30 Read block 506 len 32768
44142024-02-28T22:14:52.910Z29/30 Read block 305 len 36864
44152024-02-28T22:14:52.910Z30/30 Read block 587 len 20480
44162024-02-28T22:14:52.910Z----------------------------------------------------------------
44172024-02-28T22:14:52.910Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44182024-02-28T22:14:52.910ZStates: Active Active Active
44192024-02-28T22:14:52.910ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44202024-02-28T22:14:52.910Z 1 Acked 1000 Write 10 Done Done Done false
44212024-02-28T22:14:52.910Z 2 Acked 1001 Read 2 Done Done Done false
44222024-02-28T22:14:52.910Z 3 Acked 1002 Write 8 Done Done Done false
44232024-02-28T22:14:52.910Z 4 Acked 1003 Read 5 Done Done Done false
44242024-02-28T22:14:52.910Z 5 Acked 1004 Read 10 Done Done Done false
44252024-02-28T22:14:52.910Z 6 Acked 1005 Read 5 Sent Done Done false
44262024-02-28T22:14:52.910Z 7 Acked 1006 Write 6 Sent Done Done false
44272024-02-28T22:14:52.911Z 8 Acked 1007 Read 3 Sent Done Done false
44282024-02-28T22:14:52.911Z 9 Acked 1008 Read 1 Sent Done Done false
44292024-02-28T22:14:52.911Z 10 Acked 1009 Read 8 Sent Done Done false
44302024-02-28T22:14:52.911Z 11 Acked 1010 Flush 0 Sent Done Done false
44312024-02-28T22:14:52.911Z 12 Acked 1011 Write 3 Sent Done Done false
44322024-02-28T22:14:52.911Z 13 Acked 1012 Read 3 Sent Done Done false
44332024-02-28T22:14:52.911Z 14 Acked 1013 Write 1 Sent Done Done false
44342024-02-28T22:14:52.911Z 15 Acked 1014 Read 4 Sent Done Done false
44352024-02-28T22:14:52.911Z 16 Acked 1015 Read 1 Sent Done Done false
44362024-02-28T22:14:52.911Z 17 Acked 1016 Read 9 Sent Done Done false
44372024-02-28T22:14:52.911Z 18 Acked 1017 Read 6 Sent Done Done false
44382024-02-28T22:14:52.911Z 19 Acked 1018 Write 8 Sent Done Done false
44392024-02-28T22:14:52.911Z 20 Acked 1019 Flush 0 Sent Done Done false
44402024-02-28T22:14:52.911Z 21 Acked 1020 Write 10 Sent Done Done false
44412024-02-28T22:14:52.911Z 22 Acked 1021 Write 1 Sent Done Done false
44422024-02-28T22:14:52.911Z 23 Acked 1022 Read 6 Sent Done Done false
44432024-02-28T22:14:52.911Z 24 Acked 1023 Write 6 Sent Done Done false
44442024-02-28T22:14:52.911Z 25 Acked 1024 Read 1 Sent Done Done false
44452024-02-28T22:14:52.911Z 26 Acked 1025 Read 5 Sent Done Done false
44462024-02-28T22:14:52.911Z 27 Acked 1026 Read 3 Sent Done Done false
44472024-02-28T22:14:52.911Z 28 Acked 1027 Read 8 Sent Done Done false
44482024-02-28T22:14:52.911Z 29 Acked 1028 Read 9 Sent Done Done false
44492024-02-28T22:14:52.911Z 30 Acked 1029 Read 5 Sent Sent Done false
44502024-02-28T22:14:52.911Z STATES DS:0 DS:1 DS:2 TOTAL
44512024-02-28T22:14:52.912Z New 0 0 0 0
44522024-02-28T22:14:52.912Z Sent 25 1 0 26
44532024-02-28T22:14:52.912Z Done 5 29 30 64
44542024-02-28T22:14:52.912Z Skipped 0 0 0 0
44552024-02-28T22:14:52.912Z Error 0 0 0 0
44562024-02-28T22:14:52.912ZLast Flush: 0 1019 1019
44572024-02-28T22:14:52.912ZDownstairs last five completed:
44582024-02-28T22:14:52.912ZUpstairs last five completed: 30 29 28 27 26
44592024-02-28T22:14:52.912Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:51.806698331Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1295}
44602024-02-28T22:14:52.912ZWrote out file "/tmp/test_repair_verify.data"
44612024-02-28T22:14:52.915Z
44622024-02-28T22:14:52.918ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44632024-02-28T22:14:52.932ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44642024-02-28T22:14:52.932Z 0 000-019 35 40 40 67 76 76 F F F <---
44652024-02-28T22:14:52.932Z 1 020-039 29 40 40 52 75 75 T F F <---
44662024-02-28T22:14:52.932Z 2 040-059 36 40 40 71 75 75 T F F <---
44672024-02-28T22:14:52.932Z 3 060-079 39 39 39 74 74 74 F F F
44682024-02-28T22:14:52.932Z 4 080-099 39 39 39 74 74 74 F F F
44692024-02-28T22:14:52.932Z 5 100-119 30 30 30 56 56 56 F F F
44702024-02-28T22:14:52.932Z 6 120-139 38 38 38 73 73 73 F F F
44712024-02-28T22:14:52.935Z 7 140-159 36 36 36 68 68 68 F F F
44722024-02-28T22:14:52.936Z 8 160-179 32 32 32 58 58 58 F F F
44732024-02-28T22:14:52.936Z 9 180-199 38 38 38 73 73 73 F F F
44742024-02-28T22:14:52.936Z 10 200-219 35 35 35 67 67 67 F F F
44752024-02-28T22:14:52.936Z 11 220-239 35 35 35 67 67 67 F F F
44762024-02-28T22:14:52.936Z 12 240-259 38 38 38 73 73 73 F F F
44772024-02-28T22:14:52.936Z 13 260-279 36 36 36 68 68 68 F F F
44782024-02-28T22:14:52.936Z 14 280-299 36 36 36 68 68 68 F F F
44792024-02-28T22:14:52.936Z 15 300-319 32 32 32 59 59 59 F F F
44802024-02-28T22:14:52.936Z 16 320-339 37 37 37 72 72 72 F F F
44812024-02-28T22:14:52.936Z 17 340-359 36 40 40 71 75 75 F T T <---
44822024-02-28T22:14:52.936Z 18 360-379 32 32 32 58 58 58 F F F
44832024-02-28T22:14:52.936Z 19 380-399 36 40 40 71 76 76 F F F <---
44842024-02-28T22:14:52.936Z 20 400-419 39 39 39 74 74 74 F F F
44852024-02-28T22:14:52.936Z 21 420-439 36 36 36 68 68 68 F F F
44862024-02-28T22:14:52.936Z 22 440-459 38 38 38 73 73 73 F F F
44872024-02-28T22:14:52.936Z 23 460-479 38 38 38 73 73 73 F F F
44882024-02-28T22:14:52.936Z 24 480-499 32 40 40 58 76 76 F F F <---
44892024-02-28T22:14:52.936Z 25 500-519 38 38 38 73 73 73 F F F
44902024-02-28T22:14:52.936Z 26 520-539 38 40 40 73 75 75 T F F <---
44912024-02-28T22:14:52.936Z 27 540-559 39 40 40 74 75 75 T F F <---
44922024-02-28T22:14:52.936Z 28 560-579 39 39 39 74 74 74 F F F
44932024-02-28T22:14:52.936Z 29 580-599 35 35 35 67 67 67 F F F
44942024-02-28T22:14:52.937ZMax gen: 40, Max flush: 76
44952024-02-28T22:14:52.937ZError: Difference in extent metadata found!
44962024-02-28T22:14:52.937ZA Difference in extent metadata is expected here
44972024-02-28T22:14:52.937ZOn loop 20
44982024-02-28T22:14:52.937Z
44992024-02-28T22:14:52.940ZVerifying data now
45002024-02-28T22:14:58.376ZNo change after verify
45012024-02-28T22:14:58.376ZLoop: 20 Downstairs dump after verify (and repair):
45022024-02-28T22:14:58.400ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
45032024-02-28T22:14:58.400Z 0 000-019 40 40 40 76 76 76 F F F
45042024-02-28T22:14:58.400Z 1 020-039 40 40 40 75 75 75 F F F
45052024-02-28T22:14:58.400Z 2 040-059 40 40 40 75 75 75 F F F
45062024-02-28T22:14:58.400Z 3 060-079 39 39 39 74 74 74 F F F
45072024-02-28T22:14:58.400Z 4 080-099 39 39 39 74 74 74 F F F
45082024-02-28T22:14:58.404Z 5 100-119 30 30 30 56 56 56 F F F
45092024-02-28T22:14:58.404Z 6 120-139 38 38 38 73 73 73 F F F
45102024-02-28T22:14:58.404Z 7 140-159 36 36 36 68 68 68 F F F
45112024-02-28T22:14:58.404Z 8 160-179 32 32 32 58 58 58 F F F
45122024-02-28T22:14:58.405Z 9 180-199 38 38 38 73 73 73 F F F
45132024-02-28T22:14:58.405Z 10 200-219 35 35 35 67 67 67 F F F
45142024-02-28T22:14:58.405Z 11 220-239 35 35 35 67 67 67 F F F
45152024-02-28T22:14:58.405Z 12 240-259 38 38 38 73 73 73 F F F
45162024-02-28T22:14:58.405Z 13 260-279 36 36 36 68 68 68 F F F
45172024-02-28T22:14:58.405Z 14 280-299 36 36 36 68 68 68 F F F
45182024-02-28T22:14:58.405Z 15 300-319 32 32 32 59 59 59 F F F
45192024-02-28T22:14:58.405Z 16 320-339 37 37 37 72 72 72 F F F
45202024-02-28T22:14:58.405Z 17 340-359 41 41 41 77 77 77 F F F
45212024-02-28T22:14:58.405Z 18 360-379 32 32 32 58 58 58 F F F
45222024-02-28T22:14:58.405Z 19 380-399 40 40 40 76 76 76 F F F
45232024-02-28T22:14:58.405Z 20 400-419 39 39 39 74 74 74 F F F
45242024-02-28T22:14:58.405Z 21 420-439 36 36 36 68 68 68 F F F
45252024-02-28T22:14:58.405Z 22 440-459 38 38 38 73 73 73 F F F
45262024-02-28T22:14:58.406Z 23 460-479 38 38 38 73 73 73 F F F
45272024-02-28T22:14:58.406Z 24 480-499 40 40 40 76 76 76 F F F
45282024-02-28T22:14:58.406Z 25 500-519 38 38 38 73 73 73 F F F
45292024-02-28T22:14:58.406Z 26 520-539 40 40 40 75 75 75 F F F
45302024-02-28T22:14:58.406Z 27 540-559 40 40 40 75 75 75 F F F
45312024-02-28T22:14:58.406Z 28 560-579 39 39 39 74 74 74 F F F
45322024-02-28T22:14:58.406Z 29 580-599 35 35 35 67 67 67 F F F
45332024-02-28T22:14:58.406ZMax gen: 41, Max flush: 77
45342024-02-28T22:14:58.406Z
45352024-02-28T22:14:58.406ZBegin loop 21
45362024-02-28T22:14:58.406ZDownstairs to restart: 1
45372024-02-28T22:14:58.414Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.310672723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
45382024-02-28T22:14:58.414Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible",Crucible runtime is spawned
45392024-02-28T22:14:58.421Z"level":30,"time":"2024-02-28T22:14:57.311017443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
45402024-02-28T22:14:58.421Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311057972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
45412024-02-28T22:14:58.421Z{"msg":"Crucible 1d25594c-bb83-458f-b761-cb2cca7e023c has session id: 88b6675d-2b2b-40fa-b96d-647c3029eb73","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311080771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45422024-02-28T22:14:58.421Z{"msg":"Upstairs opts: Upstairs UUID: 1d25594c-bb83-458f-b761-cb2cca7e023c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.31112356Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45432024-02-28T22:14:58.421Z{"msg":"Crucible stats registered with UUID: 1d25594c-bb83-458f-b761-cb2cca7e023c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311161719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45442024-02-28T22:14:58.421Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311185819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
45452024-02-28T22:14:58.421Z{"msg":"1d25594c-bb83-458f-b761-cb2cca7e023c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311211458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45462024-02-28T22:14:58.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311233277Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45472024-02-28T22:14:58.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311266166Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45482024-02-28T22:14:58.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311292526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45492024-02-28T22:14:58.421Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311327805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45502024-02-28T22:14:58.421Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311366064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45512024-02-28T22:14:58.422Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311391683Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45522024-02-28T22:14:58.422Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311426532Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45532024-02-28T22:14:58.422Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311461571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45542024-02-28T22:14:58.422Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.31150226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45552024-02-28T22:14:58.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311543099Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45562024-02-28T22:14:58.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311789512Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45572024-02-28T22:14:58.422Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.311835751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45582024-02-28T22:14:58.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312311688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45592024-02-28T22:14:58.422Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312358566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45602024-02-28T22:14:58.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312407585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45612024-02-28T22:14:58.422Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312451704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45622024-02-28T22:14:58.422Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312487143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45632024-02-28T22:14:58.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312515672Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45642024-02-28T22:14:58.423Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.312996129Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45652024-02-28T22:14:58.423Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313038217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45662024-02-28T22:14:58.423Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313106936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45672024-02-28T22:14:58.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313575263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45682024-02-28T22:14:58.423Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313617062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45692024-02-28T22:14:58.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313643851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45702024-02-28T22:14:58.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.31367676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45712024-02-28T22:14:58.423Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313703719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45722024-02-28T22:14:58.423Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313738338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45732024-02-28T22:14:58.423Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313759938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45742024-02-28T22:14:58.423Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313783637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45752024-02-28T22:14:58.423Z{"msg":"[0]R flush_numbers[0..12]: [76, 75, 75, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313845915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45762024-02-28T22:14:58.423Z{"msg":"[0]R generation[0..12]: [40, 40, 40, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313873285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45772024-02-28T22:14:58.424Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313888504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45782024-02-28T22:14:58.424Z{"msg":"[1]R flush_numbers[0..12]: [76, 75, 75, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313909293Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45792024-02-28T22:14:58.424Z{"msg":"[1]R generation[0..12]: [40, 40, 40, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313923413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45802024-02-28T22:14:58.424Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313944263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45812024-02-28T22:14:58.424Z{"msg":"[2]R flush_numbers[0..12]: [76, 75, 75, 74, 74, 56, 73, 68, 58, 73, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313958552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45822024-02-28T22:14:58.424Z{"msg":"[2]R generation[0..12]: [40, 40, 40, 39, 39, 30, 38, 36, 32, 38, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.313984891Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45832024-02-28T22:14:58.424Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314006251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45842024-02-28T22:14:58.424Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
45852024-02-28T22:14:58.424Z,"time":"2024-02-28T22:14:57.31403435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45862024-02-28T22:14:58.424Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.31405283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45872024-02-28T22:14:58.424Z{"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314072829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45882024-02-28T22:14:58.424Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314086549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"":"downstairs","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45892024-02-28T22:14:58.425Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314106828Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45902024-02-28T22:14:58.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314118948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"0","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45912024-02-28T22:14:58.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314139667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"1","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45922024-02-28T22:14:58.425Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314153157Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"client":"2","":"downstairs","session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45932024-02-28T22:14:58.425Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314173316Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45942024-02-28T22:14:58.425Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314186596Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45952024-02-28T22:14:58.425Z{"msg":"1d25594c-bb83-458f-b761-cb2cca7e023c is now active with session: 88b6675d-2b2b-40fa-b96d-647c3029eb73","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314206525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45962024-02-28T22:14:58.425Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314218975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303,"session_id":"88b6675d-2b2b-40fa-b96d-647c3029eb73"}
45972024-02-28T22:14:58.425Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:14:57.314237784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
45982024-02-28T22:14:58.425ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45992024-02-28T22:14:58.429ZLoading write count information from file "/tmp/test_repair_verify.data"
46002024-02-28T22:14:58.430ZRun Repair workload
46012024-02-28T22:14:58.430Z01/30 Write block 190 len 24576 data: 4 4 5 4 3 3
46022024-02-28T22:14:58.430Z02/30 Write block 561 len 4096 data: 5
46032024-02-28T22:14:58.430Z03/30 Write block 80 len 36864 data: 5 3 4 3 3 3 3 4 4
46042024-02-28T22:14:58.430Z04/30 Read block 456 len 8192
46052024-02-28T22:14:58.430Z05/30 Read block 201 len 16384
46062024-02-28T22:14:58.430Z06/30 Write block 248 len 4096 data: 7
46072024-02-28T22:14:58.430Z07/30 Read block 578 len 8192
46082024-02-28T22:14:58.430Z08/30 Read block 214 len 36864
46092024-02-28T22:14:58.430Z09/30 Write block 251 len 32768 data: 6 6 7 6 5 6 6 6
46102024-02-28T22:14:58.430Z10/30 Write block 258 len 20480 data: 7 6 4 4 4
46112024-02-28T22:14:58.431Z11/30 Read block 238 len 28672
46122024-02-28T22:14:58.431Z12/30 Read block 590 len 40960
46132024-02-28T22:14:58.431Z13/30 Read block 572 len 16384
46142024-02-28T22:14:58.431Z14/30 Write block 266 len 12288 data: 2 2 3
46152024-02-28T22:14:58.435Z15/30 Read block 323 len 20480
46162024-02-28T22:14:58.435Z16/30 Read block 566 len 8192
46172024-02-28T22:14:58.435Z17/30 Write block 402 len 28672 data: 3 4 4 4 4 5 5
46182024-02-28T22:14:58.435Z18/30 Write block 395 len 20480 data: 6 6 5 5 4
46192024-02-28T22:14:58.435Z19/30 Read block 64 len 4096
46202024-02-28T22:14:58.435Z20/30 Read block 520 len 36864
46212024-02-28T22:14:58.435Z21/30 Read block 526 len 12288
46222024-02-28T22:14:58.438Z22/30 Read block 56 len 40960
46232024-02-28T22:14:58.438Z23/30 Flush
46242024-02-28T22:14:58.442Z24/30 Write block 128 len 4096 data: 8
46252024-02-28T22:14:58.442Z25/30 Read block 470 len 8192
46262024-02-28T22:14:58.442Z26/30 Write block 90 len 20480 data: 5 5 6 4 4
46272024-02-28T22:14:58.442Z27/30 Read block 181 len 4096
46282024-02-28T22:14:58.446Z28/30 Write block 166 len 40960 data: 6 9 7 7 6 5 5 5 5 4
46292024-02-28T22:14:58.446Z29/30 Write block 516 len 24576 data: 4 3 4 4 4 4
46302024-02-28T22:14:58.446Z30/30 Read block 261 len 8192
46312024-02-28T22:14:58.446Z----------------------------------------------------------------
46322024-02-28T22:14:58.446Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46332024-02-28T22:14:58.446ZStates: Active Active Active
46342024-02-28T22:14:58.446ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46352024-02-28T22:14:58.446Z 1 Acked 1000 Write 6 Done Done Done false
46362024-02-28T22:14:58.446Z 2 Acked 1001 Write 1 Done Sent Done false
46372024-02-28T22:14:58.447Z 3 Acked 1002 Write 9 Done Sent Done false
46382024-02-28T22:14:58.447Z 4 Acked 1003 Read 2 Done Sent Done false
46392024-02-28T22:14:58.447Z 5 Acked 1004 Read 4 Done Sent Done false
46402024-02-28T22:14:58.447Z 6 Acked 1005 Write 1 Done Sent Done false
46412024-02-28T22:14:58.447Z 7 Acked 1006 Read 2 Done Sent Done false
46422024-02-28T22:14:58.447Z 8 Acked 1007 Read 9 Done Sent Done false
46432024-02-28T22:14:58.447Z 9 Acked 1008 Write 8 Done Sent Done false
46442024-02-28T22:14:58.447Z 10 Acked 1009 Write 5 Done Sent Done false
46452024-02-28T22:14:58.447Z 11 Acked 1010 Read 7 Done Sent Done false
46462024-02-28T22:14:58.447Z 12 Acked 1011 Read 10 Done Sent Done false
46472024-02-28T22:14:58.447Z 13 Acked 1012 Read 4 Done Sent Done false
46482024-02-28T22:14:58.447Z 14 Acked 1013 Write 3 Done Sent Done false
46492024-02-28T22:14:58.447Z 15 Acked 1014 Read 5 Done Sent Done false
46502024-02-28T22:14:58.447Z 16 Acked 1015 Read 2 Done Sent Done false
46512024-02-28T22:14:58.447Z 17 Acked 1016 Write 7 Done Sent Done false
46522024-02-28T22:14:58.447Z 18 Acked 1017 Write 5 Done Sent Done false
46532024-02-28T22:14:58.447Z 19 Acked 1018 Read 1 Done Sent Done false
46542024-02-28T22:14:58.447Z 20 Acked 1019 Read 9 Done Sent Done false
46552024-02-28T22:14:58.447Z 21 Acked 1020 Read 3 Done Sent Done false
46562024-02-28T22:14:58.448Z 22 Acked 1021 Read 10 Done Sent Done false
46572024-02-28T22:14:58.448Z 23 Acked 1022 Flush 0 Done Sent Done false
46582024-02-28T22:14:58.448Z 24 Acked 1023 Write 1 Done Sent Done false
46592024-02-28T22:14:58.448Z 25 Acked 1024 Read 2 Done Sent Done false
46602024-02-28T22:14:58.448Z 26 Acked 1025 Write 5 Done Sent Done false
46612024-02-28T22:14:58.448Z 27 Acked 1026 Read 1 Done Sent Done false
46622024-02-28T22:14:58.448Z 28 Acked 1027 Write 10 Done Sent Done false
46632024-02-28T22:14:58.448Z 29 Acked 1028 Write 6 Done Sent Done false
46642024-02-28T22:14:58.448Z 30 Acked 1029 Read 2 Done Sent Done false
46652024-02-28T22:14:58.448Z STATES DS:0 DS:1 DS:2 TOTAL
46662024-02-28T22:14:58.448Z New 0 0 0 0
46672024-02-28T22:14:58.448Z Sent 0 29 0 29
46682024-02-28T22:14:58.448Z Done 30 1 30 61
46692024-02-28T22:14:58.448Z Skipped 0 0 0 0
46702024-02-28T22:14:58.448Z Error 0 0 0 0
46712024-02-28T22:14:58.448ZLast Flush: 1022 0 1022
46722024-02-28T22:14:58.448ZDownstairs last five completed:
46732024-02-28T22:14:58.448ZUpstairs last five completed: 30 29 28 27 26
46742024-02-28T22:14:58.448Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:14:57.342544905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1303}
46752024-02-28T22:14:58.448ZWrote out file "/tmp/test_repair_verify.data"
46762024-02-28T22:14:58.452Z
46772024-02-28T22:14:58.455ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46782024-02-28T22:14:58.469ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46792024-02-28T22:14:58.469Z 0 000-019 40 40 40 76 76 76 F F F
46802024-02-28T22:14:58.469Z 1 020-039 40 40 40 75 75 75 F F F
46812024-02-28T22:14:58.469Z 2 040-059 40 40 40 75 75 75 F F F
46822024-02-28T22:14:58.469Z 3 060-079 39 39 39 74 74 74 F F F
46832024-02-28T22:14:58.469Z 4 080-099 42 39 42 78 74 78 T F T <---
46842024-02-28T22:14:58.469Z 5 100-119 30 30 30 56 56 56 F F F
46852024-02-28T22:14:58.469Z 6 120-139 38 38 38 73 73 73 T F T <---
46862024-02-28T22:14:58.470Z 7 140-159 36 36 36 68 68 68 F F F
46872024-02-28T22:14:58.470Z 8 160-179 32 32 32 58 58 58 T F T <---
46882024-02-28T22:14:58.470Z 9 180-199 42 38 42 78 73 78 F T F <---
46892024-02-28T22:14:58.470Z 10 200-219 35 35 35 67 67 67 F F F
46902024-02-28T22:14:58.470Z 11 220-239 35 35 35 67 67 67 F F F
46912024-02-28T22:14:58.473Z 12 240-259 42 38 42 78 73 78 F F F <---
46922024-02-28T22:14:58.473Z 13 260-279 42 36 42 78 68 78 F F F <---
46932024-02-28T22:14:58.473Z 14 280-299 36 36 36 68 68 68 F F F
46942024-02-28T22:14:58.474Z 15 300-319 32 32 32 59 59 59 F F F
46952024-02-28T22:14:58.474Z 16 320-339 37 37 37 72 72 72 F F F
46962024-02-28T22:14:58.474Z 17 340-359 41 41 41 77 77 77 F F F
46972024-02-28T22:14:58.474Z 18 360-379 32 32 32 58 58 58 F F F
46982024-02-28T22:14:58.474Z 19 380-399 42 40 42 78 76 78 F F F <---
46992024-02-28T22:14:58.474Z 20 400-419 42 39 42 78 74 78 F F F <---
47002024-02-28T22:14:58.474Z 21 420-439 36 36 36 68 68 68 F F F
47012024-02-28T22:14:58.474Z 22 440-459 38 38 38 73 73 73 F F F
47022024-02-28T22:14:58.474Z 23 460-479 38 38 38 73 73 73 F F F
47032024-02-28T22:14:58.474Z 24 480-499 40 40 40 76 76 76 F F F
47042024-02-28T22:14:58.474Z 25 500-519 38 38 38 73 73 73 T F T <---
47052024-02-28T22:14:58.474Z 26 520-539 40 40 40 75 75 75 T F T <---
47062024-02-28T22:14:58.474Z 27 540-559 40 40 40 75 75 75 F F F
47072024-02-28T22:14:58.474Z 28 560-579 42 39 42 78 74 78 F F F <---
47082024-02-28T22:14:58.474Z 29 580-599 35 35 35 67 67 67 F F F
47092024-02-28T22:14:58.474ZMax gen: 42, Max flush: 78
47102024-02-28T22:14:58.474ZError: Difference in extent metadata found!
47112024-02-28T22:14:58.474ZA Difference in extent metadata is expected here
47122024-02-28T22:14:58.474ZOn loop 21
47132024-02-28T22:14:58.474Z
47142024-02-28T22:14:58.478ZVerifying data now
47152024-02-28T22:15:03.039ZNo change after verify
47162024-02-28T22:15:03.039ZLoop: 21 Downstairs dump after verify (and repair):
47172024-02-28T22:15:03.054ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47182024-02-28T22:15:03.054Z 0 000-019 40 40 40 76 76 76 F F F
47192024-02-28T22:15:03.054Z 1 020-039 40 40 40 75 75 75 F F F
47202024-02-28T22:15:03.054Z 2 040-059 40 40 40 75 75 75 F F F
47212024-02-28T22:15:03.054Z 3 060-079 39 39 39 74 74 74 F F F
47222024-02-28T22:15:03.054Z 4 080-099 43 43 43 79 79 79 F F F
47232024-02-28T22:15:03.054Z 5 100-119 30 30 30 56 56 56 F F F
47242024-02-28T22:15:03.054Z 6 120-139 43 43 43 79 79 79 F F F
47252024-02-28T22:15:03.054Z 7 140-159 36 36 36 68 68 68 F F F
47262024-02-28T22:15:03.054Z 8 160-179 43 43 43 79 79 79 F F F
47272024-02-28T22:15:03.054Z 9 180-199 42 42 42 78 78 78 F F F
47282024-02-28T22:15:03.054Z 10 200-219 35 35 35 67 67 67 F F F
47292024-02-28T22:15:03.054Z 11 220-239 35 35 35 67 67 67 F F F
47302024-02-28T22:15:03.058Z 12 240-259 42 42 42 78 78 78 F F F
47312024-02-28T22:15:03.058Z 13 260-279 42 42 42 78 78 78 F F F
47322024-02-28T22:15:03.058Z 14 280-299 36 36 36 68 68 68 F F F
47332024-02-28T22:15:03.058Z 15 300-319 32 32 32 59 59 59 F F F
47342024-02-28T22:15:03.058Z 16 320-339 37 37 37 72 72 72 F F F
47352024-02-28T22:15:03.058Z 17 340-359 41 41 41 77 77 77 F F F
47362024-02-28T22:15:03.058Z 18 360-379 32 32 32 58 58 58 F F F
47372024-02-28T22:15:03.058Z 19 380-399 42 42 42 78 78 78 F F F
47382024-02-28T22:15:03.058Z 20 400-419 42 42 42 78 78 78 F F F
47392024-02-28T22:15:03.058Z 21 420-439 36 36 36 68 68 68 F F F
47402024-02-28T22:15:03.058Z 22 440-459 38 38 38 73 73 73 F F F
47412024-02-28T22:15:03.058Z 23 460-479 38 38 38 73 73 73 F F F
47422024-02-28T22:15:03.059Z 24 480-499 40 40 40 76 76 76 F F F
47432024-02-28T22:15:03.059Z 25 500-519 43 43 43 79 79 79 F F F
47442024-02-28T22:15:03.059Z 26 520-539 43 43 43 79 79 79 F F F
47452024-02-28T22:15:03.059Z 27 540-559 40 40 40 75 75 75 F F F
47462024-02-28T22:15:03.059Z 28 560-579 42 42 42 78 78 78 F F F
47472024-02-28T22:15:03.059Z 29 580-599 35 35 35 67 67 67 F F F
47482024-02-28T22:15:03.059ZMax gen: 43, Max flush: 79
47492024-02-28T22:15:03.059Z
47502024-02-28T22:15:03.059ZBegin loop 22
47512024-02-28T22:15:03.059ZDownstairs to restart: 0
47522024-02-28T22:15:03.078Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.974734522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
47532024-02-28T22:15:03.078Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975011445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
47542024-02-28T22:15:03.078Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975045794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
47552024-02-28T22:15:03.079Z{"msg":"Crucible eb6d2616-28f7-414a-8dc4-7459fe74d867 has session id: 2fd17053-d27d-4b79-b890-e562e84da9cc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975075153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47562024-02-28T22:15:03.085Z{"msg":"Upstairs opts: Upstairs UUID: eb6d2616-28f7-414a-8dc4-7459fe74d867, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975126692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47572024-02-28T22:15:03.085Z{"msg":"Crucible stats registered with UUID: eb6d2616-28f7-414a-8dc4-7459fe74d867","v":0,"name":"crucible","level":30Crucible runtime is spawned
47582024-02-28T22:15:03.085Z,"time":"2024-02-28T22:15:01.97516872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47592024-02-28T22:15:03.085Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975287897Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
47602024-02-28T22:15:03.085Z{"msg":"eb6d2616-28f7-414a-8dc4-7459fe74d867 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975467012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47612024-02-28T22:15:03.085Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975510191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47622024-02-28T22:15:03.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.9755501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47632024-02-28T22:15:03.085Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975577759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47642024-02-28T22:15:03.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975604358Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47652024-02-28T22:15:03.085Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975632878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47662024-02-28T22:15:03.085Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975660797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47672024-02-28T22:15:03.085Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975695586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47682024-02-28T22:15:03.085Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975727485Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47692024-02-28T22:15:03.086Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975773964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47702024-02-28T22:15:03.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975892651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47712024-02-28T22:15:03.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975942119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47722024-02-28T22:15:03.086Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.975987308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47732024-02-28T22:15:03.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.976939862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47742024-02-28T22:15:03.086Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.97701046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47752024-02-28T22:15:03.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977083128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47762024-02-28T22:15:03.086Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977150416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47772024-02-28T22:15:03.086Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977194824Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47782024-02-28T22:15:03.086Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977227914Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47792024-02-28T22:15:03.086Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977486507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47802024-02-28T22:15:03.086Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977525226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47812024-02-28T22:15:03.086Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.977757049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47822024-02-28T22:15:03.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.97808981Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47832024-02-28T22:15:03.086Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978138998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47842024-02-28T22:15:03.086Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978184377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47852024-02-28T22:15:03.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978212696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47862024-02-28T22:15:03.086Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978239096Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47872024-02-28T22:15:03.086Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978263815Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47882024-02-28T22:15:03.086Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978336113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47892024-02-28T22:15:03.086Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978373682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47902024-02-28T22:15:03.086Z{"msg":"[0]R flush_numbers[0..12]: [76, 75, 75, 74, 79, 56, 79, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978391251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47912024-02-28T22:15:03.087Z{"msg":"[0]R generation[0..12]: [40, 40, 40, 39, 43, 30, 43, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978424261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47922024-02-28T22:15:03.087Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.97844643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47932024-02-28T22:15:03.087Z{"msg":"[1]R flush_numbers[0..12]: [76, 75, 75, 74, 79, 56, 79, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978476609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47942024-02-28T22:15:03.087Z{"msg":"[1]R generation[0..12]: [40, 40, 40, 39, 43, 30, 43, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978493359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47952024-02-28T22:15:03.087Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
47962024-02-28T22:15:03.087Z2024-02-28T22:15:01.978525248Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47972024-02-28T22:15:03.087Z{"msg":"[2]R flush_numbers[0..12]: [76, 75, 75, 74, 79, 56, 79, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978549517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47982024-02-28T22:15:03.087Z{"msg":"[2]R generation[0..12]: [40, 40, 40, 39, 43, 30, 43, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978580476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
47992024-02-28T22:15:03.087Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978596736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48002024-02-28T22:15:03.087Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978626405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48012024-02-28T22:15:03.087Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978644395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48022024-02-28T22:15:03.087Z{"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978673904Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48032024-02-28T22:15:03.087Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978689603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"":"downstairs","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48042024-02-28T22:15:03.087Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978720042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48052024-02-28T22:15:03.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978734532Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"0","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48062024-02-28T22:15:03.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978749222Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"1","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48072024-02-28T22:15:03.087Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978779421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"client":"2","":"downstairs","session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48082024-02-28T22:15:03.087Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.9787947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48092024-02-28T22:15:03.087Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.97881505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48102024-02-28T22:15:03.087Z{"msg":"eb6d2616-28f7-414a-8dc4-7459fe74d867 is now active with session: 2fd17053-d27d-4b79-b890-e562e84da9cc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978844129Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48112024-02-28T22:15:03.087Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:01.978857909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311,"session_id":"2fd17053-d27d-4b79-b890-e562e84da9cc"}
48122024-02-28T22:15:03.087Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48132024-02-28T22:15:03.088Z,"time":"2024-02-28T22:15:01.978877588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
48142024-02-28T22:15:03.088ZLoading write count information from file "/tmp/test_repair_verify.data"
48152024-02-28T22:15:03.088ZRun Repair workload
48162024-02-28T22:15:03.088Z01/30 Write block 311 len 8192 data: 6 6
48172024-02-28T22:15:03.092Z02/30 Read block 388 len 36864
48182024-02-28T22:15:03.092Z03/30 Write block 103 len 4096 data: 5
48192024-02-28T22:15:03.092Z04/30 Read block 334 len 28672
48202024-02-28T22:15:03.092Z05/30 Read block 277 len 16384
48212024-02-28T22:15:03.092Z06/30 Write block 291 len 36864 data: 4 5 4 3 6 6 6 4 3
48222024-02-28T22:15:03.092Z07/30 Read block 252 len 12288
48232024-02-28T22:15:03.092Z08/30 Write block 517 len 16384 data: 4 5 5 5
48242024-02-28T22:15:03.092Z09/30 Read block 404 len 20480
48252024-02-28T22:15:03.092Z10/30 Read block 222 len 24576
48262024-02-28T22:15:03.096Z11/30 Flush
48272024-02-28T22:15:03.096Z12/30 Write block 516 len 12288 data: 5 5 6
48282024-02-28T22:15:03.096Z13/30 Write block 362 len 12288 data: 4 5 3
48292024-02-28T22:15:03.096Z14/30 Read block 467 len 32768
48302024-02-28T22:15:03.100Z15/30 Read block 170 len 32768
48312024-02-28T22:15:03.100Z16/30 Flush
48322024-02-28T22:15:03.108Z17/30 Write block 10 len 24576 data: 6 6 7 6 5 5
48332024-02-28T22:15:03.108Z18/30 Read block 571 len 16384
48342024-02-28T22:15:03.108Z19/30 Read block 60 len 32768
48352024-02-28T22:15:03.108Z20/30 Write block 418 len 32768 data: 4 4 5 5 5 5 5 5
48362024-02-28T22:15:03.108Z21/30 Write block 2 len 40960 data: 3 5 5 5 6 7 7 6 7 7
48372024-02-28T22:15:03.112Z22/30 Read block 456 len 4096
48382024-02-28T22:15:03.112Z23/30 Write block 572 len 40960 data: 5 4 3 4 3 4 4 4 4 4
48392024-02-28T22:15:03.112Z24/30 Read block 96 len 8192
48402024-02-28T22:15:03.112Z25/30 Read block 523 len 12288
48412024-02-28T22:15:03.112Z26/30 Flush
48422024-02-28T22:15:03.112Z27/30 Write block 79 len 32768 data: 3 6 4 5 4 4 4 4
48432024-02-28T22:15:03.112Z28/30 Write block 61 len 16384 data: 4 5 6 6
48442024-02-28T22:15:03.117Z29/30 Write block 131 len 32768 data: 8 8 8 7 7 6 6 6
48452024-02-28T22:15:03.117Z30/30 Write block 288 len 36864 data: 2 2 3 5 6 5 4 7 7
48462024-02-28T22:15:03.117Z----------------------------------------------------------------
48472024-02-28T22:15:03.117Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48482024-02-28T22:15:03.117ZStates: Active Active Active
48492024-02-28T22:15:03.117ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48502024-02-28T22:15:03.117Z 1 Acked 1000 Write 2 Sent Done Done false
48512024-02-28T22:15:03.117Z 2 Acked 1001 Read 9 Sent Done Done false
48522024-02-28T22:15:03.117Z 3 Acked 1002 Write 1 Sent Done Done false
48532024-02-28T22:15:03.117Z 4 Acked 1003 Read 7 Sent Done Done false
48542024-02-28T22:15:03.117Z 5 Acked 1004 Read 4 Sent Done Done false
48552024-02-28T22:15:03.117Z 6 Acked 1005 Write 9 Sent Done Done false
48562024-02-28T22:15:03.117Z 7 Acked 1006 Read 3 Sent Done Done false
48572024-02-28T22:15:03.117Z 8 Acked 1007 Write 4 Sent Done Done false
48582024-02-28T22:15:03.117Z 9 Acked 1008 Read 5 Sent Done Done false
48592024-02-28T22:15:03.117Z 10 Acked 1009 Read 6 Sent Done Done false
48602024-02-28T22:15:03.117Z 11 Acked 1010 Flush 0 Sent Done Done false
48612024-02-28T22:15:03.118Z 12 Acked 1011 Write 3 Sent Done Done false
48622024-02-28T22:15:03.118Z 13 Acked 1012 Write 3 Sent Done Done false
48632024-02-28T22:15:03.118Z 14 Acked 1013 Read 8 Sent Done Done false
48642024-02-28T22:15:03.118Z 15 Acked 1014 Read 8 Sent Done Done false
48652024-02-28T22:15:03.118Z 16 Acked 1015 Flush 0 Sent Done Done false
48662024-02-28T22:15:03.118Z 17 Acked 1016 Write 6 Sent Done Done false
48672024-02-28T22:15:03.118Z 18 Acked 1017 Read 4 Sent Done Done false
48682024-02-28T22:15:03.118Z 19 Acked 1018 Read 8 Sent Done Done false
48692024-02-28T22:15:03.118Z 20 Acked 1019 Write 8 Sent Done Done false
48702024-02-28T22:15:03.118Z 21 Acked 1020 Write 10 Sent Done Done false
48712024-02-28T22:15:03.118Z 22 Acked 1021 Read 1 Sent Done Done false
48722024-02-28T22:15:03.118Z 23 Acked 1022 Write 10 Sent Done Done false
48732024-02-28T22:15:03.118Z 24 Acked 1023 Read 2 Sent Done Done false
48742024-02-28T22:15:03.118Z 25 Acked 1024 Read 3 Sent Done Done false
48752024-02-28T22:15:03.118Z 26 Acked 1025 Flush 0 Sent Done Done false
48762024-02-28T22:15:03.118Z 27 Acked 1026 Write 8 Sent Done Done false
48772024-02-28T22:15:03.118Z 28 Acked 1027 Write 4 Sent Done Done false
48782024-02-28T22:15:03.118Z 29 Acked 1028 Write 8 Sent Done Done false
48792024-02-28T22:15:03.118Z 30 Acked 1029 Write 9 Sent Sent Sent false
48802024-02-28T22:15:03.118Z STATES DS:0 DS:1 DS:2 TOTAL
48812024-02-28T22:15:03.118Z New 0 0 0 0
48822024-02-28T22:15:03.118Z Sent 30 1 1 32
48832024-02-28T22:15:03.118Z Done 0 29 29 58
48842024-02-28T22:15:03.119Z Skipped 0 0 0 0
48852024-02-28T22:15:03.119Z Error 0 0 0 0
48862024-02-28T22:15:03.119ZLast Flush: 0 1025 1025
48872024-02-28T22:15:03.119ZDownstairs last five completed:
48882024-02-28T22:15:03.119ZUpstairs last five completed: 30 29 28 27 26
48892024-02-28T22:15:03.119Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:02.01148766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1311}
48902024-02-28T22:15:03.119ZWrote out file "/tmp/test_repair_verify.data"
48912024-02-28T22:15:03.123Z
48922024-02-28T22:15:03.127ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48932024-02-28T22:15:03.141ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48942024-02-28T22:15:03.141Z 0 000-019 40 44 44 76 82 82 F F F <---
48952024-02-28T22:15:03.141Z 1 020-039 40 40 40 75 75 75 F F F
48962024-02-28T22:15:03.141Z 2 040-059 40 40 40 75 75 75 F F F
48972024-02-28T22:15:03.141Z 3 060-079 39 39 39 74 74 74 F T T <---
48982024-02-28T22:15:03.141Z 4 080-099 43 43 43 79 79 79 F T T <---
48992024-02-28T22:15:03.141Z 5 100-119 30 44 44 56 80 80 F F F <---
49002024-02-28T22:15:03.141Z 6 120-139 43 43 43 79 79 79 F T T <---
49012024-02-28T22:15:03.141Z 7 140-159 36 36 36 68 68 68 F F F
49022024-02-28T22:15:03.141Z 8 160-179 43 43 43 79 79 79 F F F
49032024-02-28T22:15:03.141Z 9 180-199 42 42 42 78 78 78 F F F
49042024-02-28T22:15:03.141Z 10 200-219 35 35 35 67 67 67 F F F
49052024-02-28T22:15:03.141Z 11 220-239 35 35 35 67 67 67 F F F
49062024-02-28T22:15:03.141Z 12 240-259 42 42 42 78 78 78 F F F
49072024-02-28T22:15:03.141Z 13 260-279 42 42 42 78 78 78 F F F
49082024-02-28T22:15:03.141Z 14 280-299 36 44 44 68 80 80 F T T <---
49092024-02-28T22:15:03.141Z 15 300-319 32 44 44 59 80 80 F F F <---
49102024-02-28T22:15:03.141Z 16 320-339 37 37 37 72 72 72 F F F
49112024-02-28T22:15:03.142Z 17 340-359 41 41 41 77 77 77 F F F
49122024-02-28T22:15:03.142Z 18 360-379 32 44 44 58 81 81 F F F <---
49132024-02-28T22:15:03.151Z 19 380-399 42 42 42 78 78 78 F F F
49142024-02-28T22:15:03.151Z 20 400-419 42 44 44 78 82 82 F F F <---
49152024-02-28T22:15:03.151Z 21 420-439 36 44 44 68 82 82 F F F <---
49162024-02-28T22:15:03.151Z 22 440-459 38 38 38 73 73 73 F F F
49172024-02-28T22:15:03.151Z 23 460-479 38 38 38 73 73 73 F F F
49182024-02-28T22:15:03.151Z 24 480-499 40 40 40 76 76 76 F F F
49192024-02-28T22:15:03.151Z 25 500-519 43 44 44 79 81 81 F F F <---
49202024-02-28T22:15:03.151Z 26 520-539 43 44 44 79 80 80 F F F <---
49212024-02-28T22:15:03.151Z 27 540-559 40 40 40 75 75 75 F F F
49222024-02-28T22:15:03.151Z 28 560-579 42 44 44 78 82 82 F F F <---
49232024-02-28T22:15:03.151Z 29 580-599 35 44 44 67 82 82 F F F <---
49242024-02-28T22:15:03.151ZMax gen: 44, Max flush: 82
49252024-02-28T22:15:03.151ZError: Difference in extent metadata found!
49262024-02-28T22:15:03.151ZA Difference in extent metadata is expected here
49272024-02-28T22:15:03.151ZOn loop 22
49282024-02-28T22:15:03.151Z
49292024-02-28T22:15:03.151ZVerifying data now
49302024-02-28T22:15:07.852ZNo change after verify
49312024-02-28T22:15:07.852ZLoop: 22 Downstairs dump after verify (and repair):
49322024-02-28T22:15:07.867ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49332024-02-28T22:15:07.867Z 0 000-019 44 44 44 82 82 82 F F F
49342024-02-28T22:15:07.867Z 1 020-039 40 40 40 75 75 75 F F F
49352024-02-28T22:15:07.867Z 2 040-059 40 40 40 75 75 75 F F F
49362024-02-28T22:15:07.867Z 3 060-079 45 45 45 83 83 83 F F F
49372024-02-28T22:15:07.867Z 4 080-099 45 45 45 83 83 83 F F F
49382024-02-28T22:15:07.867Z 5 100-119 44 44 44 80 80 80 F F F
49392024-02-28T22:15:07.867Z 6 120-139 45 45 45 83 83 83 F F F
49402024-02-28T22:15:07.867Z 7 140-159 36 36 36 68 68 68 F F F
49412024-02-28T22:15:07.867Z 8 160-179 43 43 43 79 79 79 F F F
49422024-02-28T22:15:07.871Z 9 180-199 42 42 42 78 78 78 F F F
49432024-02-28T22:15:07.871Z 10 200-219 35 35 35 67 67 67 F F F
49442024-02-28T22:15:07.871Z 11 220-239 35 35 35 67 67 67 F F F
49452024-02-28T22:15:07.871Z 12 240-259 42 42 42 78 78 78 F F F
49462024-02-28T22:15:07.871Z 13 260-279 42 42 42 78 78 78 F F F
49472024-02-28T22:15:07.871Z 14 280-299 45 45 45 83 83 83 F F F
49482024-02-28T22:15:07.871Z 15 300-319 44 44 44 80 80 80 F F F
49492024-02-28T22:15:07.871Z 16 320-339 37 37 37 72 72 72 F F F
49502024-02-28T22:15:07.871Z 17 340-359 41 41 41 77 77 77 F F F
49512024-02-28T22:15:07.871Z 18 360-379 44 44 44 81 81 81 F F F
49522024-02-28T22:15:07.871Z 19 380-399 42 42 42 78 78 78 F F F
49532024-02-28T22:15:07.871Z 20 400-419 44 44 44 82 82 82 F F F
49542024-02-28T22:15:07.871Z 21 420-439 44 44 44 82 82 82 F F F
49552024-02-28T22:15:07.871Z 22 440-459 38 38 38 73 73 73 F F F
49562024-02-28T22:15:07.871Z 23 460-479 38 38 38 73 73 73 F F F
49572024-02-28T22:15:07.871Z 24 480-499 40 40 40 76 76 76 F F F
49582024-02-28T22:15:07.871Z 25 500-519 44 44 44 81 81 81 F F F
49592024-02-28T22:15:07.871Z 26 520-539 44 44 44 80 80 80 F F F
49602024-02-28T22:15:07.871Z 27 540-559 40 40 40 75 75 75 F F F
49612024-02-28T22:15:07.872Z 28 560-579 44 44 44 82 82 82 F F F
49622024-02-28T22:15:07.872Z 29 580-599 44 44 44 82 82 82 F F F
49632024-02-28T22:15:07.872ZMax gen: 45, Max flush: 83
49642024-02-28T22:15:07.872Z
49652024-02-28T22:15:07.872ZBegin loop 23
49662024-02-28T22:15:07.872ZDownstairs to restart: 2
49672024-02-28T22:15:07.890Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.78658369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
49682024-02-28T22:15:07.890Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.786876372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
49692024-02-28T22:15:07.890Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.786899322Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
49702024-02-28T22:15:07.896Z{"msg":"Crucible f1388655-ccf0-40e8-ac10-357e644fd01e has session id: 7d3a4e22-77e3-4866-9326-652aacc55515","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.786931191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49712024-02-28T22:15:07.896ZCrucible runtime is spawned
49722024-02-28T22:15:07.896Z{"msg":"Upstairs opts: Upstairs UUID: f1388655-ccf0-40e8-ac10-357e644fd01e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.786976899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49732024-02-28T22:15:07.896Z{"msg":"Crucible stats registered with UUID: f1388655-ccf0-40e8-ac10-357e644fd01e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787015489Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49742024-02-28T22:15:07.896Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787040618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
49752024-02-28T22:15:07.896Z{"msg":"f1388655-ccf0-40e8-ac10-357e644fd01e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787209943Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49762024-02-28T22:15:07.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787241372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49772024-02-28T22:15:07.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787267171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49782024-02-28T22:15:07.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787294741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49792024-02-28T22:15:07.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.78731902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49802024-02-28T22:15:07.896Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787352699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49812024-02-28T22:15:07.896Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787378919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49822024-02-28T22:15:07.896Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787402068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49832024-02-28T22:15:07.896Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787491855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49842024-02-28T22:15:07.896Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787564473Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49852024-02-28T22:15:07.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787609982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49862024-02-28T22:15:07.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787704419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49872024-02-28T22:15:07.896Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.787760598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49882024-02-28T22:15:07.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.788362791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49892024-02-28T22:15:07.896Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.7884014Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49902024-02-28T22:15:07.896Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.78842505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49912024-02-28T22:15:07.897Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.788447139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49922024-02-28T22:15:07.897Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.788461729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49932024-02-28T22:15:07.897Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.788475658Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49942024-02-28T22:15:07.897Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789091831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49952024-02-28T22:15:07.897Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.78915441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49962024-02-28T22:15:07.897Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789218418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49972024-02-28T22:15:07.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789642756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49982024-02-28T22:15:07.897Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789682555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
49992024-02-28T22:15:07.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789701535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50002024-02-28T22:15:07.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789719784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50012024-02-28T22:15:07.897Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789744953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50022024-02-28T22:15:07.897Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789769443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50032024-02-28T22:15:07.897Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789799602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50042024-02-28T22:15:07.897Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789817661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50052024-02-28T22:15:07.897Z{"msg":"[0]R flush_numbers[0..12]: [82, 75, 75, 83, 83, 80, 83, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789847831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50062024-02-28T22:15:07.897Z{"msg":"[0]R generation[0..12]: [44, 40, 40, 45, 45, 44, 45, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.78986684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50072024-02-28T22:15:07.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789898339Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50082024-02-28T22:15:07.897Z{"msg":"[1]R flush_numbers[0..12]: [82, 75, 75, 83, 83, 80, 83, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789917938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50092024-02-28T22:15:07.897ZWait for a query_work_queue command to finish before sending IO
50102024-02-28T22:15:07.897Z{"msg":"[1]R generation[0..12]: [44, 40, 40, 45, 45, 44, 45, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789952917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50112024-02-28T22:15:07.897Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.789986147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50122024-02-28T22:15:07.897Z{"msg":"[2]R flush_numbers[0..12]: [82, 75, 75, 83, 83, 80, 83, 68, 79, 78, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790019556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50132024-02-28T22:15:07.897Z{"msg":"[2]R generation[0..12]: [44, 40, 40, 45, 45, 44, 45, 36, 43, 42, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790051985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50142024-02-28T22:15:07.898Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790075454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50152024-02-28T22:15:07.898Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790103244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50162024-02-28T22:15:07.898Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790117593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50172024-02-28T22:15:07.898Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790135732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50182024-02-28T22:15:07.898Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790163582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"":"downstairs","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50192024-02-28T22:15:07.898Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790179261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50202024-02-28T22:15:07.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.79020851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"0","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50212024-02-28T22:15:07.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.79022958Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"1","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50222024-02-28T22:15:07.898Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790257679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"client":"2","":"downstairs","session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50232024-02-28T22:15:07.898Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790274759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50242024-02-28T22:15:07.898Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790302568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50252024-02-28T22:15:07.898Z{"msg":"f1388655-ccf0-40e8-ac10-357e644fd01e is now active with session: 7d3a4e22-77e3-4866-9326-652aacc55515","v":0,"name":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50262024-02-28T22:15:07.898Z"crucible","level":30,"time":"2024-02-28T22:15:06.790322967Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50272024-02-28T22:15:07.898Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790350427Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319,"session_id":"7d3a4e22-77e3-4866-9326-652aacc55515"}
50282024-02-28T22:15:07.898Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:06.790365246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
50292024-02-28T22:15:07.898ZLoading write count information from file "/tmp/test_repair_verify.data"
50302024-02-28T22:15:07.898ZRun Repair workload
50312024-02-28T22:15:07.902Z01/30 Write block 51 len 4096 data: 4
50322024-02-28T22:15:07.902Z02/30 Read block 461 len 4096
50332024-02-28T22:15:07.902Z03/30 Write block 170 len 40960 data: 7 6 6 6 6 5 4 3 3 3
50342024-02-28T22:15:07.902Z04/30 Flush
50352024-02-28T22:15:07.902Z05/30 Flush
50362024-02-28T22:15:07.902Z06/30 Write block 555 len 12288 data: 4 4 4
50372024-02-28T22:15:07.902Z07/30 Write block 55 len 32768 data: 3 5 4 4 4 4 5 6
50382024-02-28T22:15:07.902Z08/30 Read block 433 len 16384
50392024-02-28T22:15:07.907Z09/30 Write block 414 len 12288 data: 5 5 5
50402024-02-28T22:15:07.907Z10/30 Read block 476 len 36864
50412024-02-28T22:15:07.907Z11/30 Write block 257 len 24576 data: 7 8 7 5 5 5
50422024-02-28T22:15:07.907Z12/30 Read block 155 len 16384
50432024-02-28T22:15:07.910Z13/30 Write block 192 len 28672 data: 6 5 4 4 3 3 3
50442024-02-28T22:15:07.910Z14/30 Read block 235 len 16384
50452024-02-28T22:15:07.910Z15/30 Read block 474 len 28672
50462024-02-28T22:15:07.910Z16/30 Write block 158 len 32768 data: 5 5 3 4 4 4 5 6
50472024-02-28T22:15:07.911Z17/30 Read block 506 len 36864
50482024-02-28T22:15:07.917Z18/30 Write block 351 len 16384 data: 4 5 6 7
50492024-02-28T22:15:07.917Z19/30 Flush
50502024-02-28T22:15:07.920Z20/30 Write block 588 len 4096 data: 3
50512024-02-28T22:15:07.920Z21/30 Write block 542 len 4096 data: 8
50522024-02-28T22:15:07.920Z22/30 Read block 575 len 32768
50532024-02-28T22:15:07.920Z23/30 Read block 440 len 20480
50542024-02-28T22:15:07.920Z24/30 Read block 464 len 12288
50552024-02-28T22:15:07.920Z25/30 Read block 34 len 16384
50562024-02-28T22:15:07.924Z26/30 Write block 47 len 20480 data: 4 4 5 4 5
50572024-02-28T22:15:07.924Z27/30 Read block 245 len 20480
50582024-02-28T22:15:07.924Z28/30 Read block 288 len 4096
50592024-02-28T22:15:07.924Z29/30 Write block 61 len 12288 data: 6 7 7
50602024-02-28T22:15:07.924Z30/30 Write block 520 len 16384 data: 6 5 4 4
50612024-02-28T22:15:07.928Z----------------------------------------------------------------
50622024-02-28T22:15:07.928Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:26
50632024-02-28T22:15:07.928ZStates: Active Active Active
50642024-02-28T22:15:07.928ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50652024-02-28T22:15:07.928Z 5 Acked 1004 Flush 0 Done Done Sent false
50662024-02-28T22:15:07.928Z 6 Acked 1005 Write 3 Done Done Sent false
50672024-02-28T22:15:07.928Z 7 Acked 1006 Write 8 Done Done Sent false
50682024-02-28T22:15:07.928Z 8 Acked 1007 Read 4 Done Done Sent false
50692024-02-28T22:15:07.928Z 9 Acked 1008 Write 3 Done Done Sent false
50702024-02-28T22:15:07.928Z 10 Acked 1009 Read 9 Done Done Sent false
50712024-02-28T22:15:07.928Z 11 Acked 1010 Write 6 Done Done Sent false
50722024-02-28T22:15:07.928Z 12 Acked 1011 Read 4 Done Done Sent false
50732024-02-28T22:15:07.928Z 13 Acked 1012 Write 7 Done Done Sent false
50742024-02-28T22:15:07.928Z 14 Acked 1013 Read 4 Done Done Sent false
50752024-02-28T22:15:07.928Z 15 Acked 1014 Read 7 Done Done Sent false
50762024-02-28T22:15:07.928Z 16 Acked 1015 Write 8 Done Done Sent false
50772024-02-28T22:15:07.929Z 17 Acked 1016 Read 9 Done Done Sent false
50782024-02-28T22:15:07.929Z 18 Acked 1017 Write 4 Done Done Sent false
50792024-02-28T22:15:07.929Z 19 Acked 1018 Flush 0 Done Done Sent false
50802024-02-28T22:15:07.929Z 20 Acked 1019 Write 1 Done Done Sent false
50812024-02-28T22:15:07.929Z 21 Acked 1020 Write 1 Done Done Sent false
50822024-02-28T22:15:07.929Z 22 Acked 1021 Read 8 Done Done Sent false
50832024-02-28T22:15:07.929Z 23 Acked 1022 Read 5 Done Done Sent false
50842024-02-28T22:15:07.929Z 24 Acked 1023 Read 3 Done Done Sent false
50852024-02-28T22:15:07.929Z 25 Acked 1024 Read 4 Done Done Sent false
50862024-02-28T22:15:07.929Z 26 Acked 1025 Write 5 Done Done Sent false
50872024-02-28T22:15:07.929Z 27 Acked 1026 Read 5 Done Done Sent false
50882024-02-28T22:15:07.929Z 28 Acked 1027 Read 1 Done Done Sent false
50892024-02-28T22:15:07.929Z 29 Acked 1028 Write 3 Done Done Sent false
50902024-02-28T22:15:07.929Z 30 Acked 1029 Write 4 Sent Sent Sent false
50912024-02-28T22:15:07.929Z STATES DS:0 DS:1 DS:2 TOTAL
50922024-02-28T22:15:07.929Z New 0 0 0 0
50932024-02-28T22:15:07.929Z Sent 1 1 26 28
50942024-02-28T22:15:07.929Z Done 25 25 0 50
50952024-02-28T22:15:07.929Z Skipped 0 0 0 0
50962024-02-28T22:15:07.929Z Error 0 0 0 0
50972024-02-28T22:15:07.929ZLast Flush: 1018 1018 1003
50982024-02-28T22:15:07.929ZDownstairs last five completed: 1003 1002 1001 1000
50992024-02-28T22:15:07.930ZUpstairs last five completed: 30 29 28 27 26
51002024-02-28T22:15:07.930Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:06.82144966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1319}
51012024-02-28T22:15:07.930ZWrote out file "/tmp/test_repair_verify.data"
51022024-02-28T22:15:07.933Z
51032024-02-28T22:15:07.934ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51042024-02-28T22:15:07.951ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51052024-02-28T22:15:07.951Z 0 000-019 44 44 44 82 82 82 F F F
51062024-02-28T22:15:07.951Z 1 020-039 40 40 40 75 75 75 F F F
51072024-02-28T22:15:07.952Z 2 040-059 46 46 46 86 86 84 T T F <---
51082024-02-28T22:15:07.952Z 3 060-079 46 46 45 86 86 83 T T F <---
51092024-02-28T22:15:07.952Z 4 080-099 45 45 45 83 83 83 F F F
51102024-02-28T22:15:07.952Z 5 100-119 44 44 44 80 80 80 F F F
51112024-02-28T22:15:07.952Z 6 120-139 45 45 45 83 83 83 F F F
51122024-02-28T22:15:07.952Z 7 140-159 46 46 36 86 86 68 F F F <---
51132024-02-28T22:15:07.973Z 8 160-179 46 46 46 86 86 84 F F F <---
51142024-02-28T22:15:07.973Z 9 180-199 46 46 42 86 86 78 F F F <---
51152024-02-28T22:15:07.973Z 10 200-219 35 35 35 67 67 67 F F F
51162024-02-28T22:15:07.973Z 11 220-239 35 35 35 67 67 67 F F F
51172024-02-28T22:15:07.973Z 12 240-259 46 46 42 86 86 78 F F F <---
51182024-02-28T22:15:07.973Z 13 260-279 46 46 42 86 86 78 F F F <---
51192024-02-28T22:15:07.973Z 14 280-299 45 45 45 83 83 83 F F F
51202024-02-28T22:15:07.973Z 15 300-319 44 44 44 80 80 80 F F F
51212024-02-28T22:15:07.973Z 16 320-339 37 37 37 72 72 72 F F F
51222024-02-28T22:15:07.973Z 17 340-359 46 46 41 86 86 77 F F F <---
51232024-02-28T22:15:07.973Z 18 360-379 44 44 44 81 81 81 F F F
51242024-02-28T22:15:07.973Z 19 380-399 42 42 42 78 78 78 F F F
51252024-02-28T22:15:07.973Z 20 400-419 46 46 44 86 86 82 F F F <---
51262024-02-28T22:15:07.973Z 21 420-439 44 44 44 82 82 82 F F F
51272024-02-28T22:15:07.973Z 22 440-459 38 38 38 73 73 73 F F F
51282024-02-28T22:15:07.974Z 23 460-479 38 38 38 73 73 73 F F F
51292024-02-28T22:15:07.974Z 24 480-499 40 40 40 76 76 76 F F F
51302024-02-28T22:15:07.974Z 25 500-519 44 44 44 81 81 81 F F F
51312024-02-28T22:15:07.974Z 26 520-539 44 44 44 80 80 80 T T F <---
51322024-02-28T22:15:07.974Z 27 540-559 46 46 40 86 86 75 T T F <---
51332024-02-28T22:15:07.974Z 28 560-579 44 44 44 82 82 82 F F F
51342024-02-28T22:15:07.974Z 29 580-599 44 44 44 82 82 82 T T F <---
51352024-02-28T22:15:07.974ZMax gen: 46, Max flush: 86
51362024-02-28T22:15:07.974ZError: Difference in extent metadata found!
51372024-02-28T22:15:07.974ZA Difference in extent metadata is expected here
51382024-02-28T22:15:07.974ZOn loop 23
51392024-02-28T22:15:07.974Z
51402024-02-28T22:15:07.974ZVerifying data now
51412024-02-28T22:15:12.591ZNo change after verify
51422024-02-28T22:15:12.591ZLoop: 23 Downstairs dump after verify (and repair):
51432024-02-28T22:15:12.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51442024-02-28T22:15:12.605Z 0 000-019 44 44 44 82 82 82 F F F
51452024-02-28T22:15:12.605Z 1 020-039 40 40 40 75 75 75 F F F
51462024-02-28T22:15:12.605Z 2 040-059 47 47 47 87 87 87 F F F
51472024-02-28T22:15:12.606Z 3 060-079 47 47 47 87 87 87 F F F
51482024-02-28T22:15:12.606Z 4 080-099 45 45 45 83 83 83 F F F
51492024-02-28T22:15:12.606Z 5 100-119 44 44 44 80 80 80 F F F
51502024-02-28T22:15:12.606Z 6 120-139 45 45 45 83 83 83 F F F
51512024-02-28T22:15:12.606Z 7 140-159 46 46 46 86 86 86 F F F
51522024-02-28T22:15:12.606Z 8 160-179 46 46 46 86 86 86 F F F
51532024-02-28T22:15:12.606Z 9 180-199 46 46 46 86 86 86 F F F
51542024-02-28T22:15:12.609Z 10 200-219 35 35 35 67 67 67 F F F
51552024-02-28T22:15:12.609Z 11 220-239 35 35 35 67 67 67 F F F
51562024-02-28T22:15:12.609Z 12 240-259 46 46 46 86 86 86 F F F
51572024-02-28T22:15:12.609Z 13 260-279 46 46 46 86 86 86 F F F
51582024-02-28T22:15:12.609Z 14 280-299 45 45 45 83 83 83 F F F
51592024-02-28T22:15:12.610Z 15 300-319 44 44 44 80 80 80 F F F
51602024-02-28T22:15:12.610Z 16 320-339 37 37 37 72 72 72 F F F
51612024-02-28T22:15:12.610Z 17 340-359 46 46 46 86 86 86 F F F
51622024-02-28T22:15:12.610Z 18 360-379 44 44 44 81 81 81 F F F
51632024-02-28T22:15:12.610Z 19 380-399 42 42 42 78 78 78 F F F
51642024-02-28T22:15:12.610Z 20 400-419 46 46 46 86 86 86 F F F
51652024-02-28T22:15:12.610Z 21 420-439 44 44 44 82 82 82 F F F
51662024-02-28T22:15:12.610Z 22 440-459 38 38 38 73 73 73 F F F
51672024-02-28T22:15:12.610Z 23 460-479 38 38 38 73 73 73 F F F
51682024-02-28T22:15:12.610Z 24 480-499 40 40 40 76 76 76 F F F
51692024-02-28T22:15:12.610Z 25 500-519 44 44 44 81 81 81 F F F
51702024-02-28T22:15:12.610Z 26 520-539 47 47 47 87 87 87 F F F
51712024-02-28T22:15:12.610Z 27 540-559 47 47 47 87 87 87 F F F
51722024-02-28T22:15:12.610Z 28 560-579 44 44 44 82 82 82 F F F
51732024-02-28T22:15:12.610Z 29 580-599 47 47 47 87 87 87 F F F
51742024-02-28T22:15:12.610ZMax gen: 47, Max flush: 87
51752024-02-28T22:15:12.610Z
51762024-02-28T22:15:12.610ZBegin loop 24
51772024-02-28T22:15:12.610ZDownstairs to restart: 2
51782024-02-28T22:15:12.628Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.524565994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
51792024-02-28T22:15:12.645Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.524862806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
51802024-02-28T22:15:12.645ZCrucible runtime is spawned
51812024-02-28T22:15:12.645Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.524938204Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
51822024-02-28T22:15:12.645Z{"msg":"Crucible 01c17647-3be5-4889-8671-eeee5bbbaa2e has session id: a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.524984213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51832024-02-28T22:15:12.645Z{"msg":"Upstairs opts: Upstairs UUID: 01c17647-3be5-4889-8671-eeee5bbbaa2e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525042011Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51842024-02-28T22:15:12.645Z{"msg":"Crucible stats registered with UUID: 01c17647-3be5-4889-8671-eeee5bbbaa2e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.52509059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51852024-02-28T22:15:12.645Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525139059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
51862024-02-28T22:15:12.645Z{"msg":"01c17647-3be5-4889-8671-eeee5bbbaa2e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525187997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51872024-02-28T22:15:12.645Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525216556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51882024-02-28T22:15:12.646Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525244545Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51892024-02-28T22:15:12.646Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525272385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51902024-02-28T22:15:12.646Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525315084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51912024-02-28T22:15:12.646Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525367882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51922024-02-28T22:15:12.646Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525414201Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51932024-02-28T22:15:12.646Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.52546187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51942024-02-28T22:15:12.646Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525516008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51952024-02-28T22:15:12.646Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525585346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51962024-02-28T22:15:12.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525642975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51972024-02-28T22:15:12.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525674274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51982024-02-28T22:15:12.646Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.525714193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
51992024-02-28T22:15:12.646Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526377444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52002024-02-28T22:15:12.646Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526416513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52012024-02-28T22:15:12.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526433533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52022024-02-28T22:15:12.647Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526447772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52032024-02-28T22:15:12.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526461632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52042024-02-28T22:15:12.647Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.526488691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52052024-02-28T22:15:12.647Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527071885Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52062024-02-28T22:15:12.647Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527121764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52072024-02-28T22:15:12.647Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527165233Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52082024-02-28T22:15:12.647Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527692198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52092024-02-28T22:15:12.647Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527748067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52102024-02-28T22:15:12.647Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527776696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52112024-02-28T22:15:12.647Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527805605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52122024-02-28T22:15:12.647Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527830254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52132024-02-28T22:15:12.647Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527858304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52142024-02-28T22:15:12.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527880483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52152024-02-28T22:15:12.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527908362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52162024-02-28T22:15:12.648Z{"msg":"[0]R flush_numbers[0..12]: [82, 75, 87, 87, 83, 80, 83, 86, 86, 86, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.527933582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52172024-02-28T22:15:12.648Z{"msg":"Wait for a query_work_queue command to finish before sending IO
52182024-02-28T22:15:12.648Z[0]R generation[0..12]: [44, 40, 47, 47, 45, 44, 45, 46, 46, 46, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.52796813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52192024-02-28T22:15:12.648Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.52799454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52202024-02-28T22:15:12.648Z{"msg":"[1]R flush_numbers[0..12]: [82, 75, 87, 87, 83, 80, 83, 86, 86, 86, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528019499Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52212024-02-28T22:15:12.648Z{"msg":"[1]R generation[0..12]: [44, 40, 47, 47, 45, 44, 45, 46, 46, 46, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528034819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52222024-02-28T22:15:12.648Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528049818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52232024-02-28T22:15:12.648Z{"msg":"[2]R flush_numbers[0..12]: [82, 75, 87, 87, 83, 80, 83, 86, 86, 86, 67, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528071468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52242024-02-28T22:15:12.648Z{"msg":"[2]R generation[0..12]: [44, 40, 47, 47, 45, 44, 45, 46, 46, 46, 35, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528086027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52252024-02-28T22:15:12.648Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528106887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52262024-02-28T22:15:12.649Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528120716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52272024-02-28T22:15:12.649Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528135156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52282024-02-28T22:15:12.649Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528155085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52292024-02-28T22:15:12.649Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528168285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"":"downstairs","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52302024-02-28T22:15:12.649Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528188555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52312024-02-28T22:15:12.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528201024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"0","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52322024-02-28T22:15:12.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528221354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"1","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52332024-02-28T22:15:12.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528234973Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"client":"2","":"downstairs","session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52342024-02-28T22:15:12.649Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528254913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52352024-02-28T22:15:12.649Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528267932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52362024-02-28T22:15:12.649Z{"msg":"01c17647-3be5-4889-8671-eeee5bbbaa2e is now active with session: a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528287972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52372024-02-28T22:15:12.649Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528300551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327,"session_id":"a4a5ac7c-3eb8-4f1a-b5ba-420bff7c5186"}
52382024-02-28T22:15:12.649Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:11.528313641Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
52392024-02-28T22:15:12.650ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52402024-02-28T22:15:12.654ZLoading write count information from file "/tmp/test_repair_verify.data"
52412024-02-28T22:15:12.654ZRun Repair workload
52422024-02-28T22:15:12.654Z01/30 Write block 391 len 8192 data: 6 5
52432024-02-28T22:15:12.654Z02/30 Flush
52442024-02-28T22:15:12.654Z03/30 Write block 403 len 28672 data: 5 5 5 5 6 6 5
52452024-02-28T22:15:12.654Z04/30 Read block 82 len 36864
52462024-02-28T22:15:12.654Z05/30 Flush
52472024-02-28T22:15:12.654Z06/30 Flush
52482024-02-28T22:15:12.654Z07/30 Write block 548 len 4096 data: 5
52492024-02-28T22:15:12.654Z08/30 Read block 49 len 28672
52502024-02-28T22:15:12.654Z09/30 Write block 400 len 20480 data: 4 4 4 6 6
52512024-02-28T22:15:12.654Z10/30 Read block 524 len 28672
52522024-02-28T22:15:12.654Z11/30 Read block 110 len 24576
52532024-02-28T22:15:12.654Z12/30 Write block 82 len 36864 data: 6 5 5 5 5 5 5 5 6
52542024-02-28T22:15:12.654Z13/30 Read block 205 len 32768
52552024-02-28T22:15:12.654Z14/30 Write block 211 len 4096 data: 4
52562024-02-28T22:15:12.655Z15/30 Read block 59 len 32768
52572024-02-28T22:15:12.655Z16/30 Read block 322 len 40960
52582024-02-28T22:15:12.655Z17/30 Read block 66 len 28672
52592024-02-28T22:15:12.655Z18/30 Read block 385 len 4096
52602024-02-28T22:15:12.655Z19/30 Read block 53 len 24576
52612024-02-28T22:15:12.655Z20/30 Flush
52622024-02-28T22:15:12.658Z21/30 Write block 29 len 8192 data: 4 5
52632024-02-28T22:15:12.658Z22/30 Read block 267 len 24576
52642024-02-28T22:15:12.658Z23/30 Read block 167 len 20480
52652024-02-28T22:15:12.658Z24/30 Read block 497 len 16384
52662024-02-28T22:15:12.658Z25/30 Read block 0 len 4096
52672024-02-28T22:15:12.658Z26/30 Write block 72 len 20480 data: 4 3 3 3 3
52682024-02-28T22:15:12.663Z27/30 Write block 139 len 32768 data: 6 5 5 5 6 6 7 8
52692024-02-28T22:15:12.663Z28/30 Read block 335 len 4096
52702024-02-28T22:15:12.663Z29/30 Read block 524 len 40960
52712024-02-28T22:15:12.663Z30/30 Read block 414 len 28672
52722024-02-28T22:15:12.663Z----------------------------------------------------------------
52732024-02-28T22:15:12.663Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:28
52742024-02-28T22:15:12.663ZStates: Active Active Active
52752024-02-28T22:15:12.663ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52762024-02-28T22:15:12.663Z 3 Acked 1002 Write 7 Done Done Sent false
52772024-02-28T22:15:12.663Z 4 Acked 1003 Read 9 Done Done Sent false
52782024-02-28T22:15:12.663Z 5 Acked 1004 Flush 0 Done Done Sent false
52792024-02-28T22:15:12.663Z 6 Acked 1005 Flush 0 Done Done Sent false
52802024-02-28T22:15:12.663Z 7 Acked 1006 Write 1 Done Done Sent false
52812024-02-28T22:15:12.663Z 8 Acked 1007 Read 7 Done Done Sent false
52822024-02-28T22:15:12.664Z 9 Acked 1008 Write 5 Done Done Sent false
52832024-02-28T22:15:12.664Z 10 Acked 1009 Read 7 Done Done Sent false
52842024-02-28T22:15:12.664Z 11 Acked 1010 Read 6 Done Done Sent false
52852024-02-28T22:15:12.664Z 12 Acked 1011 Write 9 Done Done Sent false
52862024-02-28T22:15:12.664Z 13 Acked 1012 Read 8 Done Done Sent false
52872024-02-28T22:15:12.664Z 14 Acked 1013 Write 1 Done Done Sent false
52882024-02-28T22:15:12.664Z 15 Acked 1014 Read 8 Done Done Sent false
52892024-02-28T22:15:12.664Z 16 Acked 1015 Read 10 Done Done Sent false
52902024-02-28T22:15:12.664Z 17 Acked 1016 Read 7 Done Done Sent false
52912024-02-28T22:15:12.664Z 18 Acked 1017 Read 1 Done Done Sent false
52922024-02-28T22:15:12.664Z 19 Acked 1018 Read 6 Done Done Sent false
52932024-02-28T22:15:12.664Z 20 Acked 1019 Flush 0 Done Done Sent false
52942024-02-28T22:15:12.664Z 21 Acked 1020 Write 2 Done Done Sent false
52952024-02-28T22:15:12.664Z 22 Acked 1021 Read 6 Done Done Sent false
52962024-02-28T22:15:12.664Z 23 Acked 1022 Read 5 Done Done Sent false
52972024-02-28T22:15:12.664Z 24 Acked 1023 Read 4 Done Done Sent false
52982024-02-28T22:15:12.664Z 25 Acked 1024 Read 1 Done Done Sent false
52992024-02-28T22:15:12.664Z 26 Acked 1025 Write 5 Done Done Sent false
53002024-02-28T22:15:12.664Z 27 Acked 1026 Write 8 Done Done Sent false
53012024-02-28T22:15:12.664Z 28 Acked 1027 Read 1 Done Done Sent false
53022024-02-28T22:15:12.665Z 29 Acked 1028 Read 10 Done Done Sent false
53032024-02-28T22:15:12.665Z 30 Acked 1029 Read 7 Done Done Sent false
53042024-02-28T22:15:12.665Z STATES DS:0 DS:1 DS:2 TOTAL
53052024-02-28T22:15:12.665Z New 0 0 0 0
53062024-02-28T22:15:12.665Z Sent 0 0 28 28
53072024-02-28T22:15:12.665Z Done 28 28 0 56
53082024-02-28T22:15:12.665Z Skipped 0 0 0 0
53092024-02-28T22:15:12.665Z Error 0 0 0 0
53102024-02-28T22:15:12.665ZLast Flush: 1019 1019 1001
53112024-02-28T22:15:12.665ZDownstairs last five completed: 1001 1000
53122024-02-28T22:15:12.665ZUpstairs last five completed: 30 29 28 27 26
53132024-02-28T22:15:12.665Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:11.55883431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1327}
53142024-02-28T22:15:12.665ZWrote out file "/tmp/test_repair_verify.data"
53152024-02-28T22:15:12.668Z
53162024-02-28T22:15:12.672ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53172024-02-28T22:15:12.704ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53182024-02-28T22:15:12.704Z 0 000-019 44 44 44 82 82 82 F F F
53192024-02-28T22:15:12.704Z 1 020-039 40 40 40 75 75 75 T T F <---
53202024-02-28T22:15:12.704Z 2 040-059 47 47 47 87 87 87 F F F
53212024-02-28T22:15:12.704Z 3 060-079 47 47 47 87 87 87 T T F <---
53222024-02-28T22:15:12.705Z 4 080-099 48 48 45 91 91 83 F F F <---
53232024-02-28T22:15:12.705Z 5 100-119 44 44 44 80 80 80 F F F
53242024-02-28T22:15:12.705Z 6 120-139 45 45 45 83 83 83 T T F <---
53252024-02-28T22:15:12.705Z 7 140-159 46 46 46 86 86 86 T T F <---
53262024-02-28T22:15:12.705Z 8 160-179 46 46 46 86 86 86 F F F
53272024-02-28T22:15:12.712Z 9 180-199 46 46 46 86 86 86 F F F
53282024-02-28T22:15:12.713Z 10 200-219 48 48 35 91 91 67 F F F <---
53292024-02-28T22:15:12.713Z 11 220-239 35 35 35 67 67 67 F F F
53302024-02-28T22:15:12.713Z 12 240-259 46 46 46 86 86 86 F F F
53312024-02-28T22:15:12.713Z 13 260-279 46 46 46 86 86 86 F F F
53322024-02-28T22:15:12.713Z 14 280-299 45 45 45 83 83 83 F F F
53332024-02-28T22:15:12.713Z 15 300-319 44 44 44 80 80 80 F F F
53342024-02-28T22:15:12.713Z 16 320-339 37 37 37 72 72 72 F F F
53352024-02-28T22:15:12.713Z 17 340-359 46 46 46 86 86 86 F F F
53362024-02-28T22:15:12.713Z 18 360-379 44 44 44 81 81 81 F F F
53372024-02-28T22:15:12.713Z 19 380-399 48 48 48 88 88 88 F F F
53382024-02-28T22:15:12.713Z 20 400-419 48 48 46 91 91 86 F F F <---
53392024-02-28T22:15:12.713Z 21 420-439 44 44 44 82 82 82 F F F
53402024-02-28T22:15:12.714Z 22 440-459 38 38 38 73 73 73 F F F
53412024-02-28T22:15:12.714Z 23 460-479 38 38 38 73 73 73 F F F
53422024-02-28T22:15:12.714Z 24 480-499 40 40 40 76 76 76 F F F
53432024-02-28T22:15:12.714Z 25 500-519 44 44 44 81 81 81 F F F
53442024-02-28T22:15:12.714Z 26 520-539 47 47 47 87 87 87 F F F
53452024-02-28T22:15:12.714Z 27 540-559 48 48 47 91 91 87 F F F <---
53462024-02-28T22:15:12.714Z 28 560-579 44 44 44 82 82 82 F F F
53472024-02-28T22:15:12.714Z 29 580-599 47 47 47 87 87 87 F F F
53482024-02-28T22:15:12.714ZMax gen: 48, Max flush: 91
53492024-02-28T22:15:12.714ZError: Difference in extent metadata found!
53502024-02-28T22:15:12.714ZA Difference in extent metadata is expected here
53512024-02-28T22:15:12.714ZOn loop 24
53522024-02-28T22:15:12.714Z
53532024-02-28T22:15:12.714ZVerifying data now
53542024-02-28T22:15:17.138ZNo change after verify
53552024-02-28T22:15:17.138ZLoop: 24 Downstairs dump after verify (and repair):
53562024-02-28T22:15:17.153ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53572024-02-28T22:15:17.153Z 0 000-019 44 44 44 82 82 82 F F F
53582024-02-28T22:15:17.153Z 1 020-039 49 49 49 92 92 92 F F F
53592024-02-28T22:15:17.153Z 2 040-059 47 47 47 87 87 87 F F F
53602024-02-28T22:15:17.153Z 3 060-079 49 49 49 92 92 92 F F F
53612024-02-28T22:15:17.153Z 4 080-099 48 48 48 91 91 91 F F F
53622024-02-28T22:15:17.153Z 5 100-119 44 44 44 80 80 80 F F F
53632024-02-28T22:15:17.153Z 6 120-139 49 49 49 92 92 92 F F F
53642024-02-28T22:15:17.153Z 7 140-159 49 49 49 92 92 92 F F F
53652024-02-28T22:15:17.153Z 8 160-179 46 46 46 86 86 86 F F F
53662024-02-28T22:15:17.153Z 9 180-199 46 46 46 86 86 86 F F F
53672024-02-28T22:15:17.153Z 10 200-219 48 48 48 91 91 91 F F F
53682024-02-28T22:15:17.153Z 11 220-239 35 35 35 67 67 67 F F F
53692024-02-28T22:15:17.153Z 12 240-259 46 46 46 86 86 86 F F F
53702024-02-28T22:15:17.153Z 13 260-279 46 46 46 86 86 86 F F F
53712024-02-28T22:15:17.153Z 14 280-299 45 45 45 83 83 83 F F F
53722024-02-28T22:15:17.153Z 15 300-319 44 44 44 80 80 80 F F F
53732024-02-28T22:15:17.153Z 16 320-339 37 37 37 72 72 72 F F F
53742024-02-28T22:15:17.153Z 17 340-359 46 46 46 86 86 86 F F F
53752024-02-28T22:15:17.153Z 18 360-379 44 44 44 81 81 81 F F F
53762024-02-28T22:15:17.156Z 19 380-399 48 48 48 88 88 88 F F F
53772024-02-28T22:15:17.156Z 20 400-419 48 48 48 91 91 91 F F F
53782024-02-28T22:15:17.157Z 21 420-439 44 44 44 82 82 82 F F F
53792024-02-28T22:15:17.157Z 22 440-459 38 38 38 73 73 73 F F F
53802024-02-28T22:15:17.157Z 23 460-479 38 38 38 73 73 73 F F F
53812024-02-28T22:15:17.157Z 24 480-499 40 40 40 76 76 76 F F F
53822024-02-28T22:15:17.157Z 25 500-519 44 44 44 81 81 81 F F F
53832024-02-28T22:15:17.157Z 26 520-539 47 47 47 87 87 87 F F F
53842024-02-28T22:15:17.157Z 27 540-559 48 48 48 91 91 91 F F F
53852024-02-28T22:15:17.157Z 28 560-579 44 44 44 82 82 82 F F F
53862024-02-28T22:15:17.157Z 29 580-599 47 47 47 87 87 87 F F F
53872024-02-28T22:15:17.157ZMax gen: 49, Max flush: 92
53882024-02-28T22:15:17.157Z
53892024-02-28T22:15:17.157ZBegin loop 25
53902024-02-28T22:15:17.157ZDownstairs to restart: 2
53912024-02-28T22:15:17.175Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071209179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
53922024-02-28T22:15:17.175Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071452412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
53932024-02-28T22:15:17.175Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071475831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
53942024-02-28T22:15:17.180Z{"msg":"Crucible 7f65ab8d-ab9d-4ecb-b247-5a64695b87da has session id: 869137dc-604b-48df-9e85-fa894064dd1c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.07150292Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
53952024-02-28T22:15:17.180ZCrucible runtime is spawned
53962024-02-28T22:15:17.180Z{"msg":"Upstairs opts: Upstairs UUID: 7f65ab8d-ab9d-4ecb-b247-5a64695b87da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071561829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
53972024-02-28T22:15:17.180Z{"msg":"Crucible stats registered with UUID: 7f65ab8d-ab9d-4ecb-b247-5a64695b87da","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071601098Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
53982024-02-28T22:15:17.181Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071626277Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
53992024-02-28T22:15:17.181Z{"msg":"7f65ab8d-ab9d-4ecb-b247-5a64695b87da active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071743724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54002024-02-28T22:15:17.181Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071786023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54012024-02-28T22:15:17.181Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071803952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54022024-02-28T22:15:17.181Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071836341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54032024-02-28T22:15:17.181Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.07185875Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54042024-02-28T22:15:17.181Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071900439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54052024-02-28T22:15:17.181Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071927559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54062024-02-28T22:15:17.181Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071951578Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54072024-02-28T22:15:17.181Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.071982687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54082024-02-28T22:15:17.181Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072009036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54092024-02-28T22:15:17.182Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072034936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54102024-02-28T22:15:17.182Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072137313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54112024-02-28T22:15:17.182Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072195911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54122024-02-28T22:15:17.182Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072778175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54132024-02-28T22:15:17.182Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072823484Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54142024-02-28T22:15:17.182Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.072903122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54152024-02-28T22:15:17.182Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.07296786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54162024-02-28T22:15:17.182Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073291141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54172024-02-28T22:15:17.182Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073411218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54182024-02-28T22:15:17.182Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073523535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54192024-02-28T22:15:17.182Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073567514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54202024-02-28T22:15:17.182Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073711019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54212024-02-28T22:15:17.183Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073748138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54222024-02-28T22:15:17.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073774128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54232024-02-28T22:15:17.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073802557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54242024-02-28T22:15:17.183Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073827416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54252024-02-28T22:15:17.183Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.073856426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54262024-02-28T22:15:17.183Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074133818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54272024-02-28T22:15:17.183Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074533037Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54282024-02-28T22:15:17.183Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074566126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54292024-02-28T22:15:17.183Z{"msg":"[0]R flush_numbers[0..12]: [82, 92, 87, 92, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074581715Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54302024-02-28T22:15:17.183Z{"msg":"[0]R generation[0..12]: [44, 49, 47, 49, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074614765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54312024-02-28T22:15:17.183Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074630554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54322024-02-28T22:15:17.184Z{"msg":"[1]R flush_numbers[0..12]: [82, 92, 87, 92, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074651073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54332024-02-28T22:15:17.184Z{"msg":"[1]R generation[0..12]: [44, 49, 47, 49, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074665623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54342024-02-28T22:15:17.184Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074687752Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54352024-02-28T22:15:17.184Z{"msg":"[2]R flush_numbers[0..12]: [82, 92, 87, 92, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074702182Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54362024-02-28T22:15:17.184Z{"msg":"[2]R generation[0..12]: [44, 49, 47, 49, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074722452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54372024-02-28T22:15:17.184Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
54382024-02-28T22:15:17.184Z,"time":"2024-02-28T22:15:16.074737111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54392024-02-28T22:15:17.184Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074763841Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54402024-02-28T22:15:17.184Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.0747861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54412024-02-28T22:15:17.184Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074800549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54422024-02-28T22:15:17.184Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074824089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"":"downstairs","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54432024-02-28T22:15:17.189Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074837439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54442024-02-28T22:15:17.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074856918Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"0","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54452024-02-28T22:15:17.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074870507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"1","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54462024-02-28T22:15:17.189Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074883847Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"client":"2","":"downstairs","session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54472024-02-28T22:15:17.189Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074910186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54482024-02-28T22:15:17.189Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074923116Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54492024-02-28T22:15:17.189Z{"msg":"7f65ab8d-ab9d-4ecb-b247-5a64695b87da is now active with session: 869137dc-604b-48df-9e85-fa894064dd1c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074936176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54502024-02-28T22:15:17.189Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074948145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335,"session_id":"869137dc-604b-48df-9e85-fa894064dd1c"}
54512024-02-28T22:15:17.189Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:16.074967365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
54522024-02-28T22:15:17.189ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54532024-02-28T22:15:17.189ZLoading write count information from file "/tmp/test_repair_verify.data"
54542024-02-28T22:15:17.189ZRun Repair workload
54552024-02-28T22:15:17.189Z01/30 Write block 468 len 36864 data: 3 3 4 5 5 5 7 7 6
54562024-02-28T22:15:17.190Z02/30 Flush
54572024-02-28T22:15:17.190Z03/30 Write block 535 len 28672 data: 4 5 4 4 4 4 6
54582024-02-28T22:15:17.190Z04/30 Read block 156 len 16384
54592024-02-28T22:15:17.190Z05/30 Read block 590 len 12288
54602024-02-28T22:15:17.190Z06/30 Read block 374 len 28672
54612024-02-28T22:15:17.193Z07/30 Write block 414 len 40960 data: 6 6 6 3 5 5 6 6 6 6
54622024-02-28T22:15:17.193Z08/30 Write block 347 len 4096 data: 4
54632024-02-28T22:15:17.193Z09/30 Read block 251 len 20480
54642024-02-28T22:15:17.193Z10/30 Read block 168 len 8192
54652024-02-28T22:15:17.193Z11/30 Read block 102 len 8192
54662024-02-28T22:15:17.193Z12/30 Read block 358 len 12288
54672024-02-28T22:15:17.193Z13/30 Flush
54682024-02-28T22:15:17.196Z14/30 Write block 57 len 16384 data: 5 5 5 5
54692024-02-28T22:15:17.199Z15/30 Write block 589 len 32768 data: 3 6 6 6 6 6 6 5
54702024-02-28T22:15:17.199Z16/30 Write block 318 len 20480 data: 2 2 2 3 4
54712024-02-28T22:15:17.199Z17/30 Write block 458 len 4096 data: 5
54722024-02-28T22:15:17.199Z18/30 Read block 74 len 8192
54732024-02-28T22:15:17.199Z19/30 Read block 450 len 12288
54742024-02-28T22:15:17.202Z20/30 Flush
54752024-02-28T22:15:17.202Z21/30 Flush
54762024-02-28T22:15:17.206Z22/30 Write block 25 len 40960 data: 2 3 4 4 5 6 5 4 4 3
54772024-02-28T22:15:17.206Z23/30 Read block 541 len 4096
54782024-02-28T22:15:17.206Z24/30 Write block 376 len 36864 data: 4 4 4 4 4 5 7 6 7
54792024-02-28T22:15:17.206Z25/30 Flush
54802024-02-28T22:15:17.209Z26/30 Write block 367 len 24576 data: 3 5 5 5 6 7
54812024-02-28T22:15:17.209Z27/30 Write block 355 len 4096 data: 6
54822024-02-28T22:15:17.213Z28/30 Read block 58 len 4096
54832024-02-28T22:15:17.213Z29/30 Read block 268 len 4096
54842024-02-28T22:15:17.213Z30/30 Read block 273 len 16384
54852024-02-28T22:15:17.213Z----------------------------------------------------------------
54862024-02-28T22:15:17.213Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:28
54872024-02-28T22:15:17.213ZStates: Active Active Active
54882024-02-28T22:15:17.213ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54892024-02-28T22:15:17.213Z 3 Acked 1002 Write 7 Done Done Sent false
54902024-02-28T22:15:17.213Z 4 Acked 1003 Read 4 Done Done Sent false
54912024-02-28T22:15:17.213Z 5 Acked 1004 Read 3 Done Done Sent false
54922024-02-28T22:15:17.213Z 6 Acked 1005 Read 7 Done Done Sent false
54932024-02-28T22:15:17.213Z 7 Acked 1006 Write 10 Done Done Sent false
54942024-02-28T22:15:17.213Z 8 Acked 1007 Write 1 Done Done Sent false
54952024-02-28T22:15:17.213Z 9 Acked 1008 Read 5 Done Done Sent false
54962024-02-28T22:15:17.213Z 10 Acked 1009 Read 2 Done Done Sent false
54972024-02-28T22:15:17.213Z 11 Acked 1010 Read 2 Done Done Sent false
54982024-02-28T22:15:17.213Z 12 Acked 1011 Read 3 Done Done Sent false
54992024-02-28T22:15:17.213Z 13 Acked 1012 Flush 0 Done Done Sent false
55002024-02-28T22:15:17.213Z 14 Acked 1013 Write 4 Done Done Sent false
55012024-02-28T22:15:17.213Z 15 Acked 1014 Write 8 Done Done Sent false
55022024-02-28T22:15:17.213Z 16 Acked 1015 Write 5 Done Done Sent false
55032024-02-28T22:15:17.213Z 17 Acked 1016 Write 1 Done Done Sent false
55042024-02-28T22:15:17.214Z 18 Acked 1017 Read 2 Done Done Sent false
55052024-02-28T22:15:17.214Z 19 Acked 1018 Read 3 Done Done Sent false
55062024-02-28T22:15:17.214Z 20 Acked 1019 Flush 0 Done Done Sent false
55072024-02-28T22:15:17.214Z 21 Acked 1020 Flush 0 Done Done Sent false
55082024-02-28T22:15:17.214Z 22 Acked 1021 Write 10 Done Done Sent false
55092024-02-28T22:15:17.214Z 23 Acked 1022 Read 1 Done Done Sent false
55102024-02-28T22:15:17.214Z 24 Acked 1023 Write 9 Done Done Sent false
55112024-02-28T22:15:17.214Z 25 Acked 1024 Flush 0 Done Done Sent false
55122024-02-28T22:15:17.214Z 26 Acked 1025 Write 6 Done Done Sent false
55132024-02-28T22:15:17.214Z 27 Acked 1026 Write 1 Done Done Sent false
55142024-02-28T22:15:17.214Z 28 Acked 1027 Read 1 Done Done Sent false
55152024-02-28T22:15:17.214Z 29 Acked 1028 Read 1 Done Done Sent false
55162024-02-28T22:15:17.214Z 30 Acked 1029 Read 4 Done Done Sent false
55172024-02-28T22:15:17.214Z STATES DS:0 DS:1 DS:2 TOTAL
55182024-02-28T22:15:17.214Z New 0 0 0 0
55192024-02-28T22:15:17.214Z Sent 0 0 28 28
55202024-02-28T22:15:17.214Z Done 28 28 0 56
55212024-02-28T22:15:17.214Z Skipped 0 0 0 0
55222024-02-28T22:15:17.214Z Error 0 0 0 0
55232024-02-28T22:15:17.214ZLast Flush: 1024 1024 1001
55242024-02-28T22:15:17.214ZDownstairs last five completed: 1001 1000
55252024-02-28T22:15:17.214ZUpstairs last five completed: 30 29 28 27 26
55262024-02-28T22:15:17.214Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:16.108514501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1335}
55272024-02-28T22:15:17.215ZWrote out file "/tmp/test_repair_verify.data"
55282024-02-28T22:15:17.218Z
55292024-02-28T22:15:17.221ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55302024-02-28T22:15:17.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55312024-02-28T22:15:17.235Z 0 000-019 44 44 44 82 82 82 F F F
55322024-02-28T22:15:17.235Z 1 020-039 50 50 49 97 97 92 F F F <---
55332024-02-28T22:15:17.235Z 2 040-059 50 50 47 95 95 87 F F F <---
55342024-02-28T22:15:17.235Z 3 060-079 50 50 49 95 95 92 F F F <---
55352024-02-28T22:15:17.235Z 4 080-099 48 48 48 91 91 91 F F F
55362024-02-28T22:15:17.235Z 5 100-119 44 44 44 80 80 80 F F F
55372024-02-28T22:15:17.235Z 6 120-139 49 49 49 92 92 92 F F F
55382024-02-28T22:15:17.235Z 7 140-159 49 49 49 92 92 92 F F F
55392024-02-28T22:15:17.235Z 8 160-179 46 46 46 86 86 86 F F F
55402024-02-28T22:15:17.236Z 9 180-199 46 46 46 86 86 86 F F F
55412024-02-28T22:15:17.240Z 10 200-219 48 48 48 91 91 91 F F F
55422024-02-28T22:15:17.240Z 11 220-239 35 35 35 67 67 67 F F F
55432024-02-28T22:15:17.240Z 12 240-259 46 46 46 86 86 86 F F F
55442024-02-28T22:15:17.240Z 13 260-279 46 46 46 86 86 86 F F F
55452024-02-28T22:15:17.240Z 14 280-299 45 45 45 83 83 83 F F F
55462024-02-28T22:15:17.240Z 15 300-319 50 50 44 95 95 80 F F F <---
55472024-02-28T22:15:17.240Z 16 320-339 50 50 37 95 95 72 F F F <---
55482024-02-28T22:15:17.240Z 17 340-359 50 50 46 94 94 86 T T F <---
55492024-02-28T22:15:17.240Z 18 360-379 50 50 44 97 97 81 T T F <---
55502024-02-28T22:15:17.241Z 19 380-399 50 50 48 97 97 88 F F F <---
55512024-02-28T22:15:17.241Z 20 400-419 50 50 48 94 94 91 F F F <---
55522024-02-28T22:15:17.241Z 21 420-439 50 50 44 94 94 82 F F F <---
55532024-02-28T22:15:17.241Z 22 440-459 50 50 38 95 95 73 F F F <---
55542024-02-28T22:15:17.241Z 23 460-479 50 50 50 93 93 93 F F F
55552024-02-28T22:15:17.241Z 24 480-499 40 40 40 76 76 76 F F F
55562024-02-28T22:15:17.241Z 25 500-519 44 44 44 81 81 81 F F F
55572024-02-28T22:15:17.241Z 26 520-539 50 50 47 94 94 87 F F F <---
55582024-02-28T22:15:17.241Z 27 540-559 50 50 48 94 94 91 F F F <---
55592024-02-28T22:15:17.241Z 28 560-579 44 44 44 82 82 82 F F F
55602024-02-28T22:15:17.241Z 29 580-599 50 50 47 95 95 87 F F F <---
55612024-02-28T22:15:17.241ZMax gen: 50, Max flush: 97
55622024-02-28T22:15:17.241ZError: Difference in extent metadata found!
55632024-02-28T22:15:17.241ZA Difference in extent metadata is expected here
55642024-02-28T22:15:17.241ZOn loop 25
55652024-02-28T22:15:17.241Z
55662024-02-28T22:15:17.244ZVerifying data now
55672024-02-28T22:15:21.941ZNo change after verify
55682024-02-28T22:15:21.941ZLoop: 25 Downstairs dump after verify (and repair):
55692024-02-28T22:15:21.957ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55702024-02-28T22:15:21.958Z 0 000-019 44 44 44 82 82 82 F F F
55712024-02-28T22:15:21.958Z 1 020-039 50 50 50 97 97 97 F F F
55722024-02-28T22:15:21.958Z 2 040-059 50 50 50 95 95 95 F F F
55732024-02-28T22:15:21.958Z 3 060-079 50 50 50 95 95 95 F F F
55742024-02-28T22:15:21.958Z 4 080-099 48 48 48 91 91 91 F F F
55752024-02-28T22:15:21.958Z 5 100-119 44 44 44 80 80 80 F F F
55762024-02-28T22:15:21.958Z 6 120-139 49 49 49 92 92 92 F F F
55772024-02-28T22:15:21.958Z 7 140-159 49 49 49 92 92 92 F F F
55782024-02-28T22:15:21.958Z 8 160-179 46 46 46 86 86 86 F F F
55792024-02-28T22:15:21.958Z 9 180-199 46 46 46 86 86 86 F F F
55802024-02-28T22:15:21.958Z 10 200-219 48 48 48 91 91 91 F F F
55812024-02-28T22:15:21.958Z 11 220-239 35 35 35 67 67 67 F F F
55822024-02-28T22:15:21.958Z 12 240-259 46 46 46 86 86 86 F F F
55832024-02-28T22:15:21.958Z 13 260-279 46 46 46 86 86 86 F F F
55842024-02-28T22:15:21.965Z 14 280-299 45 45 45 83 83 83 F F F
55852024-02-28T22:15:21.965Z 15 300-319 50 50 50 95 95 95 F F F
55862024-02-28T22:15:21.965Z 16 320-339 50 50 50 95 95 95 F F F
55872024-02-28T22:15:21.965Z 17 340-359 51 51 51 98 98 98 F F F
55882024-02-28T22:15:21.965Z 18 360-379 51 51 51 98 98 98 F F F
55892024-02-28T22:15:21.965Z 19 380-399 50 50 50 97 97 97 F F F
55902024-02-28T22:15:21.965Z 20 400-419 50 50 50 94 94 94 F F F
55912024-02-28T22:15:21.965Z 21 420-439 50 50 50 94 94 94 F F F
55922024-02-28T22:15:21.965Z 22 440-459 50 50 50 95 95 95 F F F
55932024-02-28T22:15:21.965Z 23 460-479 50 50 50 93 93 93 F F F
55942024-02-28T22:15:21.965Z 24 480-499 40 40 40 76 76 76 F F F
55952024-02-28T22:15:21.965Z 25 500-519 44 44 44 81 81 81 F F F
55962024-02-28T22:15:21.965Z 26 520-539 50 50 50 94 94 94 F F F
55972024-02-28T22:15:21.965Z 27 540-559 50 50 50 94 94 94 F F F
55982024-02-28T22:15:21.965Z 28 560-579 44 44 44 82 82 82 F F F
55992024-02-28T22:15:21.966Z 29 580-599 50 50 50 95 95 95 F F F
56002024-02-28T22:15:21.966ZMax gen: 51, Max flush: 98
56012024-02-28T22:15:21.966Z
56022024-02-28T22:15:21.966ZBegin loop 26
56032024-02-28T22:15:21.966ZDownstairs to restart: 1
56042024-02-28T22:15:21.976Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871472016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
56052024-02-28T22:15:21.976Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",Crucible runtime is spawned
56062024-02-28T22:15:22.387Z\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871785117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
56072024-02-28T22:15:22.387Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871823546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
56082024-02-28T22:15:22.387Z{"msg":"Crucible 5efe9498-deff-4b90-b781-a14593577763 has session id: 5785ccfa-9c5e-4f08-abbd-5af280a0a52a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871857375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56092024-02-28T22:15:22.387Z{"msg":"Upstairs opts: Upstairs UUID: 5efe9498-deff-4b90-b781-a14593577763, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871910434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56102024-02-28T22:15:22.387Z{"msg":"Crucible stats registered with UUID: 5efe9498-deff-4b90-b781-a14593577763","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871938863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56112024-02-28T22:15:22.387Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871952903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
56122024-02-28T22:15:22.387Z{"msg":"5efe9498-deff-4b90-b781-a14593577763 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.871975472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56132024-02-28T22:15:22.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872017261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56142024-02-28T22:15:22.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87204742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56152024-02-28T22:15:22.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872079179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56162024-02-28T22:15:22.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872112878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56172024-02-28T22:15:22.387Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872137788Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56182024-02-28T22:15:22.387Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872229045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56192024-02-28T22:15:22.387Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872266914Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56202024-02-28T22:15:22.387Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872366871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56212024-02-28T22:15:22.387Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87240086Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56222024-02-28T22:15:22.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87242169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56232024-02-28T22:15:22.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872454739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56242024-02-28T22:15:22.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.872481308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56252024-02-28T22:15:22.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87314551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56262024-02-28T22:15:22.388Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873188289Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56272024-02-28T22:15:22.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873215138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56282024-02-28T22:15:22.388Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873248317Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56292024-02-28T22:15:22.388Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873281576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56302024-02-28T22:15:22.388Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873306586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56312024-02-28T22:15:22.388Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873772853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56322024-02-28T22:15:22.388Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873917609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56332024-02-28T22:15:22.388Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.873948198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56342024-02-28T22:15:22.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874273309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56352024-02-28T22:15:22.388Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874325227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56362024-02-28T22:15:22.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874355926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56372024-02-28T22:15:22.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874398005Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56382024-02-28T22:15:22.388Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874434334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56392024-02-28T22:15:22.388Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874453334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56402024-02-28T22:15:22.388Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874478303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56412024-02-28T22:15:22.388Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874493653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56422024-02-28T22:15:22.388Z{"msg":"[0]R flush_numbers[0..12]: [82, 97, 95, 95, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874515902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56432024-02-28T22:15:22.388Z{"msg":"[0]R generation[0..12]: [44, 50, 50, 50, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874530572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56442024-02-28T22:15:22.388Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874552311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56452024-02-28T22:15:22.388Z{"msg":"[1]R flush_numbers[0..12]: [82, 97, 95, 95, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874566621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56462024-02-28T22:15:22.389Z{"msg":"[1]R generation[0..12]: [44, 50, 50, 50, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87459268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56472024-02-28T22:15:22.389Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874616619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56482024-02-28T22:15:22.389Z{"msg":"[2]R flush_numbers[0..12]: [82, 97, 95, 95, 91, 80, 92, 92, 86, 86, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874653308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56492024-02-28T22:15:22.389Z{"msg":"[2]R generation[0..12]: [44, 50, 50, 50, 48, 44, 49, 49, 46, 46, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874670878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
56502024-02-28T22:15:22.389Zsession_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56512024-02-28T22:15:22.389Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874697527Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56522024-02-28T22:15:22.389Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874712827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56532024-02-28T22:15:22.389Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874745496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56542024-02-28T22:15:22.389Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874783325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56552024-02-28T22:15:22.389Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874822664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"":"downstairs","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56562024-02-28T22:15:22.389Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874861712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56572024-02-28T22:15:22.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874898802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"0","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56582024-02-28T22:15:22.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874933331Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"1","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56592024-02-28T22:15:22.389Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.87496119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"client":"2","":"downstairs","session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56602024-02-28T22:15:22.389Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.874987789Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56612024-02-28T22:15:22.389Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.875039048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56622024-02-28T22:15:22.389Z{"msg":"5efe9498-deff-4b90-b781-a14593577763 is now active with session: 5785ccfa-9c5e-4f08-abbd-5af280a0a52a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.875077027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56632024-02-28T22:15:22.389Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.875103666Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343,"session_id":"5785ccfa-9c5e-4f08-abbd-5af280a0a52a"}
56642024-02-28T22:15:22.389Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:20.875124495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
56652024-02-28T22:15:22.389ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56662024-02-28T22:15:22.396ZLoading write count information from file "/tmp/test_repair_verify.data"
56672024-02-28T22:15:22.397ZRun Repair workload
56682024-02-28T22:15:22.397Z01/30 Write block 592 len 32768 data: 7 7 7 7 6 5 4 3
56692024-02-28T22:15:22.397Z02/30 Read block 56 len 12288
56702024-02-28T22:15:22.397Z03/30 Write block 109 len 36864 data: 6 5 6 6 3 3 4 4 4
56712024-02-28T22:15:22.397Z04/30 Read block 16 len 24576
56722024-02-28T22:15:22.397Z05/30 Read block 252 len 16384
56732024-02-28T22:15:22.397Z06/30 Write block 120 len 8192 data: 5 7
56742024-02-28T22:15:22.397Z07/30 Write block 65 len 28672 data: 6 6 7 6 5 6 6
56752024-02-28T22:15:22.397Z08/30 Write block 409 len 20480 data: 6 6 4 4 4
56762024-02-28T22:15:22.397Z09/30 Read block 167 len 32768
56772024-02-28T22:15:22.397Z10/30 Read block 251 len 28672
56782024-02-28T22:15:22.397Z11/30 Read block 83 len 36864
56792024-02-28T22:15:22.397Z12/30 Write block 184 len 12288 data: 4 3 3
56802024-02-28T22:15:22.397Z13/30 Read block 67 len 16384
56812024-02-28T22:15:22.397Z14/30 Read block 147 len 28672
56822024-02-28T22:15:22.397Z15/30 Write block 400 len 16384 data: 5 5 5 7
56832024-02-28T22:15:22.397Z16/30 Read block 398 len 32768
56842024-02-28T22:15:22.397Z17/30 Write block 275 len 24576 data: 2 3 3 3 3 3
56852024-02-28T22:15:22.397Z18/30 Read block 531 len 12288
56862024-02-28T22:15:22.397Z19/30 Read block 316 len 32768
56872024-02-28T22:15:22.397Z20/30 Write block 182 len 24576 data: 4 4 5 4 4 5
56882024-02-28T22:15:22.397Z21/30 Read block 152 len 36864
56892024-02-28T22:15:22.397Z22/30 Write block 382 len 12288 data: 8 7 8
56902024-02-28T22:15:22.398Z23/30 Read block 386 len 20480
56912024-02-28T22:15:22.398Z24/30 Write block 106 len 20480 data: 6 6 7 7 6
56922024-02-28T22:15:22.398Z25/30 Write block 565 len 24576 data: 6 7 9 9 8 6
56932024-02-28T22:15:22.398Z26/30 Read block 553 len 32768
56942024-02-28T22:15:22.398Z27/30 Read block 371 len 32768
56952024-02-28T22:15:22.398Z28/30 Write block 458 len 32768 data: 6 5 4 4 4 5 5 5
56962024-02-28T22:15:22.398Z29/30 Read block 534 len 12288
56972024-02-28T22:15:22.398Z30/30 Write block 169 len 24576 data: 8 8 7 7 7 7
56982024-02-28T22:15:22.398Z----------------------------------------------------------------
56992024-02-28T22:15:22.398Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
57002024-02-28T22:15:22.398ZStates: Active Active Active
57012024-02-28T22:15:22.398ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57022024-02-28T22:15:22.398Z 1 Acked 1000 Write 8 Done Sent Done false
57032024-02-28T22:15:22.398Z 2 Acked 1001 Read 3 Done Sent Done false
57042024-02-28T22:15:22.398Z 3 Acked 1002 Write 9 Done Sent Done false
57052024-02-28T22:15:22.398Z 4 Acked 1003 Read 6 Done Sent Done false
57062024-02-28T22:15:22.398Z 5 Acked 1004 Read 4 Done Sent Done false
57072024-02-28T22:15:22.398Z 6 Acked 1005 Write 2 Done Sent Done false
57082024-02-28T22:15:22.398Z 7 Acked 1006 Write 7 Done Sent Done false
57092024-02-28T22:15:22.398Z 8 Acked 1007 Write 5 Done Sent Done false
57102024-02-28T22:15:22.398Z 9 Acked 1008 Read 8 Done Sent Done false
57112024-02-28T22:15:22.398Z 10 Acked 1009 Read 7 Done Sent Done false
57122024-02-28T22:15:22.398Z 11 Acked 1010 Read 9 Done Sent Done false
57132024-02-28T22:15:22.399Z 12 Acked 1011 Write 3 Done Sent Done false
57142024-02-28T22:15:22.399Z 13 Acked 1012 Read 4 Done Sent Done false
57152024-02-28T22:15:22.399Z 14 Acked 1013 Read 7 Done Sent Done false
57162024-02-28T22:15:22.399Z 15 Acked 1014 Write 4 Done Sent Done false
57172024-02-28T22:15:22.399Z 16 Acked 1015 Read 8 Done Sent Done false
57182024-02-28T22:15:22.399Z 17 Acked 1016 Write 6 Done Sent Done false
57192024-02-28T22:15:22.399Z 18 Acked 1017 Read 3 Done Sent Done false
57202024-02-28T22:15:22.399Z 19 Acked 1018 Read 8 Done Sent Done false
57212024-02-28T22:15:22.399Z 20 Acked 1019 Write 6 Done Sent Done false
57222024-02-28T22:15:22.399Z 21 Acked 1020 Read 9 Done Sent Done false
57232024-02-28T22:15:22.399Z 22 Acked 1021 Write 3 Done Sent Done false
57242024-02-28T22:15:22.399Z 23 Acked 1022 Read 5 Done Sent Done false
57252024-02-28T22:15:22.399Z 24 Acked 1023 Write 5 Done Sent Done false
57262024-02-28T22:15:22.399Z 25 Acked 1024 Write 6 Done Sent Done false
57272024-02-28T22:15:22.399Z 26 Acked 1025 Read 8 Done Sent Done false
57282024-02-28T22:15:22.399Z 27 Acked 1026 Read 8 Done Sent Done false
57292024-02-28T22:15:22.399Z 28 Acked 1027 Write 8 Done Sent Done false
57302024-02-28T22:15:22.399Z 29 Acked 1028 Read 3 Done Sent Done false
57312024-02-28T22:15:22.399Z 30 Acked 1029 Write 6 Sent Sent Sent false
57322024-02-28T22:15:22.399Z STATES DS:0 DS:1 DS:2 TOTAL
57332024-02-28T22:15:22.399Z New 0 0 0 0
57342024-02-28T22:15:22.399Z Sent 1 30 1 32
57352024-02-28T22:15:22.400Z Done 29 0 29 58
57362024-02-28T22:15:22.400Z Skipped 0 0 0 0
57372024-02-28T22:15:22.400Z Error 0 0 0 0
57382024-02-28T22:15:22.400ZLast Flush: 0 0 0
57392024-02-28T22:15:22.400ZDownstairs last five completed:
57402024-02-28T22:15:22.400ZUpstairs last five completed: 30 29 28 27 26
57412024-02-28T22:15:22.400Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:20.904446348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1343}
57422024-02-28T22:15:22.400ZWrote out file "/tmp/test_repair_verify.data"
57432024-02-28T22:15:22.400Z
57442024-02-28T22:15:22.400ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57452024-02-28T22:15:22.400ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57462024-02-28T22:15:22.400Z 0 000-019 44 44 44 82 82 82 F F F
57472024-02-28T22:15:22.400Z 1 020-039 50 50 50 97 97 97 F F F
57482024-02-28T22:15:22.400Z 2 040-059 50 50 50 95 95 95 F F F
57492024-02-28T22:15:22.400Z 3 060-079 50 50 50 95 95 95 T F T <---
57502024-02-28T22:15:22.400Z 4 080-099 48 48 48 91 91 91 F F F
57512024-02-28T22:15:22.400Z 5 100-119 44 44 44 80 80 80 T F T <---
57522024-02-28T22:15:22.400Z 6 120-139 49 49 49 92 92 92 T F T <---
57532024-02-28T22:15:22.400Z 7 140-159 49 49 49 92 92 92 F F F
57542024-02-28T22:15:22.400Z 8 160-179 46 46 46 86 86 86 T F T <---
57552024-02-28T22:15:22.400Z 9 180-199 46 46 46 86 86 86 T F T <---
57562024-02-28T22:15:22.400Z 10 200-219 48 48 48 91 91 91 F F F
57572024-02-28T22:15:22.400Z 11 220-239 35 35 35 67 67 67 F F F
57582024-02-28T22:15:22.400Z 12 240-259 46 46 46 86 86 86 F F F
57592024-02-28T22:15:22.401Z 13 260-279 46 46 46 86 86 86 T F T <---
57602024-02-28T22:15:22.401Z 14 280-299 45 45 45 83 83 83 T F T <---
57612024-02-28T22:15:22.401Z 15 300-319 50 50 50 95 95 95 F F F
57622024-02-28T22:15:22.401Z 16 320-339 50 50 50 95 95 95 F F F
57632024-02-28T22:15:22.401Z 17 340-359 51 51 51 98 98 98 F F F
57642024-02-28T22:15:22.401Z 18 360-379 51 51 51 98 98 98 F F F
57652024-02-28T22:15:22.401Z 19 380-399 50 50 50 97 97 97 T F T <---
57662024-02-28T22:15:22.405Z 20 400-419 50 50 50 94 94 94 T F T <---
57672024-02-28T22:15:22.405Z 21 420-439 50 50 50 94 94 94 F F F
57682024-02-28T22:15:22.405Z 22 440-459 50 50 50 95 95 95 T F T <---
57692024-02-28T22:15:22.405Z 23 460-479 50 50 50 93 93 93 T F T <---
57702024-02-28T22:15:22.405Z 24 480-499 40 40 40 76 76 76 F F F
57712024-02-28T22:15:22.405Z 25 500-519 44 44 44 81 81 81 F F F
57722024-02-28T22:15:22.405Z 26 520-539 50 50 50 94 94 94 F F F
57732024-02-28T22:15:22.405Z 27 540-559 50 50 50 94 94 94 F F F
57742024-02-28T22:15:22.405Z 28 560-579 44 44 44 82 82 82 T F T <---
57752024-02-28T22:15:22.405Z 29 580-599 50 50 50 95 95 95 T F T <---
57762024-02-28T22:15:22.405ZMax gen: 51, Max flush: 98
57772024-02-28T22:15:22.405ZError: Difference in extent metadata found!
57782024-02-28T22:15:22.405ZA Difference in extent metadata is expected here
57792024-02-28T22:15:22.405ZOn loop 26
57802024-02-28T22:15:22.405Z
57812024-02-28T22:15:22.405ZVerifying data now
57822024-02-28T22:15:26.716ZNo change after verify
57832024-02-28T22:15:26.716ZLoop: 26 Downstairs dump after verify (and repair):
57842024-02-28T22:15:26.730ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57852024-02-28T22:15:26.730Z 0 000-019 44 44 44 82 82 82 F F F
57862024-02-28T22:15:26.730Z 1 020-039 50 50 50 97 97 97 F F F
57872024-02-28T22:15:26.730Z 2 040-059 50 50 50 95 95 95 F F F
57882024-02-28T22:15:26.730Z 3 060-079 52 52 52 99 99 99 F F F
57892024-02-28T22:15:26.730Z 4 080-099 48 48 48 91 91 91 F F F
57902024-02-28T22:15:26.731Z 5 100-119 52 52 52 99 99 99 F F F
57912024-02-28T22:15:26.731Z 6 120-139 52 52 52 99 99 99 F F F
57922024-02-28T22:15:26.731Z 7 140-159 49 49 49 92 92 92 F F F
57932024-02-28T22:15:26.731Z 8 160-179 52 52 52 99 99 99 F F F
57942024-02-28T22:15:26.731Z 9 180-199 52 52 52 99 99 99 F F F
57952024-02-28T22:15:26.737Z 10 200-219 48 48 48 91 91 91 F F F
57962024-02-28T22:15:26.737Z 11 220-239 35 35 35 67 67 67 F F F
57972024-02-28T22:15:26.737Z 12 240-259 46 46 46 86 86 86 F F F
57982024-02-28T22:15:26.737Z 13 260-279 52 52 52 99 99 99 F F F
57992024-02-28T22:15:26.737Z 14 280-299 52 52 52 99 99 99 F F F
58002024-02-28T22:15:26.737Z 15 300-319 50 50 50 95 95 95 F F F
58012024-02-28T22:15:26.737Z 16 320-339 50 50 50 95 95 95 F F F
58022024-02-28T22:15:26.737Z 17 340-359 51 51 51 98 98 98 F F F
58032024-02-28T22:15:26.737Z 18 360-379 51 51 51 98 98 98 F F F
58042024-02-28T22:15:26.737Z 19 380-399 52 52 52 99 99 99 F F F
58052024-02-28T22:15:26.737Z 20 400-419 52 52 52 99 99 99 F F F
58062024-02-28T22:15:26.737Z 21 420-439 50 50 50 94 94 94 F F F
58072024-02-28T22:15:26.738Z 22 440-459 52 52 52 99 99 99 F F F
58082024-02-28T22:15:26.738Z 23 460-479 52 52 52 99 99 99 F F F
58092024-02-28T22:15:26.738Z 24 480-499 40 40 40 76 76 76 F F F
58102024-02-28T22:15:26.738Z 25 500-519 44 44 44 81 81 81 F F F
58112024-02-28T22:15:26.738Z 26 520-539 50 50 50 94 94 94 F F F
58122024-02-28T22:15:26.738Z 27 540-559 50 50 50 94 94 94 F F F
58132024-02-28T22:15:26.738Z 28 560-579 52 52 52 99 99 99 F F F
58142024-02-28T22:15:26.738Z 29 580-599 52 52 52 99 99 99 F F F
58152024-02-28T22:15:26.738ZMax gen: 52, Max flush: 99
58162024-02-28T22:15:26.738Z
58172024-02-28T22:15:26.738ZBegin loop 27
58182024-02-28T22:15:26.738ZDownstairs to restart: 0
58192024-02-28T22:15:26.752Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.647832042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
58202024-02-28T22:15:26.752Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
58212024-02-28T22:15:26.752Z,"time":"2024-02-28T22:15:25.648167193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
58222024-02-28T22:15:26.752Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648205562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
58232024-02-28T22:15:26.752Z{"msg":"Crucible e96f76ae-2247-420c-8e07-9c715821e7ca has session id: 76001fae-651f-4199-8123-a5687ea4fc65","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648231471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58242024-02-28T22:15:26.759Z{"msg":"Upstairs opts: Upstairs UUID: e96f76ae-2247-420c-8e07-9c715821e7ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.6482871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58252024-02-28T22:15:26.759Z{"msg":"Crucible stats registered with UUID: e96f76ae-2247-420c-8e07-9c715821e7ca","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648370118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58262024-02-28T22:15:26.759Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648410066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
58272024-02-28T22:15:26.759Z{"msg":"e96f76ae-2247-420c-8e07-9c715821e7ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648444565Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58282024-02-28T22:15:26.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648499744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58292024-02-28T22:15:26.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648546842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58302024-02-28T22:15:26.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648584572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58312024-02-28T22:15:26.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648611001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58322024-02-28T22:15:26.759Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.64863513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58332024-02-28T22:15:26.759Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648659039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58342024-02-28T22:15:26.759Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648684189Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58352024-02-28T22:15:26.759Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648717658Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58362024-02-28T22:15:26.759Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648742907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58372024-02-28T22:15:26.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648890183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58382024-02-28T22:15:26.759Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.648954741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58392024-02-28T22:15:26.760Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.64898337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58402024-02-28T22:15:26.760Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.649661582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58412024-02-28T22:15:26.760Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.64973797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58422024-02-28T22:15:26.760Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.649772069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58432024-02-28T22:15:26.760Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.649799488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58442024-02-28T22:15:26.760Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.649826827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58452024-02-28T22:15:26.760Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.649858566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58462024-02-28T22:15:26.760Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650314174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58472024-02-28T22:15:26.760Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650402851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58482024-02-28T22:15:26.760Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650627325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58492024-02-28T22:15:26.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650902338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58502024-02-28T22:15:26.760Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650945207Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58512024-02-28T22:15:26.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650967906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58522024-02-28T22:15:26.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.650990865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58532024-02-28T22:15:26.760Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651006495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58542024-02-28T22:15:26.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651021714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58552024-02-28T22:15:26.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651154661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58562024-02-28T22:15:26.760Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.65119298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58572024-02-28T22:15:26.760Z{"msg":"[0]R flush_numbers[0..12]: [82, 97, 95, 99, 91, 99, 99, 92, 99, 99, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651210849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58582024-02-28T22:15:26.760Z{"msg":"[0]R generation[0..12]: [44, 50, 50, 52, 48, 52, 52, 49, 52, 52, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651233319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58592024-02-28T22:15:26.760Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651248108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58602024-02-28T22:15:26.760Z{"msg":"[1]R flush_numbers[0..12]: [82, 97, 95, 99, 91, 99, 99, 92, 99, 99, 91, 67]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651269158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58612024-02-28T22:15:26.760Z{"msg":"[1]R generation[0..12]: [44, 50, 50, 52, 48, 52, 52, 49, 52, 52, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651283867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58622024-02-28T22:15:26.761Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651304957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58632024-02-28T22:15:26.761Z{"msg":"[2]R flush_numbers[0..12]: [82, 97, 95, 99, 91, 99, 99, 92, 99, 99, 91, 67]","v":0,"nameWait for a query_work_queue command to finish before sending IO
58642024-02-28T22:15:26.761Z":"crucible","level":30,"time":"2024-02-28T22:15:25.651325216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58652024-02-28T22:15:26.761Z{"msg":"[2]R generation[0..12]: [44, 50, 50, 52, 48, 52, 52, 49, 52, 52, 48, 35]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651345905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58662024-02-28T22:15:26.761Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651360055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58672024-02-28T22:15:26.761Z{"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651379975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58682024-02-28T22:15:26.761Z{"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651395354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58692024-02-28T22:15:26.761Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651421693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58702024-02-28T22:15:26.761Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651436443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"":"downstairs","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58712024-02-28T22:15:26.761Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651462802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58722024-02-28T22:15:26.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651483812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"0","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58732024-02-28T22:15:26.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651507011Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"1","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58742024-02-28T22:15:26.761Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.65152734Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"client":"2","":"downstairs","session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58752024-02-28T22:15:26.761Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651559859Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58762024-02-28T22:15:26.761Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651591629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58772024-02-28T22:15:26.761Z{"msg":"e96f76ae-2247-420c-8e07-9c715821e7ca is now active with session: 76001fae-651f-4199-8123-a5687ea4fc65","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651615628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58782024-02-28T22:15:26.761ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58792024-02-28T22:15:26.761Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651647157Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351,"session_id":"76001fae-651f-4199-8123-a5687ea4fc65"}
58802024-02-28T22:15:26.761Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:25.651669467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
58812024-02-28T22:15:26.765ZLoading write count information from file "/tmp/test_repair_verify.data"
58822024-02-28T22:15:26.765ZRun Repair workload
58832024-02-28T22:15:26.765Z01/30 Write block 127 len 20480 data: 8 9 7 7 9
58842024-02-28T22:15:26.765Z02/30 Read block 297 len 40960
58852024-02-28T22:15:26.765Z03/30 Write block 587 len 28672 data: 3 4 4 7 7 8 8
58862024-02-28T22:15:26.765Z04/30 Read block 577 len 36864
58872024-02-28T22:15:26.765Z05/30 Write block 238 len 4096 data: 4
58882024-02-28T22:15:26.765Z06/30 Read block 508 len 12288
58892024-02-28T22:15:26.765Z07/30 Read block 261 len 12288
58902024-02-28T22:15:26.765Z08/30 Read block 562 len 40960
58912024-02-28T22:15:26.765Z09/30 Read block 326 len 8192
58922024-02-28T22:15:26.765Z10/30 Read block 220 len 32768
58932024-02-28T22:15:26.768Z11/30 Write block 59 len 8192 data: 6 6
58942024-02-28T22:15:26.768Z12/30 Read block 430 len 12288
58952024-02-28T22:15:26.768Z13/30 Flush
58962024-02-28T22:15:26.772Z14/30 Write block 436 len 28672 data: 7 7 8 6 5 5 5
58972024-02-28T22:15:26.772Z15/30 Write block 259 len 36864 data: 8 6 6 6 3 3 3 3 3
58982024-02-28T22:15:26.775Z16/30 Write block 181 len 24576 data: 4 5 5 6 5 5
58992024-02-28T22:15:26.775Z17/30 Write block 192 len 28672 data: 7 6 5 5 4 4 4
59002024-02-28T22:15:26.775Z18/30 Read block 93 len 4096
59012024-02-28T22:15:26.775Z19/30 Read block 344 len 12288
59022024-02-28T22:15:26.778Z20/30 Read block 548 len 12288
59032024-02-28T22:15:26.778Z21/30 Read block 442 len 8192
59042024-02-28T22:15:26.778Z22/30 Read block 76 len 36864
59052024-02-28T22:15:26.778Z23/30 Write block 191 len 32768 data: 5 8 7 6 6 5 5 5
59062024-02-28T22:15:26.782Z24/30 Write block 472 len 24576 data: 6 6 8 8 7 5
59072024-02-28T22:15:26.782Z25/30 Write block 318 len 12288 data: 3 3 3
59082024-02-28T22:15:26.782Z26/30 Write block 328 len 28672 data: 7 7 8 5 4 4 5
59092024-02-28T22:15:26.782Z27/30 Read block 344 len 8192
59102024-02-28T22:15:26.786Z28/30 Write block 479 len 36864 data: 4 4 3 5 5 6 7 4 4
59112024-02-28T22:15:26.786Z29/30 Write block 503 len 40960 data: 7 7 6 6 6 6 6 6 4 3
59122024-02-28T22:15:26.786Z30/30 Write block 248 len 4096 data: 8
59132024-02-28T22:15:26.786Z----------------------------------------------------------------
59142024-02-28T22:15:26.786Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59152024-02-28T22:15:26.786ZStates: Active Active Active
59162024-02-28T22:15:26.786ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59172024-02-28T22:15:26.786Z 1 Acked 1000 Write 5 Done Done Done false
59182024-02-28T22:15:26.786Z 2 Acked 1001 Read 10 Sent Done Done false
59192024-02-28T22:15:26.787Z 3 Acked 1002 Write 7 Sent Done Done false
59202024-02-28T22:15:26.787Z 4 Acked 1003 Read 9 Sent Done Done false
59212024-02-28T22:15:26.787Z 5 Acked 1004 Write 1 Sent Done Done false
59222024-02-28T22:15:26.787Z 6 Acked 1005 Read 3 Sent Done Done false
59232024-02-28T22:15:26.787Z 7 Acked 1006 Read 3 Sent Done Done false
59242024-02-28T22:15:26.787Z 8 Acked 1007 Read 10 Sent Done Done false
59252024-02-28T22:15:26.787Z 9 Acked 1008 Read 2 Sent Done Done false
59262024-02-28T22:15:26.787Z 10 Acked 1009 Read 8 Sent Done Done false
59272024-02-28T22:15:26.787Z 11 Acked 1010 Write 2 Sent Done Done false
59282024-02-28T22:15:26.787Z 12 Acked 1011 Read 3 Sent Done Done false
59292024-02-28T22:15:26.787Z 13 Acked 1012 Flush 0 Sent Done Done false
59302024-02-28T22:15:26.787Z 14 Acked 1013 Write 7 Sent Done Done false
59312024-02-28T22:15:26.787Z 15 Acked 1014 Write 9 Sent Done Done false
59322024-02-28T22:15:26.787Z 16 Acked 1015 Write 6 Sent Done Done false
59332024-02-28T22:15:26.787Z 17 Acked 1016 Write 7 Sent Done Done false
59342024-02-28T22:15:26.787Z 18 Acked 1017 Read 1 Sent Done Done false
59352024-02-28T22:15:26.787Z 19 Acked 1018 Read 3 Sent Done Done false
59362024-02-28T22:15:26.787Z 20 Acked 1019 Read 3 Sent Done Done false
59372024-02-28T22:15:26.787Z 21 Acked 1020 Read 2 Sent Done Done false
59382024-02-28T22:15:26.787Z 22 Acked 1021 Read 9 Sent Done Done false
59392024-02-28T22:15:26.787Z 23 Acked 1022 Write 8 Sent Done Done false
59402024-02-28T22:15:26.787Z 24 Acked 1023 Write 6 Sent Done Done false
59412024-02-28T22:15:26.787Z 25 Acked 1024 Write 3 Sent Done Done false
59422024-02-28T22:15:26.788Z 26 Acked 1025 Write 7 Sent Done Done false
59432024-02-28T22:15:26.788Z 27 Acked 1026 Read 2 Sent Done Done false
59442024-02-28T22:15:26.788Z 28 Acked 1027 Write 9 Sent Done Done false
59452024-02-28T22:15:26.788Z 29 Acked 1028 Write 10 Sent Sent Sent false
59462024-02-28T22:15:26.788Z 30 Acked 1029 Write 1 Sent Sent Sent false
59472024-02-28T22:15:26.788Z STATES DS:0 DS:1 DS:2 TOTAL
59482024-02-28T22:15:26.788Z New 0 0 0 0
59492024-02-28T22:15:26.788Z Sent 29 2 2 33
59502024-02-28T22:15:26.788Z Done 1 28 28 57
59512024-02-28T22:15:26.788Z Skipped 0 0 0 0
59522024-02-28T22:15:26.788Z Error 0 0 0 0
59532024-02-28T22:15:26.788ZLast Flush: 0 1012 1012
59542024-02-28T22:15:26.788ZDownstairs last five completed:
59552024-02-28T22:15:26.788ZUpstairs last five completed: 30 29 28 27 26
59562024-02-28T22:15:26.788Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:25.680340187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1351}
59572024-02-28T22:15:26.788ZWrote out file "/tmp/test_repair_verify.data"
59582024-02-28T22:15:26.792Z
59592024-02-28T22:15:26.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59602024-02-28T22:15:26.809ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59612024-02-28T22:15:26.809Z 0 000-019 44 44 44 82 82 82 F F F
59622024-02-28T22:15:26.810Z 1 020-039 50 50 50 97 97 97 F F F
59632024-02-28T22:15:26.810Z 2 040-059 50 54 54 95 100 100 F F F <---
59642024-02-28T22:15:26.810Z 3 060-079 52 54 54 99 100 100 F F F <---
59652024-02-28T22:15:26.810Z 4 080-099 48 48 48 91 91 91 F F F
59662024-02-28T22:15:26.810Z 5 100-119 52 52 52 99 99 99 F F F
59672024-02-28T22:15:26.810Z 6 120-139 52 54 54 99 100 100 T F F <---
59682024-02-28T22:15:26.810Z 7 140-159 49 49 49 92 92 92 F F F
59692024-02-28T22:15:26.810Z 8 160-179 52 52 52 99 99 99 F F F
59702024-02-28T22:15:26.810Z 9 180-199 52 52 52 99 99 99 F T T <---
59712024-02-28T22:15:26.813Z 10 200-219 48 48 48 91 91 91 F F F
59722024-02-28T22:15:26.813Z 11 220-239 35 54 54 67 100 100 F F F <---
59732024-02-28T22:15:26.813Z 12 240-259 46 46 46 86 86 86 F T T <---
59742024-02-28T22:15:26.813Z 13 260-279 52 52 52 99 99 99 F T T <---
59752024-02-28T22:15:26.813Z 14 280-299 52 52 52 99 99 99 F F F
59762024-02-28T22:15:26.813Z 15 300-319 50 50 50 95 95 95 F T T <---
59772024-02-28T22:15:26.813Z 16 320-339 50 50 50 95 95 95 F T T <---
59782024-02-28T22:15:26.813Z 17 340-359 51 51 51 98 98 98 F F F
59792024-02-28T22:15:26.813Z 18 360-379 51 51 51 98 98 98 F F F
59802024-02-28T22:15:26.814Z 19 380-399 52 52 52 99 99 99 F F F
59812024-02-28T22:15:26.814Z 20 400-419 52 52 52 99 99 99 F F F
59822024-02-28T22:15:26.814Z 21 420-439 50 50 50 94 94 94 F T T <---
59832024-02-28T22:15:26.814Z 22 440-459 52 52 52 99 99 99 F T T <---
59842024-02-28T22:15:26.814Z 23 460-479 52 52 52 99 99 99 F T T <---
59852024-02-28T22:15:26.814Z 24 480-499 40 40 40 76 76 76 F T T <---
59862024-02-28T22:15:26.814Z 25 500-519 44 44 44 81 81 81 F T T <---
59872024-02-28T22:15:26.814Z 26 520-539 50 50 50 94 94 94 F F F
59882024-02-28T22:15:26.814Z 27 540-559 50 50 50 94 94 94 F F F
59892024-02-28T22:15:26.814Z 28 560-579 52 52 52 99 99 99 F F F
59902024-02-28T22:15:26.814Z 29 580-599 52 54 54 99 100 100 F F F <---
59912024-02-28T22:15:26.814ZMax gen: 54, Max flush: 100
59922024-02-28T22:15:26.814ZError: Difference in extent metadata found!
59932024-02-28T22:15:26.814ZA Difference in extent metadata is expected here
59942024-02-28T22:15:26.814ZOn loop 27
59952024-02-28T22:15:26.814Z
59962024-02-28T22:15:26.818ZVerifying data now
59972024-02-28T22:15:31.584ZNo change after verify
59982024-02-28T22:15:31.584ZLoop: 27 Downstairs dump after verify (and repair):
59992024-02-28T22:15:31.593ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60002024-02-28T22:15:31.593Z 0 000-019 44 44 44 82 82 82 F F F
60012024-02-28T22:15:31.593Z 1 020-039 50 50 50 97 97 97 F F F
60022024-02-28T22:15:31.593Z 2 040-059 54 54 54 100 100 100 F F F
60032024-02-28T22:15:31.593Z 3 060-079 54 54 54 100 100 100 F F F
60042024-02-28T22:15:31.593Z 4 080-099 48 48 48 91 91 91 F F F
60052024-02-28T22:15:31.593Z 5 100-119 52 52 52 99 99 99 F F F
60062024-02-28T22:15:31.593Z 6 120-139 54 54 54 100 100 100 F F F
60072024-02-28T22:15:31.593Z 7 140-159 49 49 49 92 92 92 F F F
60082024-02-28T22:15:31.593Z 8 160-179 52 52 52 99 99 99 F F F
60092024-02-28T22:15:31.593Z 9 180-199 55 55 55 101 101 101 F F F
60102024-02-28T22:15:31.593Z 10 200-219 48 48 48 91 91 91 F F F
60112024-02-28T22:15:31.593Z 11 220-239 54 54 54 100 100 100 F F F
60122024-02-28T22:15:31.597Z 12 240-259 55 55 55 101 101 101 F F F
60132024-02-28T22:15:31.597Z 13 260-279 55 55 55 101 101 101 F F F
60142024-02-28T22:15:31.597Z 14 280-299 52 52 52 99 99 99 F F F
60152024-02-28T22:15:31.597Z 15 300-319 55 55 55 101 101 101 F F F
60162024-02-28T22:15:31.597Z 16 320-339 55 55 55 101 101 101 F F F
60172024-02-28T22:15:31.597Z 17 340-359 51 51 51 98 98 98 F F F
60182024-02-28T22:15:31.597Z 18 360-379 51 51 51 98 98 98 F F F
60192024-02-28T22:15:31.597Z 19 380-399 52 52 52 99 99 99 F F F
60202024-02-28T22:15:31.597Z 20 400-419 52 52 52 99 99 99 F F F
60212024-02-28T22:15:31.597Z 21 420-439 55 55 55 101 101 101 F F F
60222024-02-28T22:15:31.597Z 22 440-459 55 55 55 101 101 101 F F F
60232024-02-28T22:15:31.598Z 23 460-479 55 55 55 101 101 101 F F F
60242024-02-28T22:15:31.598Z 24 480-499 55 55 55 101 101 101 F F F
60252024-02-28T22:15:31.598Z 25 500-519 55 55 55 101 101 101 F F F
60262024-02-28T22:15:31.598Z 26 520-539 50 50 50 94 94 94 F F F
60272024-02-28T22:15:31.598Z 27 540-559 50 50 50 94 94 94 F F F
60282024-02-28T22:15:31.598Z 28 560-579 52 52 52 99 99 99 F F F
60292024-02-28T22:15:31.598Z 29 580-599 54 54 54 100 100 100 F F F
60302024-02-28T22:15:31.598ZMax gen: 55, Max flush: 101
60312024-02-28T22:15:31.598Z
60322024-02-28T22:15:31.598ZBegin loop 28
60332024-02-28T22:15:31.598ZDownstairs to restart: 1
60342024-02-28T22:15:31.614Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.509603205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359}
60352024-02-28T22:15:31.614Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.509896367Z","hostname":"Crucible runtime is spawned
60362024-02-28T22:15:31.614Zip-10-150-1-47.us-west-2.compute.internal","pid":1359}
60372024-02-28T22:15:31.614Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.509938746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359}
60382024-02-28T22:15:31.621Z{"msg":"Crucible 16f52fb9-7c04-4190-8451-a9e63341518d has session id: a8a98852-35cf-4308-a3aa-93e614009c17","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.509968905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60392024-02-28T22:15:31.621Z{"msg":"Upstairs opts: Upstairs UUID: 16f52fb9-7c04-4190-8451-a9e63341518d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510014574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60402024-02-28T22:15:31.621Z{"msg":"Crucible stats registered with UUID: 16f52fb9-7c04-4190-8451-a9e63341518d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510054713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60412024-02-28T22:15:31.621Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510080612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359}
60422024-02-28T22:15:31.621Z{"msg":"16f52fb9-7c04-4190-8451-a9e63341518d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510137541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60432024-02-28T22:15:31.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51017364Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60442024-02-28T22:15:31.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510221659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60452024-02-28T22:15:31.621Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510245528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60462024-02-28T22:15:31.621Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510282627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60472024-02-28T22:15:31.622Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510309156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60482024-02-28T22:15:31.622Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510347275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60492024-02-28T22:15:31.622Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510381464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60502024-02-28T22:15:31.622Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510418843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60512024-02-28T22:15:31.622Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510499031Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60522024-02-28T22:15:31.622Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510644187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60532024-02-28T22:15:31.622Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510701775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60542024-02-28T22:15:31.622Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.510727885Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60552024-02-28T22:15:31.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.5112556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60562024-02-28T22:15:31.622Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511292619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60572024-02-28T22:15:31.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511308729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60582024-02-28T22:15:31.622Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511326128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60592024-02-28T22:15:31.622Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511361917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60602024-02-28T22:15:31.622Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511388696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60612024-02-28T22:15:31.622Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.511940091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60622024-02-28T22:15:31.622Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51198383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60632024-02-28T22:15:31.622Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512002529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60642024-02-28T22:15:31.622Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512598283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60652024-02-28T22:15:31.622Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512636402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60662024-02-28T22:15:31.622Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512653001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60672024-02-28T22:15:31.622Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512680161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60682024-02-28T22:15:31.622Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51269464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60692024-02-28T22:15:31.622Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51270917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60702024-02-28T22:15:31.623Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512722159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60712024-02-28T22:15:31.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512744249Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60722024-02-28T22:15:31.623Z{"msg":"[0]R flush_numbers[0..12]: [82, 97, 100, 100, 91, 99, 100, 92, 99, 101, 91, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512759729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60732024-02-28T22:15:31.623Z{"msg":"[0]R generation[0..12]: [44, 50, 54, 54, 48, 52, 54, 49, 52, 55, 48, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512774068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60742024-02-28T22:15:31.623Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512788568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60752024-02-28T22:15:31.623Z{"msg":"[1]R flush_numbers[0..12]: [82, 97, 100, 100, 91, 99, 100, 92, 99, 101, 91, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512803397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60762024-02-28T22:15:31.623Z{"msg":"[1]R generation[0..12]: [44, 50, 54, 54, 48, 52, 54, 49, 52, 55, 48, 54]","v":0,"Wait for a query_work_queue command to finish before sending IO
60772024-02-28T22:15:31.623Zname":"crucible","level":30,"time":"2024-02-28T22:15:30.512828737Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60782024-02-28T22:15:31.623Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512843486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60792024-02-28T22:15:31.623Z{"msg":"[2]R flush_numbers[0..12]: [82, 97, 100, 100, 91, 99, 100, 92, 99, 101, 91, 100]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512865336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60802024-02-28T22:15:31.623Z{"msg":"[2]R generation[0..12]: [44, 50, 54, 54, 48, 52, 54, 49, 52, 55, 48, 54]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512879235Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60812024-02-28T22:15:31.623Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512901045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60822024-02-28T22:15:31.623Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512914594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60832024-02-28T22:15:31.623Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512934874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60842024-02-28T22:15:31.623Z{"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512948193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60852024-02-28T22:15:31.623Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512968793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"":"downstairs","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60862024-02-28T22:15:31.623Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.512982282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60872024-02-28T22:15:31.623Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.513000832Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"0","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60882024-02-28T22:15:31.623Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.513014771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"1","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60892024-02-28T22:15:31.623Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.513027861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"client":"2","":"downstairs","session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60902024-02-28T22:15:31.623Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51304859Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60912024-02-28T22:15:31.623Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51306135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60922024-02-28T22:15:31.624Z{"msg":"16f52fb9-7c04-4190-8451-a9e63341518d is now active with session: a8a98852-35cf-4308-a3aa-93e614009c17","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.51308129Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60932024-02-28T22:15:31.624Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.513093559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359,"session_id":"a8a98852-35cf-4308-a3aa-93e614009c17"}
60942024-02-28T22:15:31.624Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:30.513105799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359}
60952024-02-28T22:15:31.624ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60962024-02-28T22:15:31.627ZLoading write count information from file "/tmp/test_repair_verify.data"
60972024-02-28T22:15:31.627ZRun Repair workload
60982024-02-28T22:15:31.627Z01/30 Write block 240 len 24576 data: 4 4 3 3 5 5
60992024-02-28T22:15:31.627Z02/30 Read block 379 len 36864
61002024-02-28T22:15:31.627Z03/30 Read block 184 len 16384
61012024-02-28T22:15:31.628Z04/30 Read block 347 len 36864
61022024-02-28T22:15:31.628Z05/30 Read block 137 len 28672
61032024-02-28T22:15:31.628Z06/30 Write block 72 len 12288 data: 5 4 4
61042024-02-28T22:15:31.628Z07/30 Read block 494 len 36864
61052024-02-28T22:15:31.628Z08/30 Read block 394 len 8192
61062024-02-28T22:15:31.628Z09/30 Read block 489 len 36864
61072024-02-28T22:15:31.628Z10/30 Read block 5 len 8192
61082024-02-28T22:15:31.631Z11/30 Write block 4 len 24576 data: 6 6 7 8 8 7
61092024-02-28T22:15:31.631Z12/30 Write block 18 len 28672 data: 6 6 5 4 3 3 3
61102024-02-28T22:15:31.631Z13/30 Read block 234 len 16384
61112024-02-28T22:15:31.631Z14/30 Read block 556 len 28672
61122024-02-28T22:15:31.631Z15/30 Write block 172 len 8192 data: 8 8
61132024-02-28T22:15:31.634Z16/30 Read block 313 len 8192
61142024-02-28T22:15:31.634Z17/30 Read block 211 len 8192
61152024-02-28T22:15:31.634Z18/30 Flush
61162024-02-28T22:15:31.640Z19/30 Write block 35 len 20480 data: 3 4 5 5 4
61172024-02-28T22:15:31.640Z20/30 Read block 160 len 32768
61182024-02-28T22:15:31.643Z21/30 Read block 583 len 40960
61192024-02-28T22:15:31.643Z22/30 Read block 57 len 12288
61202024-02-28T22:15:31.643Z23/30 Read block 22 len 8192
61212024-02-28T22:15:31.643Z24/30 Write block 305 len 16384 data: 5 5 7 6
61222024-02-28T22:15:31.643Z25/30 Flush
61232024-02-28T22:15:31.646Z26/30 Write block 109 len 28672 data: 8 7 7 7 4 4 5
61242024-02-28T22:15:31.646Z27/30 Read block 49 len 12288
61252024-02-28T22:15:31.646Z28/30 Write block 225 len 8192 data: 4 5
61262024-02-28T22:15:31.647Z29/30 Write block 480 len 32768 data: 5 4 6 6 7 8 5 5
61272024-02-28T22:15:31.650Z30/30 Read block 514 len 12288
61282024-02-28T22:15:31.650Z----------------------------------------------------------------
61292024-02-28T22:15:31.650Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61302024-02-28T22:15:31.650ZStates: Active Active Active
61312024-02-28T22:15:31.650ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61322024-02-28T22:15:31.650Z 1 Acked 1000 Write 6 Done Done Done false
61332024-02-28T22:15:31.650Z 2 Acked 1001 Read 9 Done Done Done false
61342024-02-28T22:15:31.650Z 3 Acked 1002 Read 4 Done Done Done false
61352024-02-28T22:15:31.650Z 4 Acked 1003 Read 9 Done Done Done false
61362024-02-28T22:15:31.650Z 5 Acked 1004 Read 7 Done Done Done false
61372024-02-28T22:15:31.650Z 6 Acked 1005 Write 3 Done Done Done false
61382024-02-28T22:15:31.651Z 7 Acked 1006 Read 9 Done Done Done false
61392024-02-28T22:15:31.651Z 8 Acked 1007 Read 2 Done Sent Done false
61402024-02-28T22:15:31.651Z 9 Acked 1008 Read 9 Done Sent Done false
61412024-02-28T22:15:31.651Z 10 Acked 1009 Read 2 Done Sent Done false
61422024-02-28T22:15:31.651Z 11 Acked 1010 Write 6 Done Sent Done false
61432024-02-28T22:15:31.651Z 12 Acked 1011 Write 7 Done Sent Done false
61442024-02-28T22:15:31.651Z 13 Acked 1012 Read 4 Done Sent Done false
61452024-02-28T22:15:31.651Z 14 Acked 1013 Read 7 Done Sent Done false
61462024-02-28T22:15:31.651Z 15 Acked 1014 Write 2 Done Sent Done false
61472024-02-28T22:15:31.651Z 16 Acked 1015 Read 2 Done Sent Done false
61482024-02-28T22:15:31.651Z 17 Acked 1016 Read 2 Done Sent Done false
61492024-02-28T22:15:31.651Z 18 Acked 1017 Flush 0 Done Sent Done false
61502024-02-28T22:15:31.651Z 19 Acked 1018 Write 5 Done Sent Done false
61512024-02-28T22:15:31.651Z 20 Acked 1019 Read 8 Done Sent Done false
61522024-02-28T22:15:31.651Z 21 Acked 1020 Read 10 Done Sent Done false
61532024-02-28T22:15:31.651Z 22 Acked 1021 Read 3 Done Sent Done false
61542024-02-28T22:15:31.651Z 23 Acked 1022 Read 2 Done Sent Done false
61552024-02-28T22:15:31.651Z 24 Acked 1023 Write 4 Done Sent Done false
61562024-02-28T22:15:31.651Z 25 Acked 1024 Flush 0 Done Sent Done false
61572024-02-28T22:15:31.651Z 26 Acked 1025 Write 7 Done Sent Done false
61582024-02-28T22:15:31.651Z 27 Acked 1026 Read 3 Done Sent Done false
61592024-02-28T22:15:31.651Z 28 Acked 1027 Write 2 Done Sent Done false
61602024-02-28T22:15:31.651Z 29 Acked 1028 Write 8 Done Sent Done false
61612024-02-28T22:15:31.652Z 30 Acked 1029 Read 3 Done Sent Sent false
61622024-02-28T22:15:31.652Z STATES DS:0 DS:1 DS:2 TOTAL
61632024-02-28T22:15:31.652Z New 0 0 0 0
61642024-02-28T22:15:31.652Z Sent 0 23 1 24
61652024-02-28T22:15:31.652Z Done 30 7 29 66
61662024-02-28T22:15:31.652Z Skipped 0 0 0 0
61672024-02-28T22:15:31.652Z Error 0 0 0 0
61682024-02-28T22:15:31.652ZLast Flush: 1024 0 1024
61692024-02-28T22:15:31.652ZDownstairs last five completed:
61702024-02-28T22:15:31.652ZUpstairs last five completed: 30 29 28 27 26
61712024-02-28T22:15:31.652Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:30.544180823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1359}
61722024-02-28T22:15:31.652ZWrote out file "/tmp/test_repair_verify.data"
61732024-02-28T22:15:31.655Z
61742024-02-28T22:15:31.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61752024-02-28T22:15:31.672ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61762024-02-28T22:15:31.672Z 0 000-019 56 44 56 102 82 102 F F F <---
61772024-02-28T22:15:31.672Z 1 020-039 56 50 56 103 97 103 F F F <---
61782024-02-28T22:15:31.672Z 2 040-059 54 54 54 100 100 100 F F F
61792024-02-28T22:15:31.672Z 3 060-079 56 54 56 102 100 102 F T F <---
61802024-02-28T22:15:31.672Z 4 080-099 48 48 48 91 91 91 F F F
61812024-02-28T22:15:31.672Z 5 100-119 52 52 52 99 99 99 T F T <---
61822024-02-28T22:15:31.672Z 6 120-139 54 54 54 100 100 100 F F F
61832024-02-28T22:15:31.672Z 7 140-159 49 49 49 92 92 92 F F F
61842024-02-28T22:15:31.672Z 8 160-179 56 52 56 102 99 102 F F F <---
61852024-02-28T22:15:31.676Z 9 180-199 55 55 55 101 101 101 F F F
61862024-02-28T22:15:31.676Z 10 200-219 48 48 48 91 91 91 F F F
61872024-02-28T22:15:31.676Z 11 220-239 54 54 54 100 100 100 T F T <---
61882024-02-28T22:15:31.676Z 12 240-259 56 55 56 102 101 102 F T F <---
61892024-02-28T22:15:31.676Z 13 260-279 55 55 55 101 101 101 F F F
61902024-02-28T22:15:31.676Z 14 280-299 52 52 52 99 99 99 F F F
61912024-02-28T22:15:31.676Z 15 300-319 56 55 56 103 101 103 F F F <---
61922024-02-28T22:15:31.676Z 16 320-339 55 55 55 101 101 101 F F F
61932024-02-28T22:15:31.676Z 17 340-359 51 51 51 98 98 98 F F F
61942024-02-28T22:15:31.676Z 18 360-379 51 51 51 98 98 98 F F F
61952024-02-28T22:15:31.676Z 19 380-399 52 52 52 99 99 99 F F F
61962024-02-28T22:15:31.676Z 20 400-419 52 52 52 99 99 99 F F F
61972024-02-28T22:15:31.676Z 21 420-439 55 55 55 101 101 101 F F F
61982024-02-28T22:15:31.676Z 22 440-459 55 55 55 101 101 101 F F F
61992024-02-28T22:15:31.676Z 23 460-479 55 55 55 101 101 101 F F F
62002024-02-28T22:15:31.676Z 24 480-499 55 55 55 101 101 101 T F T <---
62012024-02-28T22:15:31.676Z 25 500-519 55 55 55 101 101 101 F F F
62022024-02-28T22:15:31.676Z 26 520-539 50 50 50 94 94 94 F F F
62032024-02-28T22:15:31.676Z 27 540-559 50 50 50 94 94 94 F F F
62042024-02-28T22:15:31.676Z 28 560-579 52 52 52 99 99 99 F F F
62052024-02-28T22:15:31.676Z 29 580-599 54 54 54 100 100 100 F F F
62062024-02-28T22:15:31.677ZMax gen: 56, Max flush: 103
62072024-02-28T22:15:31.677ZError: Difference in extent metadata found!
62082024-02-28T22:15:31.677ZA Difference in extent metadata is expected here
62092024-02-28T22:15:31.677ZOn loop 28
62102024-02-28T22:15:31.677Z
62112024-02-28T22:15:31.680ZVerifying data now
62122024-02-28T22:15:37.167ZNo change after verify
62132024-02-28T22:15:37.167ZLoop: 28 Downstairs dump after verify (and repair):
62142024-02-28T22:15:37.181ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62152024-02-28T22:15:37.181Z 0 000-019 56 56 56 102 102 102 F F F
62162024-02-28T22:15:37.181Z 1 020-039 56 56 56 103 103 103 F F F
62172024-02-28T22:15:37.181Z 2 040-059 54 54 54 100 100 100 F F F
62182024-02-28T22:15:37.181Z 3 060-079 56 56 56 102 102 102 F F F
62192024-02-28T22:15:37.181Z 4 080-099 48 48 48 91 91 91 F F F
62202024-02-28T22:15:37.181Z 5 100-119 57 57 57 104 104 104 F F F
62212024-02-28T22:15:37.181Z 6 120-139 54 54 54 100 100 100 F F F
62222024-02-28T22:15:37.181Z 7 140-159 49 49 49 92 92 92 F F F
62232024-02-28T22:15:37.181Z 8 160-179 56 56 56 102 102 102 F F F
62242024-02-28T22:15:37.181Z 9 180-199 55 55 55 101 101 101 F F F
62252024-02-28T22:15:37.181Z 10 200-219 48 48 48 91 91 91 F F F
62262024-02-28T22:15:37.182Z 11 220-239 57 57 57 104 104 104 F F F
62272024-02-28T22:15:37.182Z 12 240-259 56 56 56 102 102 102 F F F
62282024-02-28T22:15:37.185Z 13 260-279 55 55 55 101 101 101 F F F
62292024-02-28T22:15:37.185Z 14 280-299 52 52 52 99 99 99 F F F
62302024-02-28T22:15:37.185Z 15 300-319 56 56 56 103 103 103 F F F
62312024-02-28T22:15:37.185Z 16 320-339 55 55 55 101 101 101 F F F
62322024-02-28T22:15:37.186Z 17 340-359 51 51 51 98 98 98 F F F
62332024-02-28T22:15:37.186Z 18 360-379 51 51 51 98 98 98 F F F
62342024-02-28T22:15:37.186Z 19 380-399 52 52 52 99 99 99 F F F
62352024-02-28T22:15:37.186Z 20 400-419 52 52 52 99 99 99 F F F
62362024-02-28T22:15:37.186Z 21 420-439 55 55 55 101 101 101 F F F
62372024-02-28T22:15:37.186Z 22 440-459 55 55 55 101 101 101 F F F
62382024-02-28T22:15:37.186Z 23 460-479 55 55 55 101 101 101 F F F
62392024-02-28T22:15:37.186Z 24 480-499 57 57 57 104 104 104 F F F
62402024-02-28T22:15:37.186Z 25 500-519 55 55 55 101 101 101 F F F
62412024-02-28T22:15:37.186Z 26 520-539 50 50 50 94 94 94 F F F
62422024-02-28T22:15:37.186Z 27 540-559 50 50 50 94 94 94 F F F
62432024-02-28T22:15:37.186Z 28 560-579 52 52 52 99 99 99 F F F
62442024-02-28T22:15:37.187Z 29 580-599 54 54 54 100 100 100 F F F
62452024-02-28T22:15:37.187ZMax gen: 57, Max flush: 104
62462024-02-28T22:15:37.187Z
62472024-02-28T22:15:37.187ZBegin loop 29
62482024-02-28T22:15:37.187ZDownstairs to restart: 2
62492024-02-28T22:15:37.202Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.097783876Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
62502024-02-28T22:15:37.203Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098094028Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
62512024-02-28T22:15:37.209Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098148526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
62522024-02-28T22:15:37.209Z{"msg":"Crucible 26c15f5a-f584-491d-926c-34c84964f4b2 has session id: cbfb093e-316f-49d1-a886-ef40a63bad86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098198345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62532024-02-28T22:15:37.210Z{Crucible runtime is spawned
62542024-02-28T22:15:37.210Z"msg":"Upstairs opts: Upstairs UUID: 26c15f5a-f584-491d-926c-34c84964f4b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098254623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62552024-02-28T22:15:37.210Z{"msg":"Crucible stats registered with UUID: 26c15f5a-f584-491d-926c-34c84964f4b2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098289543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62562024-02-28T22:15:37.210Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098313802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
62572024-02-28T22:15:37.210Z{"msg":"26c15f5a-f584-491d-926c-34c84964f4b2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098444598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62582024-02-28T22:15:37.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098475388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62592024-02-28T22:15:37.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098502367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62602024-02-28T22:15:37.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098538576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62612024-02-28T22:15:37.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098572385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62622024-02-28T22:15:37.210Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098602644Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62632024-02-28T22:15:37.210Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098625833Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62642024-02-28T22:15:37.210Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098679962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62652024-02-28T22:15:37.211Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.09872913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62662024-02-28T22:15:37.211Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098763109Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62672024-02-28T22:15:37.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098804739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62682024-02-28T22:15:37.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098833108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62692024-02-28T22:15:37.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.098876597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62702024-02-28T22:15:37.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.09947233Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62712024-02-28T22:15:37.211Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.099516939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62722024-02-28T22:15:37.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.099545448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62732024-02-28T22:15:37.211Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.099581577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62742024-02-28T22:15:37.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.099622236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62752024-02-28T22:15:37.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.099652155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62762024-02-28T22:15:37.212Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100128622Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62772024-02-28T22:15:37.212Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100176661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62782024-02-28T22:15:37.212Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100216269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62792024-02-28T22:15:37.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100662997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62802024-02-28T22:15:37.212Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100737475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62812024-02-28T22:15:37.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100767134Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62822024-02-28T22:15:37.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100781554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62832024-02-28T22:15:37.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100796253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62842024-02-28T22:15:37.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100809933Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62852024-02-28T22:15:37.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100830413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62862024-02-28T22:15:37.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100844262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62872024-02-28T22:15:37.212Z{"msg":"[0]R flush_numbers[0..12]: [102, 103, 100, 102, 91, 104, 100, 92, 102, 101, 91, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100866192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62882024-02-28T22:15:37.213Z{"msg":"[0]R generation[0..12]: [56, 56, 54, 56, 48, 57, 54, 49, 56, 55, 48, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100880551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62892024-02-28T22:15:37.213Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100895251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62902024-02-28T22:15:37.213Z{"msg":"[1]R flush_numbers[0..12]: [102, 103, 100, 102, 91, 104, 100, 92, 102, 101, 91, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100908901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62912024-02-28T22:15:37.213Z{"msg":"[1]R generation[0..12]: [56, 56, 54, 56, 48, 57, 54, 49, 56, 55, 48, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.10092956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62922024-02-28T22:15:37.213Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100944049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62932024-02-28T22:15:37.213Z{"msg":"[2]R flush_numbers[0..12]: [102, 103, 100, 102, 91, 104, 100, 92, 102, 101, 91, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.100970059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62942024-02-28T22:15:37.213Z{"msg":"[2]R generation[0..12]: [56, 56, 54, 56, 48, 57, 54, 49, 56, 55, 48, 57]","v":0,"name":Wait for a query_work_queue command to finish before sending IO
62952024-02-28T22:15:37.213Z"crucible","level":30,"time":"2024-02-28T22:15:36.101026687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62962024-02-28T22:15:37.213Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101062826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62972024-02-28T22:15:37.213Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101100245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62982024-02-28T22:15:37.213Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101137854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
62992024-02-28T22:15:37.213Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101176163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63002024-02-28T22:15:37.214Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101201842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"":"downstairs","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63012024-02-28T22:15:37.214Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101239831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63022024-02-28T22:15:37.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101262191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"0","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63032024-02-28T22:15:37.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.10128591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"1","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63042024-02-28T22:15:37.214Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.10130028Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"client":"2","":"downstairs","session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63052024-02-28T22:15:37.214Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101314729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63062024-02-28T22:15:37.214Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101327149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63072024-02-28T22:15:37.214Z{"msg":"26c15f5a-f584-491d-926c-34c84964f4b2 is now active with session: cbfb093e-316f-49d1-a886-ef40a63bad86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101347078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63082024-02-28T22:15:37.214Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101360908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367,"session_id":"cbfb093e-316f-49d1-a886-ef40a63bad86"}
63092024-02-28T22:15:37.214Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:36.101384307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
63102024-02-28T22:15:37.214ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63112024-02-28T22:15:37.218ZLoading write count information from file "/tmp/test_repair_verify.data"
63122024-02-28T22:15:37.218ZRun Repair workload
63132024-02-28T22:15:37.219Z01/30 Write block 378 len 40960 data: 5 5 5 6 9 8 9 7 7 5
63142024-02-28T22:15:37.219Z02/30 Read block 224 len 40960
63152024-02-28T22:15:37.219Z03/30 Write block 167 len 20480 data: 10 8 9 9 8
63162024-02-28T22:15:37.219Z04/30 Read block 62 len 12288
63172024-02-28T22:15:37.219Z05/30 Flush
63182024-02-28T22:15:37.219Z06/30 Write block 67 len 8192 data: 8 7
63192024-02-28T22:15:37.219Z07/30 Read block 570 len 40960
63202024-02-28T22:15:37.219Z08/30 Read block 503 len 20480
63212024-02-28T22:15:37.219Z09/30 Write block 82 len 8192 data: 7 6
63222024-02-28T22:15:37.219Z10/30 Write block 545 len 36864 data: 8 9 9 6 6 6 5 4 4
63232024-02-28T22:15:37.224Z11/30 Write block 487 len 20480 data: 6 4 4 4 4
63242024-02-28T22:15:37.224Z12/30 Read block 333 len 40960
63252024-02-28T22:15:37.224Z13/30 Write block 493 len 40960 data: 4 3 5 5 5 5 5 6 6 6
63262024-02-28T22:15:37.224Z14/30 Write block 60 len 12288 data: 7 7 8
63272024-02-28T22:15:37.224Z15/30 Flush
63282024-02-28T22:15:37.228Z16/30 Write block 174 len 20480 data: 8 6 5 4 4
63292024-02-28T22:15:37.228Z17/30 Read block 130 len 32768
63302024-02-28T22:15:37.228Z18/30 Write block 213 len 40960 data: 4 4 3 2 2 2 2 3 3 3
63312024-02-28T22:15:37.228Z19/30 Read block 413 len 40960
63322024-02-28T22:15:37.240Z20/30 Write block 80 len 36864 data: 7 5 8 7 6 6 6 6 6
63332024-02-28T22:15:37.240Z21/30 Flush
63342024-02-28T22:15:37.245Z22/30 Write block 497 len 24576 data: 6 6 6 7 7 7
63352024-02-28T22:15:37.245Z23/30 Write block 279 len 16384 data: 4 4 3 4
63362024-02-28T22:15:37.245Z24/30 Read block 191 len 4096
63372024-02-28T22:15:37.245Z25/30 Flush
63382024-02-28T22:15:37.245Z26/30 Write block 59 len 40960 data: 7 8 8 9 8 7 7 7 9 8
63392024-02-28T22:15:37.245Z27/30 Write block 531 len 16384 data: 5 4 5 6
63402024-02-28T22:15:37.245Z28/30 Read block 304 len 20480
63412024-02-28T22:15:37.245Z29/30 Write block 159 len 32768 data: 6 4 5 5 5 6 7 7
63422024-02-28T22:15:37.245Z30/30 Write block 468 len 24576 data: 4 4 5 6 7 7
63432024-02-28T22:15:37.250Z----------------------------------------------------------------
63442024-02-28T22:15:37.250Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
63452024-02-28T22:15:37.250ZStates: Active Active Active
63462024-02-28T22:15:37.250ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63472024-02-28T22:15:37.250Z 1 Acked 1000 Write 10 Done Done Done false
63482024-02-28T22:15:37.250Z 2 Acked 1001 Read 10 Done Done Done false
63492024-02-28T22:15:37.250Z 3 Acked 1002 Write 5 Done Done Done false
63502024-02-28T22:15:37.250Z 4 Acked 1003 Read 3 Done Done Sent false
63512024-02-28T22:15:37.250Z 5 Acked 1004 Flush 0 Done Done Sent false
63522024-02-28T22:15:37.250Z 6 Acked 1005 Write 2 Done Done Sent false
63532024-02-28T22:15:37.250Z 7 Acked 1006 Read 10 Done Done Sent false
63542024-02-28T22:15:37.250Z 8 Acked 1007 Read 5 Done Done Sent false
63552024-02-28T22:15:37.250Z 9 Acked 1008 Write 2 Done Done Sent false
63562024-02-28T22:15:37.250Z 10 Acked 1009 Write 9 Done Done Sent false
63572024-02-28T22:15:37.250Z 11 Acked 1010 Write 5 Done Done Sent false
63582024-02-28T22:15:37.251Z 12 Acked 1011 Read 10 Done Done Sent false
63592024-02-28T22:15:37.251Z 13 Acked 1012 Write 10 Done Done Sent false
63602024-02-28T22:15:37.251Z 14 Acked 1013 Write 3 Done Done Sent false
63612024-02-28T22:15:37.251Z 15 Acked 1014 Flush 0 Done Done Sent false
63622024-02-28T22:15:37.251Z 16 Acked 1015 Write 5 Done Done Sent false
63632024-02-28T22:15:37.251Z 17 Acked 1016 Read 8 Done Done Sent false
63642024-02-28T22:15:37.251Z 18 Acked 1017 Write 10 Done Done Sent false
63652024-02-28T22:15:37.251Z 19 Acked 1018 Read 10 Done Done Sent false
63662024-02-28T22:15:37.251Z 20 Acked 1019 Write 9 Done Done Sent false
63672024-02-28T22:15:37.251Z 21 Acked 1020 Flush 0 Done Done Sent false
63682024-02-28T22:15:37.251Z 22 Acked 1021 Write 6 Done Done Sent false
63692024-02-28T22:15:37.251Z 23 Acked 1022 Write 4 Done Done Sent false
63702024-02-28T22:15:37.251Z 24 Acked 1023 Read 1 Done Done Sent false
63712024-02-28T22:15:37.252Z 25 Acked 1024 Flush 0 Done Done Sent false
63722024-02-28T22:15:37.252Z 26 Acked 1025 Write 10 Done Done Sent false
63732024-02-28T22:15:37.252Z 27 Acked 1026 Write 4 Done Done Sent false
63742024-02-28T22:15:37.252Z 28 Acked 1027 Read 5 Done Done Sent false
63752024-02-28T22:15:37.252Z 29 Acked 1028 Write 8 Done Done Sent false
63762024-02-28T22:15:37.252Z 30 Acked 1029 Write 6 Sent Sent Sent false
63772024-02-28T22:15:37.252Z STATES DS:0 DS:1 DS:2 TOTAL
63782024-02-28T22:15:37.252Z New 0 0 0 0
63792024-02-28T22:15:37.252Z Sent 1 1 27 29
63802024-02-28T22:15:37.252Z Done 29 29 3 61
63812024-02-28T22:15:37.252Z Skipped 0 0 0 0
63822024-02-28T22:15:37.252Z Error 0 0 0 0
63832024-02-28T22:15:37.252ZLast Flush: 1024 1024 0
63842024-02-28T22:15:37.252ZDownstairs last five completed:
63852024-02-28T22:15:37.252ZUpstairs last five completed: 30 29 28 27 26
63862024-02-28T22:15:37.252Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:36.141241309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1367}
63872024-02-28T22:15:37.252ZWrote out file "/tmp/test_repair_verify.data"
63882024-02-28T22:15:37.257Z
63892024-02-28T22:15:37.258ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63902024-02-28T22:15:37.269ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63912024-02-28T22:15:37.270Z 0 000-019 56 56 56 102 102 102 F F F
63922024-02-28T22:15:37.270Z 1 020-039 56 56 56 103 103 103 F F F
63932024-02-28T22:15:37.270Z 2 040-059 54 54 54 100 100 100 T T F <---
63942024-02-28T22:15:37.270Z 3 060-079 58 58 56 106 106 102 T T F <---
63952024-02-28T22:15:37.270Z 4 080-099 58 58 48 107 107 91 F F F <---
63962024-02-28T22:15:37.274Z 5 100-119 57 57 57 104 104 104 F F F
63972024-02-28T22:15:37.274Z 6 120-139 54 54 54 100 100 100 F F F
63982024-02-28T22:15:37.274Z 7 140-159 49 49 49 92 92 92 T T F <---
63992024-02-28T22:15:37.274Z 8 160-179 58 58 56 107 107 102 T T T <---
64002024-02-28T22:15:37.274Z 9 180-199 55 55 55 101 101 101 F F F
64012024-02-28T22:15:37.274Z 10 200-219 58 58 48 107 107 91 F F F <---
64022024-02-28T22:15:37.274Z 11 220-239 58 58 57 107 107 104 F F F <---
64032024-02-28T22:15:37.274Z 12 240-259 56 56 56 102 102 102 F F F
64042024-02-28T22:15:37.275Z 13 260-279 58 58 55 108 108 101 F F F <---
64052024-02-28T22:15:37.275Z 14 280-299 58 58 52 108 108 99 F F F <---
64062024-02-28T22:15:37.275Z 15 300-319 56 56 56 103 103 103 F F F
64072024-02-28T22:15:37.275Z 16 320-339 55 55 55 101 101 101 F F F
64082024-02-28T22:15:37.275Z 17 340-359 51 51 51 98 98 98 F F F
64092024-02-28T22:15:37.275Z 18 360-379 58 58 51 105 105 98 F F T <---
64102024-02-28T22:15:37.275Z 19 380-399 58 58 52 105 105 99 F F T <---
64112024-02-28T22:15:37.275Z 20 400-419 52 52 52 99 99 99 F F F
64122024-02-28T22:15:37.275Z 21 420-439 55 55 55 101 101 101 F F F
64132024-02-28T22:15:37.275Z 22 440-459 55 55 55 101 101 101 F F F
64142024-02-28T22:15:37.275Z 23 460-479 55 55 55 101 101 101 T T F <---
64152024-02-28T22:15:37.275Z 24 480-499 58 58 57 108 108 104 F F F <---
64162024-02-28T22:15:37.276Z 25 500-519 58 58 55 108 108 101 F F F <---
64172024-02-28T22:15:37.276Z 26 520-539 50 50 50 94 94 94 T T F <---
64182024-02-28T22:15:37.276Z 27 540-559 58 58 50 106 106 94 F F F <---
64192024-02-28T22:15:37.276Z 28 560-579 52 52 52 99 99 99 F F F
64202024-02-28T22:15:37.276Z 29 580-599 54 54 54 100 100 100 F F F
64212024-02-28T22:15:37.276ZMax gen: 58, Max flush: 108
64222024-02-28T22:15:37.276ZError: Difference in extent metadata found!
64232024-02-28T22:15:37.276ZA Difference in extent metadata is expected here
64242024-02-28T22:15:37.276ZOn loop 29
64252024-02-28T22:15:37.276Z
64262024-02-28T22:15:37.280ZVerifying data now
64272024-02-28T22:15:43.075ZNo change after verify
64282024-02-28T22:15:43.075ZLoop: 29 Downstairs dump after verify (and repair):
64292024-02-28T22:15:43.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64302024-02-28T22:15:43.086Z 0 000-019 56 56 56 102 102 102 F F F
64312024-02-28T22:15:43.086Z 1 020-039 56 56 56 103 103 103 F F F
64322024-02-28T22:15:43.086Z 2 040-059 59 59 59 109 109 109 F F F
64332024-02-28T22:15:43.086Z 3 060-079 59 59 59 109 109 109 F F F
64342024-02-28T22:15:43.086Z 4 080-099 58 58 58 107 107 107 F F F
64352024-02-28T22:15:43.086Z 5 100-119 57 57 57 104 104 104 F F F
64362024-02-28T22:15:43.086Z 6 120-139 54 54 54 100 100 100 F F F
64372024-02-28T22:15:43.086Z 7 140-159 59 59 59 109 109 109 F F F
64382024-02-28T22:15:43.086Z 8 160-179 59 59 59 109 109 109 F F F
64392024-02-28T22:15:43.087Z 9 180-199 55 55 55 101 101 101 F F F
64402024-02-28T22:15:43.087Z 10 200-219 58 58 58 107 107 107 F F F
64412024-02-28T22:15:43.087Z 11 220-239 58 58 58 107 107 107 F F F
64422024-02-28T22:15:43.090Z 12 240-259 56 56 56 102 102 102 F F F
64432024-02-28T22:15:43.090Z 13 260-279 58 58 58 108 108 108 F F F
64442024-02-28T22:15:43.090Z 14 280-299 58 58 58 108 108 108 F F F
64452024-02-28T22:15:43.090Z 15 300-319 56 56 56 103 103 103 F F F
64462024-02-28T22:15:43.090Z 16 320-339 55 55 55 101 101 101 F F F
64472024-02-28T22:15:43.090Z 17 340-359 51 51 51 98 98 98 F F F
64482024-02-28T22:15:43.090Z 18 360-379 58 58 58 105 105 105 F F F
64492024-02-28T22:15:43.090Z 19 380-399 58 58 58 105 105 105 F F F
64502024-02-28T22:15:43.090Z 20 400-419 52 52 52 99 99 99 F F F
64512024-02-28T22:15:43.090Z 21 420-439 55 55 55 101 101 101 F F F
64522024-02-28T22:15:43.091Z 22 440-459 55 55 55 101 101 101 F F F
64532024-02-28T22:15:43.091Z 23 460-479 59 59 59 109 109 109 F F F
64542024-02-28T22:15:43.091Z 24 480-499 58 58 58 108 108 108 F F F
64552024-02-28T22:15:43.091Z 25 500-519 58 58 58 108 108 108 F F F
64562024-02-28T22:15:43.091Z 26 520-539 59 59 59 109 109 109 F F F
64572024-02-28T22:15:43.091Z 27 540-559 58 58 58 106 106 106 F F F
64582024-02-28T22:15:43.091Z 28 560-579 52 52 52 99 99 99 F F F
64592024-02-28T22:15:43.091Z 29 580-599 54 54 54 100 100 100 F F F
64602024-02-28T22:15:43.091ZMax gen: 59, Max flush: 109
64612024-02-28T22:15:43.091Z
64622024-02-28T22:15:43.091ZBegin loop 30
64632024-02-28T22:15:43.091ZDownstairs to restart: 0
64642024-02-28T22:15:43.109Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.003913519Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
64652024-02-28T22:15:43.114Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
64662024-02-28T22:15:43.114Z,"time":"2024-02-28T22:15:42.004270889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
64672024-02-28T22:15:43.114Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004310168Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
64682024-02-28T22:15:43.114Z{"msg":"Crucible d65fdf60-1cf6-40d6-9bf5-71d502f56b59 has session id: ebf0217e-31d4-4a85-bcb0-a1423080345a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004337937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64692024-02-28T22:15:43.114Z{"msg":"Upstairs opts: Upstairs UUID: d65fdf60-1cf6-40d6-9bf5-71d502f56b59, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004378136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64702024-02-28T22:15:43.115Z{"msg":"Crucible stats registered with UUID: d65fdf60-1cf6-40d6-9bf5-71d502f56b59","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004394855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64712024-02-28T22:15:43.115Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004424564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
64722024-02-28T22:15:43.115Z{"msg":"d65fdf60-1cf6-40d6-9bf5-71d502f56b59 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004507032Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64732024-02-28T22:15:43.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004534761Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64742024-02-28T22:15:43.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.00457061Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64752024-02-28T22:15:43.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004611399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64762024-02-28T22:15:43.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004650308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64772024-02-28T22:15:43.115Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004684187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64782024-02-28T22:15:43.115Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004717996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64792024-02-28T22:15:43.115Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004743036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64802024-02-28T22:15:43.115Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004784915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64812024-02-28T22:15:43.115Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004812554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64822024-02-28T22:15:43.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004910271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64832024-02-28T22:15:43.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.00494437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64842024-02-28T22:15:43.115Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.004985389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64852024-02-28T22:15:43.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.005630181Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64862024-02-28T22:15:43.115Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.00567317Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64872024-02-28T22:15:43.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.005712349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64882024-02-28T22:15:43.115Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.005752458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64892024-02-28T22:15:43.115Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006082399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64902024-02-28T22:15:43.115Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006131327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64912024-02-28T22:15:43.115Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006245804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64922024-02-28T22:15:43.115Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006313132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64932024-02-28T22:15:43.116Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.00678196Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64942024-02-28T22:15:43.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006947755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64952024-02-28T22:15:43.116Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006979484Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64962024-02-28T22:15:43.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.006995694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64972024-02-28T22:15:43.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007027633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64982024-02-28T22:15:43.116Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007046572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
64992024-02-28T22:15:43.116Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007077011Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65002024-02-28T22:15:43.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007251796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65012024-02-28T22:15:43.120Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007325774Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65022024-02-28T22:15:43.121Z{"msg":"[0]R flush_numbers[0..12]: [102, 103, 109, 109, 107, 104, 100, 109, 109, 101, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007363184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65032024-02-28T22:15:43.121Z{"msg":"[0]R generation[0..12]: [56, 56, 59, 59, 58, 57, 54, 59, 59, 55, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007380083Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65042024-02-28T22:15:43.121Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007395372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65052024-02-28T22:15:43.121Z{"msg":"[1]R flush_numbers[0..12]: [102, 103, 109, 109, 107, 104, 100, 109, 109, 101, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007410412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65062024-02-28T22:15:43.121Z{"msg":"[1]R generation[0..12]: [56, 56, 59, 59, 58, 57, 54, 59, 59, 55, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007424472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65072024-02-28T22:15:43.121Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007447341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
65082024-02-28T22:15:43.121Zdownstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65092024-02-28T22:15:43.121Z{"msg":"[2]R flush_numbers[0..12]: [102, 103, 109, 109, 107, 104, 100, 109, 109, 101, 107, 107]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007466591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65102024-02-28T22:15:43.121Z{"msg":"[2]R generation[0..12]: [56, 56, 59, 59, 58, 57, 54, 59, 59, 55, 58, 58]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.00748973Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65112024-02-28T22:15:43.121Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.0075049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65122024-02-28T22:15:43.121Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007526819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65132024-02-28T22:15:43.121Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007541199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65142024-02-28T22:15:43.121Z{"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007554778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65152024-02-28T22:15:43.121Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007568038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"":"downstairs","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65162024-02-28T22:15:43.121Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007581447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65172024-02-28T22:15:43.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007594467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"0","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65182024-02-28T22:15:43.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007607767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"1","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65192024-02-28T22:15:43.121Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007621796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"client":"2","":"downstairs","session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65202024-02-28T22:15:43.121Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007635266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65212024-02-28T22:15:43.121Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007648436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65222024-02-28T22:15:43.121Z{"msg":"d65fdf60-1cf6-40d6-9bf5-71d502f56b59 is now active with session: ebf0217e-31d4-4a85-bcb0-a1423080345a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007661485Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65232024-02-28T22:15:43.121Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007681925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375,"session_id":"ebf0217e-31d4-4a85-bcb0-a1423080345a"}
65242024-02-28T22:15:43.122Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:42.007694874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
65252024-02-28T22:15:43.122ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65262024-02-28T22:15:43.122ZLoading write count information from file "/tmp/test_repair_verify.data"
65272024-02-28T22:15:43.122ZRun Repair workload
65282024-02-28T22:15:43.122Z01/30 Write block 566 len 32768 data: 8 10 10 9 7 5 6 5
65292024-02-28T22:15:43.122Z02/30 Write block 233 len 8192 data: 5 5
65302024-02-28T22:15:43.122Z03/30 Read block 63 len 16384
65312024-02-28T22:15:43.122Z04/30 Flush
65322024-02-28T22:15:43.122Z05/30 Write block 12 len 32768 data: 8 7 6 6 6 6 7 7
65332024-02-28T22:15:43.122Z06/30 Write block 229 len 12288 data: 8 7 7
65342024-02-28T22:15:43.125Z07/30 Write block 100 len 4096 data: 3
65352024-02-28T22:15:43.125Z08/30 Write block 418 len 32768 data: 6 6 7 7 7 7 6 6
65362024-02-28T22:15:43.125Z09/30 Write block 570 len 40960 data: 8 6 7 6 4 5 4 5 5 5
65372024-02-28T22:15:43.125Z10/30 Read block 176 len 24576
65382024-02-28T22:15:43.126Z11/30 Read block 461 len 4096
65392024-02-28T22:15:43.126Z12/30 Read block 258 len 40960
65402024-02-28T22:15:43.129Z13/30 Read block 496 len 4096
65412024-02-28T22:15:43.129Z14/30 Write block 174 len 20480 data: 9 7 6 5 5
65422024-02-28T22:15:43.129Z15/30 Read block 570 len 28672
65432024-02-28T22:15:43.129Z16/30 Write block 41 len 16384 data: 4 3 3 2
65442024-02-28T22:15:43.129Z17/30 Read block 118 len 40960
65452024-02-28T22:15:43.132Z18/30 Read block 88 len 32768
65462024-02-28T22:15:43.132Z19/30 Write block 439 len 24576 data: 7 6 6 6 3 4
65472024-02-28T22:15:43.132Z20/30 Write block 372 len 8192 data: 8 6
65482024-02-28T22:15:43.132Z21/30 Read block 583 len 40960
65492024-02-28T22:15:43.137Z22/30 Write block 371 len 16384 data: 7 9 7 5
65502024-02-28T22:15:43.137Z23/30 Write block 187 len 36864 data: 6 5 4 5 6 9 8 7 7
65512024-02-28T22:15:43.140Z24/30 Write block 504 len 36864 data: 8 7 7 7 7 7 7 5 4
65522024-02-28T22:15:43.140Z25/30 Write block 564 len 8192 data: 5 7
65532024-02-28T22:15:43.140Z26/30 Write block 260 len 40960 data: 7 7 7 4 4 4 4 4 4 3
65542024-02-28T22:15:43.140Z27/30 Read block 531 len 12288
65552024-02-28T22:15:43.140Z28/30 Write block 402 len 28672 data: 6 8 7 6 6 7 7
65562024-02-28T22:15:43.141Z29/30 Read block 413 len 40960
65572024-02-28T22:15:43.144Z30/30 Write block 257 len 32768 data: 8 9 9 8 8 8 5 5
65582024-02-28T22:15:43.144Z----------------------------------------------------------------
65592024-02-28T22:15:43.144Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65602024-02-28T22:15:43.144ZStates: Active Active Active
65612024-02-28T22:15:43.144ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65622024-02-28T22:15:43.145Z 1 Acked 1000 Write 8 Sent Done Done false
65632024-02-28T22:15:43.145Z 2 Acked 1001 Write 2 Sent Done Done false
65642024-02-28T22:15:43.145Z 3 Acked 1002 Read 4 Sent Done Done false
65652024-02-28T22:15:43.145Z 4 Acked 1003 Flush 0 Sent Done Done false
65662024-02-28T22:15:43.145Z 5 Acked 1004 Write 8 Sent Done Done false
65672024-02-28T22:15:43.145Z 6 Acked 1005 Write 3 Sent Done Done false
65682024-02-28T22:15:43.145Z 7 Acked 1006 Write 1 Sent Done Done false
65692024-02-28T22:15:43.145Z 8 Acked 1007 Write 8 Sent Done Done false
65702024-02-28T22:15:43.145Z 9 Acked 1008 Write 10 Sent Done Done false
65712024-02-28T22:15:43.145Z 10 Acked 1009 Read 6 Sent Done Done false
65722024-02-28T22:15:43.145Z 11 Acked 1010 Read 1 Sent Done Done false
65732024-02-28T22:15:43.145Z 12 Acked 1011 Read 10 Sent Done Done false
65742024-02-28T22:15:43.145Z 13 Acked 1012 Read 1 Sent Done Done false
65752024-02-28T22:15:43.145Z 14 Acked 1013 Write 5 Sent Done Done false
65762024-02-28T22:15:43.145Z 15 Acked 1014 Read 7 Sent Done Done false
65772024-02-28T22:15:43.145Z 16 Acked 1015 Write 4 Sent Done Done false
65782024-02-28T22:15:43.145Z 17 Acked 1016 Read 10 Sent Done Done false
65792024-02-28T22:15:43.145Z 18 Acked 1017 Read 8 Sent Done Done false
65802024-02-28T22:15:43.145Z 19 Acked 1018 Write 6 Sent Done Done false
65812024-02-28T22:15:43.145Z 20 Acked 1019 Write 2 Sent Done Done false
65822024-02-28T22:15:43.145Z 21 Acked 1020 Read 10 Sent Done Done false
65832024-02-28T22:15:43.145Z 22 Acked 1021 Write 4 Sent Done Done false
65842024-02-28T22:15:43.145Z 23 Acked 1022 Write 9 Sent Done Done false
65852024-02-28T22:15:43.146Z 24 Acked 1023 Write 9 Sent Done Done false
65862024-02-28T22:15:43.146Z 25 Acked 1024 Write 2 Sent Done Done false
65872024-02-28T22:15:43.146Z 26 Acked 1025 Write 10 Sent Done Done false
65882024-02-28T22:15:43.146Z 27 Acked 1026 Read 3 Sent Done Done false
65892024-02-28T22:15:43.146Z 28 Acked 1027 Write 7 Sent Done Done false
65902024-02-28T22:15:43.146Z 29 Acked 1028 Read 10 Sent Done Done false
65912024-02-28T22:15:43.146Z 30 Acked 1029 Write 8 Sent Sent Sent false
65922024-02-28T22:15:43.146Z STATES DS:0 DS:1 DS:2 TOTAL
65932024-02-28T22:15:43.146Z New 0 0 0 0
65942024-02-28T22:15:43.146Z Sent 30 1 1 32
65952024-02-28T22:15:43.146Z Done 0 29 29 58
65962024-02-28T22:15:43.146Z Skipped 0 0 0 0
65972024-02-28T22:15:43.146Z Error 0 0 0 0
65982024-02-28T22:15:43.146ZLast Flush: 0 1003 1003
65992024-02-28T22:15:43.146ZDownstairs last five completed:
66002024-02-28T22:15:43.146ZUpstairs last five completed: 30 29 28 27 26
66012024-02-28T22:15:43.146Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:42.037100884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1375}
66022024-02-28T22:15:43.146ZWrote out file "/tmp/test_repair_verify.data"
66032024-02-28T22:15:43.150Z
66042024-02-28T22:15:43.150ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
66052024-02-28T22:15:43.169ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66062024-02-28T22:15:43.169Z 0 000-019 56 56 56 102 102 102 F T T <---
66072024-02-28T22:15:43.169Z 1 020-039 56 56 56 103 103 103 F F F
66082024-02-28T22:15:43.169Z 2 040-059 59 59 59 109 109 109 F T T <---
66092024-02-28T22:15:43.169Z 3 060-079 59 59 59 109 109 109 F F F
66102024-02-28T22:15:43.169Z 4 080-099 58 58 58 107 107 107 F F F
66112024-02-28T22:15:43.169Z 5 100-119 57 57 57 104 104 104 F T T <---
66122024-02-28T22:15:43.169Z 6 120-139 54 54 54 100 100 100 F F F
66132024-02-28T22:15:43.169Z 7 140-159 59 59 59 109 109 109 F F F
66142024-02-28T22:15:43.169Z 8 160-179 59 59 59 109 109 109 F T T <---
66152024-02-28T22:15:43.169Z 9 180-199 55 55 55 101 101 101 F T T <---
66162024-02-28T22:15:43.173Z 10 200-219 58 58 58 107 107 107 F F F
66172024-02-28T22:15:43.173Z 11 220-239 58 60 60 107 110 110 F T T <---
66182024-02-28T22:15:43.173Z 12 240-259 56 56 56 102 102 102 F T T <---
66192024-02-28T22:15:43.173Z 13 260-279 58 58 58 108 108 108 F T T <---
66202024-02-28T22:15:43.173Z 14 280-299 58 58 58 108 108 108 F F F
66212024-02-28T22:15:43.173Z 15 300-319 56 56 56 103 103 103 F F F
66222024-02-28T22:15:43.173Z 16 320-339 55 55 55 101 101 101 F F F
66232024-02-28T22:15:43.173Z 17 340-359 51 51 51 98 98 98 F F F
66242024-02-28T22:15:43.173Z 18 360-379 58 58 58 105 105 105 F T T <---
66252024-02-28T22:15:43.173Z 19 380-399 58 58 58 105 105 105 F F F
66262024-02-28T22:15:43.173Z 20 400-419 52 52 52 99 99 99 F T T <---
66272024-02-28T22:15:43.173Z 21 420-439 55 55 55 101 101 101 F T T <---
66282024-02-28T22:15:43.173Z 22 440-459 55 55 55 101 101 101 F T T <---
66292024-02-28T22:15:43.173Z 23 460-479 59 59 59 109 109 109 F F F
66302024-02-28T22:15:43.173Z 24 480-499 58 58 58 108 108 108 F F F
66312024-02-28T22:15:43.173Z 25 500-519 58 58 58 108 108 108 F T T <---
66322024-02-28T22:15:43.173Z 26 520-539 59 59 59 109 109 109 F F F
66332024-02-28T22:15:43.173Z 27 540-559 58 58 58 106 106 106 F F F
66342024-02-28T22:15:43.173Z 28 560-579 52 60 60 99 110 110 F T T <---
66352024-02-28T22:15:43.173Z 29 580-599 54 54 54 100 100 100 F F F
66362024-02-28T22:15:43.174ZMax gen: 60, Max flush: 110
66372024-02-28T22:15:43.174ZError: Difference in extent metadata found!
66382024-02-28T22:15:43.174ZA Difference in extent metadata is expected here
66392024-02-28T22:15:43.174ZOn loop 30
66402024-02-28T22:15:43.174Z
66412024-02-28T22:15:43.177ZVerifying data now
66422024-02-28T22:15:47.898ZNo change after verify
66432024-02-28T22:15:47.898ZLoop: 30 Downstairs dump after verify (and repair):
66442024-02-28T22:15:47.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66452024-02-28T22:15:47.913Z 0 000-019 61 61 61 111 111 111 F F F
66462024-02-28T22:15:47.913Z 1 020-039 56 56 56 103 103 103 F F F
66472024-02-28T22:15:47.913Z 2 040-059 61 61 61 111 111 111 F F F
66482024-02-28T22:15:47.919Z 3 060-079 59 59 59 109 109 109 F F F
66492024-02-28T22:15:47.919Z 4 080-099 58 58 58 107 107 107 F F F
66502024-02-28T22:15:47.919Z 5 100-119 61 61 61 111 111 111 F F F
66512024-02-28T22:15:47.919Z 6 120-139 54 54 54 100 100 100 F F F
66522024-02-28T22:15:47.919Z 7 140-159 59 59 59 109 109 109 F F F
66532024-02-28T22:15:47.919Z 8 160-179 61 61 61 111 111 111 F F F
66542024-02-28T22:15:47.919Z 9 180-199 61 61 61 111 111 111 F F F
66552024-02-28T22:15:47.919Z 10 200-219 58 58 58 107 107 107 F F F
66562024-02-28T22:15:47.919Z 11 220-239 61 61 61 111 111 111 F F F
66572024-02-28T22:15:47.919Z 12 240-259 61 61 61 111 111 111 F F F
66582024-02-28T22:15:47.919Z 13 260-279 61 61 61 111 111 111 F F F
66592024-02-28T22:15:47.919Z 14 280-299 58 58 58 108 108 108 F F F
66602024-02-28T22:15:47.919Z 15 300-319 56 56 56 103 103 103 F F F
66612024-02-28T22:15:47.919Z 16 320-339 55 55 55 101 101 101 F F F
66622024-02-28T22:15:47.919Z 17 340-359 51 51 51 98 98 98 F F F
66632024-02-28T22:15:47.919Z 18 360-379 61 61 61 111 111 111 F F F
66642024-02-28T22:15:47.919Z 19 380-399 58 58 58 105 105 105 F F F
66652024-02-28T22:15:47.919Z 20 400-419 61 61 61 111 111 111 F F F
66662024-02-28T22:15:47.919Z 21 420-439 61 61 61 111 111 111 F F F
66672024-02-28T22:15:47.919Z 22 440-459 61 61 61 111 111 111 F F F
66682024-02-28T22:15:47.919Z 23 460-479 59 59 59 109 109 109 F F F
66692024-02-28T22:15:47.920Z 24 480-499 58 58 58 108 108 108 F F F
66702024-02-28T22:15:47.920Z 25 500-519 61 61 61 111 111 111 F F F
66712024-02-28T22:15:47.920Z 26 520-539 59 59 59 109 109 109 F F F
66722024-02-28T22:15:47.920Z 27 540-559 58 58 58 106 106 106 F F F
66732024-02-28T22:15:47.920Z 28 560-579 61 61 61 111 111 111 F F F
66742024-02-28T22:15:47.920Z 29 580-599 54 54 54 100 100 100 F F F
66752024-02-28T22:15:47.920ZMax gen: 61, Max flush: 111
66762024-02-28T22:15:47.920Z
66772024-02-28T22:15:47.920ZBegin loop 31
66782024-02-28T22:15:47.920ZDownstairs to restart: 1
66792024-02-28T22:15:47.935Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.829417781Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
66802024-02-28T22:15:47.935Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\nCrucible runtime is spawned
66812024-02-28T22:15:47.942Z rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.8297866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
66822024-02-28T22:15:47.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.829842839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
66832024-02-28T22:15:47.942Z{"msg":"Crucible 42850d09-2702-4a8c-a86c-40492710f8f8 has session id: e5517f5b-e792-4f68-9910-a9145e292fb4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.829884928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66842024-02-28T22:15:47.942Z{"msg":"Upstairs opts: Upstairs UUID: 42850d09-2702-4a8c-a86c-40492710f8f8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.829931387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66852024-02-28T22:15:47.942Z{"msg":"Crucible stats registered with UUID: 42850d09-2702-4a8c-a86c-40492710f8f8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.829973675Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66862024-02-28T22:15:47.943Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830009194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
66872024-02-28T22:15:47.943Z{"msg":"42850d09-2702-4a8c-a86c-40492710f8f8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830041223Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66882024-02-28T22:15:47.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830073412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66892024-02-28T22:15:47.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830113911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66902024-02-28T22:15:47.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83014669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66912024-02-28T22:15:47.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830197119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66922024-02-28T22:15:47.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830220588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66932024-02-28T22:15:47.943Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830254698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66942024-02-28T22:15:47.943Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830302496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66952024-02-28T22:15:47.943Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830399734Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66962024-02-28T22:15:47.943Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830443912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66972024-02-28T22:15:47.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830471252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66982024-02-28T22:15:47.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.830502911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
66992024-02-28T22:15:47.943Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83051855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67002024-02-28T22:15:47.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831038086Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67012024-02-28T22:15:47.943Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831070495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67022024-02-28T22:15:47.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831085605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67032024-02-28T22:15:47.943Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831100054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67042024-02-28T22:15:47.943Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831129763Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67052024-02-28T22:15:47.943Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831147543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67062024-02-28T22:15:47.943Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831685688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67072024-02-28T22:15:47.943Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831746626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67082024-02-28T22:15:47.944Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.831779925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67092024-02-28T22:15:47.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832298971Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67102024-02-28T22:15:47.944Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83234716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67112024-02-28T22:15:47.944Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832385899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67122024-02-28T22:15:47.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832418348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67132024-02-28T22:15:47.944Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832453677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67142024-02-28T22:15:47.944Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832487446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67152024-02-28T22:15:47.944Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832520445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67162024-02-28T22:15:47.944Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832555244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client"Wait for a query_work_queue command to finish before sending IO
67172024-02-28T22:15:47.944Z:"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67182024-02-28T22:15:47.944Z{"msg":"[0]R flush_numbers[0..12]: [111, 103, 111, 109, 107, 111, 100, 109, 111, 111, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832598573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67192024-02-28T22:15:47.944Z{"msg":"[0]R generation[0..12]: [61, 56, 61, 59, 58, 61, 54, 59, 61, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832638332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67202024-02-28T22:15:47.944Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832675171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67212024-02-28T22:15:47.944Z{"msg":"[1]R flush_numbers[0..12]: [111, 103, 111, 109, 107, 111, 100, 109, 111, 111, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83271622Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67222024-02-28T22:15:47.944Z{"msg":"[1]R generation[0..12]: [61, 56, 61, 59, 58, 61, 54, 59, 61, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832755879Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67232024-02-28T22:15:47.944Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832796208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67242024-02-28T22:15:47.944Z{"msg":"[2]R flush_numbers[0..12]: [111, 103, 111, 109, 107, 111, 100, 109, 111, 111, 107, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832839026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67252024-02-28T22:15:47.944Z{"msg":"[2]R generation[0..12]: [61, 56, 61, 59, 58, 61, 54, 59, 61, 61, 58, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832878695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67262024-02-28T22:15:47.944Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832919844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67272024-02-28T22:15:47.944Z"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67282024-02-28T22:15:47.944Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832959773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67292024-02-28T22:15:47.944Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832985192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67302024-02-28T22:15:47.944Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.832999682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67312024-02-28T22:15:47.944Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833021161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"":"downstairs","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67322024-02-28T22:15:47.945Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833035091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67332024-02-28T22:15:47.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83305409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"0","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67342024-02-28T22:15:47.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.83306803Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"1","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67352024-02-28T22:15:47.945Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833087829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"client":"2","":"downstairs","session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67362024-02-28T22:15:47.945Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833101549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67372024-02-28T22:15:47.945Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833120719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67382024-02-28T22:15:47.945Z{"msg":"42850d09-2702-4a8c-a86c-40492710f8f8 is now active with session: e5517f5b-e792-4f68-9910-a9145e292fb4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833134378Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67392024-02-28T22:15:47.945Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833153418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383,"session_id":"e5517f5b-e792-4f68-9910-a9145e292fb4"}
67402024-02-28T22:15:47.945Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:46.833165997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
67412024-02-28T22:15:47.954ZLoading write count information from file "/tmp/test_repair_verify.data"
67422024-02-28T22:15:47.954ZRun Repair workload
67432024-02-28T22:15:47.955Z01/30 Write block 304 len 4096 data: 6
67442024-02-28T22:15:47.955Z02/30 Read block 454 len 8192
67452024-02-28T22:15:47.955Z03/30 Read block 283 len 36864
67462024-02-28T22:15:47.955Z04/30 Write block 77 len 8192 data: 4 4
67472024-02-28T22:15:47.955Z05/30 Write block 441 len 20480 data: 7 7 4 5 5
67482024-02-28T22:15:47.956Z06/30 Read block 423 len 36864
67492024-02-28T22:15:47.956Z07/30 Write block 514 len 8192 data: 4 5
67502024-02-28T22:15:47.956Z08/30 Flush
67512024-02-28T22:15:47.956Z09/30 Write block 491 len 8192 data: 5 4
67522024-02-28T22:15:47.956Z10/30 Flush
67532024-02-28T22:15:47.975Z11/30 Write block 210 len 40960 data: 4 5 4 5 5 4 3 3 3 3
67542024-02-28T22:15:47.976Z12/30 Write block 503 len 28672 data: 8 9 8 8 8 8 8
67552024-02-28T22:15:47.976Z13/30 Flush
67562024-02-28T22:15:47.976Z14/30 Write block 266 len 4096 data: 5
67572024-02-28T22:15:47.976Z15/30 Write block 95 len 20480 data: 4 4 3 3 3
67582024-02-28T22:15:47.976Z16/30 Read block 222 len 28672
67592024-02-28T22:15:47.976Z17/30 Read block 204 len 8192
67602024-02-28T22:15:47.976Z18/30 Read block 23 len 20480
67612024-02-28T22:15:47.976Z19/30 Write block 256 len 4096 data: 7
67622024-02-28T22:15:47.976Z20/30 Read block 287 len 4096
67632024-02-28T22:15:47.977Z21/30 Read block 233 len 24576
67642024-02-28T22:15:47.977Z22/30 Write block 598 len 8192 data: 5 4
67652024-02-28T22:15:47.977Z23/30 Flush
67662024-02-28T22:15:47.977Z24/30 Write block 146 len 40960 data: 9 8 9 9 7 5 4 3 4 5
67672024-02-28T22:15:47.977Z25/30 Write block 357 len 24576 data: 6 6 4 4 4 5
67682024-02-28T22:15:47.977Z26/30 Flush
67692024-02-28T22:15:47.977Z27/30 Write block 184 len 24576 data: 7 6 6 7 6 5
67702024-02-28T22:15:47.977Z28/30 Write block 578 len 16384 data: 6 6 5 5
67712024-02-28T22:15:47.977Z29/30 Read block 247 len 32768
67722024-02-28T22:15:48.398Z30/30 Write block 101 len 40960 data: 3 3 6 5 5 7 7 8 9 8
67732024-02-28T22:15:48.398Z----------------------------------------------------------------
67742024-02-28T22:15:48.398Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:20
67752024-02-28T22:15:48.398ZStates: Active Active Active
67762024-02-28T22:15:48.398ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67772024-02-28T22:15:48.398Z 11 Acked 1010 Write 10 Done Sent Done false
67782024-02-28T22:15:48.398Z 12 Acked 1011 Write 7 Done Sent Done false
67792024-02-28T22:15:48.399Z 13 Acked 1012 Flush 0 Done Sent Done false
67802024-02-28T22:15:48.399Z 14 Acked 1013 Write 1 Done Sent Done false
67812024-02-28T22:15:48.399Z 15 Acked 1014 Write 5 Done Sent Done false
67822024-02-28T22:15:48.399Z 16 Acked 1015 Read 7 Done Sent Done false
67832024-02-28T22:15:48.399Z 17 Acked 1016 Read 2 Done Sent Done false
67842024-02-28T22:15:48.399Z 18 Acked 1017 Read 5 Done Sent Done false
67852024-02-28T22:15:48.399Z 19 Acked 1018 Write 1 Done Sent Done false
67862024-02-28T22:15:48.399Z 20 Acked 1019 Read 1 Done Sent Done false
67872024-02-28T22:15:48.399Z 21 Acked 1020 Read 6 Done Sent Done false
67882024-02-28T22:15:48.399Z 22 Acked 1021 Write 2 Done Sent Done false
67892024-02-28T22:15:48.399Z 23 Acked 1022 Flush 0 Done Sent Done false
67902024-02-28T22:15:48.399Z 24 Acked 1023 Write 10 Done Sent Done false
67912024-02-28T22:15:48.399Z 25 Acked 1024 Write 6 Done Sent Done false
67922024-02-28T22:15:48.400Z 26 Acked 1025 Flush 0 Done Sent Done false
67932024-02-28T22:15:48.400Z 27 Acked 1026 Write 6 Done Sent Done false
67942024-02-28T22:15:48.400Z 28 Acked 1027 Write 4 Done Sent Done false
67952024-02-28T22:15:48.400Z 29 Acked 1028 Read 8 Done Sent Done false
67962024-02-28T22:15:48.400Z 30 Acked 1029 Write 10 Sent Sent Sent false
67972024-02-28T22:15:48.400Z STATES DS:0 DS:1 DS:2 TOTAL
67982024-02-28T22:15:48.400Z New 0 0 0 0
67992024-02-28T22:15:48.400Z Sent 1 20 1 22
68002024-02-28T22:15:48.401Z Done 19 0 19 38
68012024-02-28T22:15:48.401Z Skipped 0 0 0 0
68022024-02-28T22:15:48.401Z Error 0 0 0 0
68032024-02-28T22:15:48.401ZLast Flush: 1025 1009 1025
68042024-02-28T22:15:48.401ZDownstairs last five completed: 1009 1008 1007 1006 1005
68052024-02-28T22:15:48.401ZUpstairs last five completed: 30 29 28 27 26
68062024-02-28T22:15:48.401Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:46.865533186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1383}
68072024-02-28T22:15:48.401ZWrote out file "/tmp/test_repair_verify.data"
68082024-02-28T22:15:48.401Z
68092024-02-28T22:15:48.401ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68102024-02-28T22:15:48.412ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68112024-02-28T22:15:48.412Z 0 000-019 61 61 61 111 111 111 F F F
68122024-02-28T22:15:48.412Z 1 020-039 56 56 56 103 103 103 F F F
68132024-02-28T22:15:48.412Z 2 040-059 61 61 61 111 111 111 F F F
68142024-02-28T22:15:48.413Z 3 060-079 62 62 62 112 112 112 F F F
68152024-02-28T22:15:48.413Z 4 080-099 62 58 62 115 107 115 F F F <---
68162024-02-28T22:15:48.413Z 5 100-119 61 61 61 111 111 111 T F T <---
68172024-02-28T22:15:48.413Z 6 120-139 54 54 54 100 100 100 F F F
68182024-02-28T22:15:48.413Z 7 140-159 62 59 62 116 109 116 F F F <---
68192024-02-28T22:15:48.413Z 8 160-179 61 61 61 111 111 111 F F F
68202024-02-28T22:15:48.413Z 9 180-199 61 61 61 111 111 111 T F T <---
68212024-02-28T22:15:48.414Z 10 200-219 62 58 62 114 107 114 F F F <---
68222024-02-28T22:15:48.414Z 11 220-239 61 61 61 111 111 111 F F F
68232024-02-28T22:15:48.414Z 12 240-259 62 61 62 115 111 115 F F F <---
68242024-02-28T22:15:48.414Z 13 260-279 62 61 62 115 111 115 F F F <---
68252024-02-28T22:15:48.414Z 14 280-299 58 58 58 108 108 108 F F F
68262024-02-28T22:15:48.415Z 15 300-319 62 62 62 112 112 112 F F F
68272024-02-28T22:15:48.415Z 16 320-339 55 55 55 101 101 101 F F F
68282024-02-28T22:15:48.415Z 17 340-359 62 51 62 116 98 116 F F F <---
68292024-02-28T22:15:48.415Z 18 360-379 62 61 62 116 111 116 F F F <---
68302024-02-28T22:15:48.415Z 19 380-399 58 58 58 105 105 105 F F F
68312024-02-28T22:15:48.415Z 20 400-419 61 61 61 111 111 111 F F F
68322024-02-28T22:15:48.415Z 21 420-439 61 61 61 111 111 111 F F F
68332024-02-28T22:15:48.415Z 22 440-459 62 62 62 112 112 112 F F F
68342024-02-28T22:15:48.416Z 23 460-479 59 59 59 109 109 109 F F F
68352024-02-28T22:15:48.416Z 24 480-499 62 62 62 113 113 113 F F F
68362024-02-28T22:15:48.416Z 25 500-519 62 62 62 114 112 114 F F F <---
68372024-02-28T22:15:48.416Z 26 520-539 59 59 59 109 109 109 F F F
68382024-02-28T22:15:48.416Z 27 540-559 58 58 58 106 106 106 F F F
68392024-02-28T22:15:48.416Z 28 560-579 61 61 61 111 111 111 T F T <---
68402024-02-28T22:15:48.416Z 29 580-599 62 54 62 115 100 115 T F T <---
68412024-02-28T22:15:48.416ZMax gen: 62, Max flush: 116
68422024-02-28T22:15:48.417ZError: Difference in extent metadata found!
68432024-02-28T22:15:48.417ZA Difference in extent metadata is expected here
68442024-02-28T22:15:48.417ZOn loop 31
68452024-02-28T22:15:48.417Z
68462024-02-28T22:15:48.417ZVerifying data now
68472024-02-28T22:15:52.634ZNo change after verify
68482024-02-28T22:15:52.634ZLoop: 31 Downstairs dump after verify (and repair):
68492024-02-28T22:15:52.648ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68502024-02-28T22:15:52.648Z 0 000-019 61 61 61 111 111 111 F F F
68512024-02-28T22:15:52.648Z 1 020-039 56 56 56 103 103 103 F F F
68522024-02-28T22:15:52.648Z 2 040-059 61 61 61 111 111 111 F F F
68532024-02-28T22:15:52.648Z 3 060-079 62 62 62 112 112 112 F F F
68542024-02-28T22:15:52.648Z 4 080-099 62 62 62 115 115 115 F F F
68552024-02-28T22:15:52.648Z 5 100-119 63 63 63 117 117 117 F F F
68562024-02-28T22:15:52.648Z 6 120-139 54 54 54 100 100 100 F F F
68572024-02-28T22:15:52.648Z 7 140-159 62 62 62 116 116 116 F F F
68582024-02-28T22:15:52.649Z 8 160-179 61 61 61 111 111 111 F F F
68592024-02-28T22:15:52.649Z 9 180-199 63 63 63 117 117 117 F F F
68602024-02-28T22:15:52.652Z 10 200-219 62 62 62 114 114 114 F F F
68612024-02-28T22:15:52.652Z 11 220-239 61 61 61 111 111 111 F F F
68622024-02-28T22:15:52.652Z 12 240-259 62 62 62 115 115 115 F F F
68632024-02-28T22:15:52.652Z 13 260-279 62 62 62 115 115 115 F F F
68642024-02-28T22:15:52.652Z 14 280-299 58 58 58 108 108 108 F F F
68652024-02-28T22:15:52.652Z 15 300-319 62 62 62 112 112 112 F F F
68662024-02-28T22:15:52.652Z 16 320-339 55 55 55 101 101 101 F F F
68672024-02-28T22:15:52.652Z 17 340-359 62 62 62 116 116 116 F F F
68682024-02-28T22:15:52.652Z 18 360-379 62 62 62 116 116 116 F F F
68692024-02-28T22:15:52.653Z 19 380-399 58 58 58 105 105 105 F F F
68702024-02-28T22:15:52.653Z 20 400-419 61 61 61 111 111 111 F F F
68712024-02-28T22:15:52.653Z 21 420-439 61 61 61 111 111 111 F F F
68722024-02-28T22:15:52.653Z 22 440-459 62 62 62 112 112 112 F F F
68732024-02-28T22:15:52.653Z 23 460-479 59 59 59 109 109 109 F F F
68742024-02-28T22:15:52.653Z 24 480-499 62 62 62 113 113 113 F F F
68752024-02-28T22:15:52.653Z 25 500-519 62 62 62 114 114 114 F F F
68762024-02-28T22:15:52.653Z 26 520-539 59 59 59 109 109 109 F F F
68772024-02-28T22:15:52.653Z 27 540-559 58 58 58 106 106 106 F F F
68782024-02-28T22:15:52.653Z 28 560-579 63 63 63 117 117 117 F F F
68792024-02-28T22:15:52.653Z 29 580-599 63 63 63 117 117 117 F F F
68802024-02-28T22:15:52.653ZMax gen: 63, Max flush: 117
68812024-02-28T22:15:52.653Z
68822024-02-28T22:15:52.653ZBegin loop 32
68832024-02-28T22:15:52.653ZDownstairs to restart: 1
68842024-02-28T22:15:52.670Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564223422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
68852024-02-28T22:15:52.670Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564501964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
68862024-02-28T22:15:52.676Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564533374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
68872024-02-28T22:15:52.676Z{"msg":"Crucible 416b1e08-07e2-4f7f-8b79-ec6286547b48 has session id: c3d945d7-fce8-4d49-a158-66e07e0ec286","v":0,"name":"crucible","level":30Crucible runtime is spawned
68882024-02-28T22:15:52.676Z,"time":"2024-02-28T22:15:51.564620691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68892024-02-28T22:15:52.676Z{"msg":"Upstairs opts: Upstairs UUID: 416b1e08-07e2-4f7f-8b79-ec6286547b48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564682069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68902024-02-28T22:15:52.676Z{"msg":"Crucible stats registered with UUID: 416b1e08-07e2-4f7f-8b79-ec6286547b48","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564727858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68912024-02-28T22:15:52.676Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564767657Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
68922024-02-28T22:15:52.676Z{"msg":"416b1e08-07e2-4f7f-8b79-ec6286547b48 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564872834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68932024-02-28T22:15:52.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564905033Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68942024-02-28T22:15:52.676Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564924603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68952024-02-28T22:15:52.676Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564957692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68962024-02-28T22:15:52.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.564984891Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68972024-02-28T22:15:52.677Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56501742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68982024-02-28T22:15:52.677Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565061599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
68992024-02-28T22:15:52.677Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565096918Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69002024-02-28T22:15:52.677Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565127367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69012024-02-28T22:15:52.677Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565264693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69022024-02-28T22:15:52.677Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565307222Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69032024-02-28T22:15:52.677Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565349351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69042024-02-28T22:15:52.677Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56538111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69052024-02-28T22:15:52.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565883216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69062024-02-28T22:15:52.677Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565924305Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69072024-02-28T22:15:52.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565951104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69082024-02-28T22:15:52.677Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.565991173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69092024-02-28T22:15:52.677Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.566071071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69102024-02-28T22:15:52.677Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56611234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69112024-02-28T22:15:52.677Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.566516839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69122024-02-28T22:15:52.677Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.566552448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69132024-02-28T22:15:52.677Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.566593187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69142024-02-28T22:15:52.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567111432Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69152024-02-28T22:15:52.677Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567150571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69162024-02-28T22:15:52.677Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567167661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69172024-02-28T22:15:52.677Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56718844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69182024-02-28T22:15:52.677Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56720238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69192024-02-28T22:15:52.678Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567224169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69202024-02-28T22:15:52.678Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567237919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69212024-02-28T22:15:52.678Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567251868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69222024-02-28T22:15:52.678Z{"msg":"[0]R flush_numbers[0..12]: [111, 103, 111, 112, 115, 117, 100, 116, 111, 117, 114, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567272368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69232024-02-28T22:15:52.678Z{"msg":"[0]R generation[0..12]: [61, 56, 61, 62, 62, 63, 54, 62, 61, 63, 62, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567287158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69242024-02-28T22:15:52.678Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567310397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69252024-02-28T22:15:52.678Z{"msg":"[1]R flush_numbers[0..12]: [111, 103, 111, 112, 115, 117, 100, 116, 111, 117, 114, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567347526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69262024-02-28T22:15:52.678Z{"msg":"[1]R generation[0..12]: [61, 56, 61, 62, 62, 63, 54, 62, 61, 63, 62, 61]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
69272024-02-28T22:15:52.678Z,"time":"2024-02-28T22:15:51.567368285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69282024-02-28T22:15:52.678Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567407814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69292024-02-28T22:15:52.678Z{"msg":"[2]R flush_numbers[0..12]: [111, 103, 111, 112, 115, 117, 100, 116, 111, 117, 114, 111]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567443433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69302024-02-28T22:15:52.678Z{"msg":"[2]R generation[0..12]: [61, 56, 61, 62, 62, 63, 54, 62, 61, 63, 62, 61]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567463503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69312024-02-28T22:15:52.678Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567492192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69322024-02-28T22:15:52.678Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567521301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69332024-02-28T22:15:52.678Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567536591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69342024-02-28T22:15:52.678Z{"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.56756594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69352024-02-28T22:15:52.678Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567596529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"":"downstairs","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69362024-02-28T22:15:52.678Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567616469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69372024-02-28T22:15:52.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567644758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"0","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69382024-02-28T22:15:52.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567661517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"1","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69392024-02-28T22:15:52.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567691236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"client":"2","":"downstairs","session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69402024-02-28T22:15:52.678Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567706266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69412024-02-28T22:15:52.679Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567736625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69422024-02-28T22:15:52.679Z{"msg":"416b1e08-07e2-4f7f-8b79-ec6286547b48 is now active with session: c3d945d7-fce8-4d49-a158-66e07e0ec286","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567755705Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69432024-02-28T22:15:52.679Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567783034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391,"session_id":"c3d945d7-fce8-4d49-a158-66e07e0ec286"}
69442024-02-28T22:15:52.679Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:51.567797793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
69452024-02-28T22:15:52.682ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69462024-02-28T22:15:52.682ZLoading write count information from file "/tmp/test_repair_verify.data"
69472024-02-28T22:15:52.682ZRun Repair workload
69482024-02-28T22:15:52.683Z01/30 Write block 232 len 32768 data: 7 6 6 4 3 3 5 4
69492024-02-28T22:15:52.683Z02/30 Write block 231 len 16384 data: 8 8 7 7
69502024-02-28T22:15:52.683Z03/30 Write block 221 len 24576 data: 4 4 4 4 5 6
69512024-02-28T22:15:52.683Z04/30 Read block 506 len 16384
69522024-02-28T22:15:52.683Z05/30 Write block 430 len 40960 data: 4 4 6 6 7 7 8 8 9 8
69532024-02-28T22:15:52.683Z06/30 Read block 239 len 28672
69542024-02-28T22:15:52.683Z07/30 Read block 99 len 16384
69552024-02-28T22:15:52.683Z08/30 Read block 475 len 12288
69562024-02-28T22:15:52.683Z09/30 Write block 38 len 16384 data: 6 5 4 5
69572024-02-28T22:15:52.683Z10/30 Read block 467 len 8192
69582024-02-28T22:15:52.686Z11/30 Read block 350 len 28672
69592024-02-28T22:15:52.686Z12/30 Write block 571 len 40960 data: 7 8 7 5 6 5 6 7 7 6
69602024-02-28T22:15:52.686Z13/30 Write block 413 len 40960 data: 5 7 7 7 4 7 7 8 8 8
69612024-02-28T22:15:52.686Z14/30 Read block 307 len 24576
69622024-02-28T22:15:52.689Z15/30 Read block 197 len 36864
69632024-02-28T22:15:52.689Z16/30 Write block 376 len 40960 data: 5 5 6 6 6 7 10 9 10 8
69642024-02-28T22:15:52.695Z17/30 Read block 335 len 12288
69652024-02-28T22:15:52.695Z18/30 Flush
69662024-02-28T22:15:52.698Z19/30 Write block 35 len 36864 data: 4 5 6 7 6 5 6 4 4
69672024-02-28T22:15:52.698Z20/30 Flush
69682024-02-28T22:15:52.701Z21/30 Write block 133 len 32768 data: 9 8 8 7 7 7 7 6
69692024-02-28T22:15:52.701Z22/30 Write block 490 len 32768 data: 5 6 5 5 4 6 6 7
69702024-02-28T22:15:52.701Z23/30 Write block 4 len 24576 data: 7 7 8 9 9 8
69712024-02-28T22:15:52.701Z24/30 Write block 577 len 16384 data: 7 8 8 7
69722024-02-28T22:15:52.704Z25/30 Write block 231 len 12288 data: 9 9 8
69732024-02-28T22:15:52.704Z26/30 Read block 35 len 28672
69742024-02-28T22:15:52.704Z27/30 Write block 35 len 28672 data: 5 6 7 8 7 6 7
69752024-02-28T22:15:52.704Z28/30 Write block 410 len 24576 data: 7 5 5 6 8 8
69762024-02-28T22:15:52.708Z29/30 Read block 0 len 4096
69772024-02-28T22:15:52.708Z30/30 Write block 382 len 16384 data: 11 10 11 9
69782024-02-28T22:15:52.708Z----------------------------------------------------------------
69792024-02-28T22:15:52.708Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69802024-02-28T22:15:52.708ZStates: Active Active Active
69812024-02-28T22:15:52.708ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69822024-02-28T22:15:52.708Z 1 Acked 1000 Write 8 Done Sent Done false
69832024-02-28T22:15:52.708Z 2 Acked 1001 Write 4 Done Sent Done false
69842024-02-28T22:15:52.708Z 3 Acked 1002 Write 6 Done Sent Done false
69852024-02-28T22:15:52.708Z 4 Acked 1003 Read 4 Done Sent Done false
69862024-02-28T22:15:52.708Z 5 Acked 1004 Write 10 Done Sent Done false
69872024-02-28T22:15:52.708Z 6 Acked 1005 Read 7 Done Sent Done false
69882024-02-28T22:15:52.708Z 7 Acked 1006 Read 4 Done Sent Done false
69892024-02-28T22:15:52.708Z 8 Acked 1007 Read 3 Done Sent Done false
69902024-02-28T22:15:52.708Z 9 Acked 1008 Write 4 Done Sent Done false
69912024-02-28T22:15:52.709Z 10 Acked 1009 Read 2 Done Sent Done false
69922024-02-28T22:15:52.709Z 11 Acked 1010 Read 7 Done Sent Done false
69932024-02-28T22:15:52.709Z 12 Acked 1011 Write 10 Done Sent Done false
69942024-02-28T22:15:52.709Z 13 Acked 1012 Write 10 Done Sent Done false
69952024-02-28T22:15:52.709Z 14 Acked 1013 Read 6 Done Sent Done false
69962024-02-28T22:15:52.709Z 15 Acked 1014 Read 9 Done Sent Done false
69972024-02-28T22:15:52.709Z 16 Acked 1015 Write 10 Done Sent Done false
69982024-02-28T22:15:52.709Z 17 Acked 1016 Read 3 Done Sent Done false
69992024-02-28T22:15:52.709Z 18 Acked 1017 Flush 0 Done Sent Done false
70002024-02-28T22:15:52.709Z 19 Acked 1018 Write 9 Done Sent Done false
70012024-02-28T22:15:52.709Z 20 Acked 1019 Flush 0 Done Sent Done false
70022024-02-28T22:15:52.709Z 21 Acked 1020 Write 8 Done Sent Done false
70032024-02-28T22:15:52.709Z 22 Acked 1021 Write 8 Done Sent Done false
70042024-02-28T22:15:52.709Z 23 Acked 1022 Write 6 Done Sent Done false
70052024-02-28T22:15:52.709Z 24 Acked 1023 Write 4 Done Sent Done false
70062024-02-28T22:15:52.709Z 25 Acked 1024 Write 3 Done Sent Done false
70072024-02-28T22:15:52.709Z 26 Acked 1025 Read 7 Done Sent Done false
70082024-02-28T22:15:52.709Z 27 Acked 1026 Write 7 Done Sent Done false
70092024-02-28T22:15:52.709Z 28 Acked 1027 Write 6 Done Sent Done false
70102024-02-28T22:15:52.709Z 29 Acked 1028 Read 1 Done Sent Done false
70112024-02-28T22:15:52.709Z 30 Acked 1029 Write 4 Sent Sent Sent false
70122024-02-28T22:15:52.709Z STATES DS:0 DS:1 DS:2 TOTAL
70132024-02-28T22:15:52.709Z New 0 0 0 0
70142024-02-28T22:15:52.710Z Sent 1 30 1 32
70152024-02-28T22:15:52.710Z Done 29 0 29 58
70162024-02-28T22:15:52.710Z Skipped 0 0 0 0
70172024-02-28T22:15:52.710Z Error 0 0 0 0
70182024-02-28T22:15:52.710ZLast Flush: 1019 0 1019
70192024-02-28T22:15:52.710ZDownstairs last five completed:
70202024-02-28T22:15:52.710ZUpstairs last five completed: 30 29 28 27 26
70212024-02-28T22:15:52.710Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:51.601125405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1391}
70222024-02-28T22:15:52.710ZWrote out file "/tmp/test_repair_verify.data"
70232024-02-28T22:15:52.713Z
70242024-02-28T22:15:52.716ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70252024-02-28T22:15:52.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70262024-02-28T22:15:52.732Z 0 000-019 61 61 61 111 111 111 T F T <---
70272024-02-28T22:15:52.732Z 1 020-039 64 56 64 119 103 119 T F T <---
70282024-02-28T22:15:52.732Z 2 040-059 64 61 64 119 111 119 T F T <---
70292024-02-28T22:15:52.732Z 3 060-079 62 62 62 112 112 112 F F F
70302024-02-28T22:15:52.733Z 4 080-099 62 62 62 115 115 115 F F F
70312024-02-28T22:15:52.733Z 5 100-119 63 63 63 117 117 117 F F F
70322024-02-28T22:15:52.733Z 6 120-139 54 54 54 100 100 100 T F T <---
70332024-02-28T22:15:52.733Z 7 140-159 62 62 62 116 116 116 T F T <---
70342024-02-28T22:15:52.733Z 8 160-179 61 61 61 111 111 111 F F F
70352024-02-28T22:15:52.736Z 9 180-199 63 63 63 117 117 117 F F F
70362024-02-28T22:15:52.736Z 10 200-219 62 62 62 114 114 114 F F F
70372024-02-28T22:15:52.736Z 11 220-239 64 61 64 118 111 118 T F T <---
70382024-02-28T22:15:52.736Z 12 240-259 62 62 62 115 115 115 F F F
70392024-02-28T22:15:52.736Z 13 260-279 62 62 62 115 115 115 F F F
70402024-02-28T22:15:52.736Z 14 280-299 58 58 58 108 108 108 F F F
70412024-02-28T22:15:52.736Z 15 300-319 62 62 62 112 112 112 F F F
70422024-02-28T22:15:52.736Z 16 320-339 55 55 55 101 101 101 F F F
70432024-02-28T22:15:52.736Z 17 340-359 62 62 62 116 116 116 F F F
70442024-02-28T22:15:52.736Z 18 360-379 64 62 64 118 116 118 F F F <---
70452024-02-28T22:15:52.736Z 19 380-399 64 58 64 118 105 118 T F T <---
70462024-02-28T22:15:52.736Z 20 400-419 64 61 64 118 111 118 T F T <---
70472024-02-28T22:15:52.737Z 21 420-439 64 61 64 118 111 118 F F F <---
70482024-02-28T22:15:52.737Z 22 440-459 62 62 62 112 112 112 F F F
70492024-02-28T22:15:52.737Z 23 460-479 59 59 59 109 109 109 F F F
70502024-02-28T22:15:52.737Z 24 480-499 62 62 62 113 113 113 T F T <---
70512024-02-28T22:15:52.737Z 25 500-519 62 62 62 114 114 114 F F F
70522024-02-28T22:15:52.737Z 26 520-539 59 59 59 109 109 109 F F F
70532024-02-28T22:15:52.737Z 27 540-559 58 58 58 106 106 106 F F F
70542024-02-28T22:15:52.737Z 28 560-579 64 63 64 118 117 118 T F T <---
70552024-02-28T22:15:52.737Z 29 580-599 64 63 64 118 117 118 T F T <---
70562024-02-28T22:15:52.737ZMax gen: 64, Max flush: 119
70572024-02-28T22:15:52.737ZError: Difference in extent metadata found!
70582024-02-28T22:15:52.737ZA Difference in extent metadata is expected here
70592024-02-28T22:15:52.737ZOn loop 32
70602024-02-28T22:15:52.737Z
70612024-02-28T22:15:52.742ZVerifying data now
70622024-02-28T22:15:57.414ZNo change after verify
70632024-02-28T22:15:57.414ZLoop: 32 Downstairs dump after verify (and repair):
70642024-02-28T22:15:57.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70652024-02-28T22:15:57.429Z 0 000-019 65 65 65 120 120 120 F F F
70662024-02-28T22:15:57.429Z 1 020-039 65 65 65 120 120 120 F F F
70672024-02-28T22:15:57.429Z 2 040-059 65 65 65 120 120 120 F F F
70682024-02-28T22:15:57.429Z 3 060-079 62 62 62 112 112 112 F F F
70692024-02-28T22:15:57.430Z 4 080-099 62 62 62 115 115 115 F F F
70702024-02-28T22:15:57.430Z 5 100-119 63 63 63 117 117 117 F F F
70712024-02-28T22:15:57.430Z 6 120-139 65 65 65 120 120 120 F F F
70722024-02-28T22:15:57.430Z 7 140-159 65 65 65 120 120 120 F F F
70732024-02-28T22:15:57.430Z 8 160-179 61 61 61 111 111 111 F F F
70742024-02-28T22:15:57.430Z 9 180-199 63 63 63 117 117 117 F F F
70752024-02-28T22:15:57.430Z 10 200-219 62 62 62 114 114 114 F F F
70762024-02-28T22:15:57.435Z 11 220-239 65 65 65 120 120 120 F F F
70772024-02-28T22:15:57.435Z 12 240-259 62 62 62 115 115 115 F F F
70782024-02-28T22:15:57.435Z 13 260-279 62 62 62 115 115 115 F F F
70792024-02-28T22:15:57.435Z 14 280-299 58 58 58 108 108 108 F F F
70802024-02-28T22:15:57.435Z 15 300-319 62 62 62 112 112 112 F F F
70812024-02-28T22:15:57.435Z 16 320-339 55 55 55 101 101 101 F F F
70822024-02-28T22:15:57.435Z 17 340-359 62 62 62 116 116 116 F F F
70832024-02-28T22:15:57.435Z 18 360-379 64 64 64 118 118 118 F F F
70842024-02-28T22:15:57.435Z 19 380-399 65 65 65 120 120 120 F F F
70852024-02-28T22:15:57.435Z 20 400-419 65 65 65 120 120 120 F F F
70862024-02-28T22:15:57.435Z 21 420-439 64 64 64 118 118 118 F F F
70872024-02-28T22:15:57.435Z 22 440-459 62 62 62 112 112 112 F F F
70882024-02-28T22:15:57.436Z 23 460-479 59 59 59 109 109 109 F F F
70892024-02-28T22:15:57.436Z 24 480-499 65 65 65 120 120 120 F F F
70902024-02-28T22:15:57.436Z 25 500-519 62 62 62 114 114 114 F F F
70912024-02-28T22:15:57.436Z 26 520-539 59 59 59 109 109 109 F F F
70922024-02-28T22:15:57.436Z 27 540-559 58 58 58 106 106 106 F F F
70932024-02-28T22:15:57.436Z 28 560-579 65 65 65 120 120 120 F F F
70942024-02-28T22:15:57.436Z 29 580-599 65 65 65 120 120 120 F F F
70952024-02-28T22:15:57.436ZMax gen: 65, Max flush: 120
70962024-02-28T22:15:57.436Z
70972024-02-28T22:15:57.436ZBegin loop 33
70982024-02-28T22:15:57.436ZDownstairs to restart: 2
70992024-02-28T22:15:57.452Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.346507615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
71002024-02-28T22:15:57.452Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: Crucible runtime is spawned
71012024-02-28T22:15:57.464Z\"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.346887254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
71022024-02-28T22:15:57.465Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.346937003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
71032024-02-28T22:15:57.465Z{"msg":"Crucible 68d385a1-3656-4f92-89b8-8d89d4a77b1d has session id: 593778f8-4d42-4c14-80ad-3b858b36a9f6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.346967492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71042024-02-28T22:15:57.465Z{"msg":"Upstairs opts: Upstairs UUID: 68d385a1-3656-4f92-89b8-8d89d4a77b1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347014111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71052024-02-28T22:15:57.465Z{"msg":"Crucible stats registered with UUID: 68d385a1-3656-4f92-89b8-8d89d4a77b1d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.34704156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71062024-02-28T22:15:57.465Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347068319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
71072024-02-28T22:15:57.465Z{"msg":"68d385a1-3656-4f92-89b8-8d89d4a77b1d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347102698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71082024-02-28T22:15:57.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347138557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71092024-02-28T22:15:57.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347174436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71102024-02-28T22:15:57.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347201066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71112024-02-28T22:15:57.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347249524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71122024-02-28T22:15:57.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347277874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71132024-02-28T22:15:57.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347317133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71142024-02-28T22:15:57.466Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347347112Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71152024-02-28T22:15:57.466Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.34739871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71162024-02-28T22:15:57.466Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347432089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71172024-02-28T22:15:57.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347456778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71182024-02-28T22:15:57.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347486068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71192024-02-28T22:15:57.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.347657053Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71202024-02-28T22:15:57.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348227877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71212024-02-28T22:15:57.467Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348266076Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71222024-02-28T22:15:57.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348282806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71232024-02-28T22:15:57.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348298625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71242024-02-28T22:15:57.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348346074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71252024-02-28T22:15:57.467Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348404512Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71262024-02-28T22:15:57.467Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.34887551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71272024-02-28T22:15:57.468Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348911958Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71282024-02-28T22:15:57.468Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.348982416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71292024-02-28T22:15:57.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349450754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71302024-02-28T22:15:57.468Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349487143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71312024-02-28T22:15:57.468Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349503012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71322024-02-28T22:15:57.469Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349612179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71332024-02-28T22:15:57.469Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349658658Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71342024-02-28T22:15:57.469Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349693347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71352024-02-28T22:15:57.469Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349725596Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71362024-02-28T22:15:57.469Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349759025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71372024-02-28T22:15:57.469Z{"msg":"[0]R flush_numbers[0..12]: [120, 120, 120, 112, 115, 117, 120, 120, 111, 117, 114, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349794414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71382024-02-28T22:15:57.470Z{"msg":"[0]R generation[0..12]: [65, 65, 65, 62, 62, 63, 65, 65, 61, 63, 62, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349828823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71392024-02-28T22:15:57.470ZWait for a query_work_queue command to finish before sending IO
71402024-02-28T22:15:57.470Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349871302Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71412024-02-28T22:15:57.471Z{"msg":"[1]R flush_numbers[0..12]: [120, 120, 120, 112, 115, 117, 120, 120, 111, 117, 114, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349911281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71422024-02-28T22:15:57.471Z{"msg":"[1]R generation[0..12]: [65, 65, 65, 62, 62, 63, 65, 65, 61, 63, 62, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.34994686Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71432024-02-28T22:15:57.471Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.349984639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71442024-02-28T22:15:57.471Z{"msg":"[2]R flush_numbers[0..12]: [120, 120, 120, 112, 115, 117, 120, 120, 111, 117, 114, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350042037Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71452024-02-28T22:15:57.472Z{"msg":"[2]R generation[0..12]: [65, 65, 65, 62, 62, 63, 65, 65, 61, 63, 62, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350082626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71462024-02-28T22:15:57.472Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350123095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71472024-02-28T22:15:57.472Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350162214Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71482024-02-28T22:15:57.472Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350200573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71492024-02-28T22:15:57.473Z{"msg":"Next flush: 121","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71502024-02-28T22:15:57.473Zcrucible","level":30,"time":"2024-02-28T22:15:56.350240792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71512024-02-28T22:15:57.473Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350273511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"":"downstairs","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71522024-02-28T22:15:57.473Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350289271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71532024-02-28T22:15:57.473Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.35030941Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"0","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71542024-02-28T22:15:57.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350324279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"1","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71552024-02-28T22:15:57.474Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350344929Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"client":"2","":"downstairs","session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71562024-02-28T22:15:57.474Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350358779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71572024-02-28T22:15:57.474Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350371108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71582024-02-28T22:15:57.475Z{"msg":"68d385a1-3656-4f92-89b8-8d89d4a77b1d is now active with session: 593778f8-4d42-4c14-80ad-3b858b36a9f6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350391318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71592024-02-28T22:15:57.475Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350403647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399,"session_id":"593778f8-4d42-4c14-80ad-3b858b36a9f6"}
71602024-02-28T22:15:57.475Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:15:56.350423027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
71612024-02-28T22:15:57.490ZLoading write count information from file "/tmp/test_repair_verify.data"
71622024-02-28T22:15:57.490ZRun Repair workload
71632024-02-28T22:15:57.490Z01/30 Write block 462 len 28672 data: 5 6 6 6 4 3 5
71642024-02-28T22:15:57.490Z02/30 Flush
71652024-02-28T22:15:57.490Z03/30 Write block 348 len 24576 data: 6 5 5 5 6 7
71662024-02-28T22:15:57.490Z04/30 Flush
71672024-02-28T22:15:57.490Z05/30 Write block 47 len 36864 data: 5 5 6 5 6 3 3 3 4
71682024-02-28T22:15:57.491Z06/30 Read block 404 len 36864
71692024-02-28T22:15:57.491Z07/30 Read block 438 len 40960
71702024-02-28T22:15:57.491Z08/30 Write block 558 len 40960 data: 4 4 4 6 5 5 6 8 9 11
71712024-02-28T22:15:57.491Z09/30 Flush
71722024-02-28T22:15:57.491Z10/30 Flush
71732024-02-28T22:15:57.491Z11/30 Write block 145 len 20480 data: 8 10 9 10 10
71742024-02-28T22:15:57.491Z12/30 Read block 266 len 16384
71752024-02-28T22:15:57.491Z13/30 Write block 229 len 20480 data: 9 8 10 10 9
71762024-02-28T22:15:57.491Z14/30 Write block 286 len 20480 data: 3 2 3 3 4
71772024-02-28T22:15:57.491Z15/30 Read block 166 len 32768
71782024-02-28T22:15:57.491Z16/30 Read block 528 len 16384
71792024-02-28T22:15:57.492Z17/30 Write block 32 len 40960 data: 5 5 4 6 7 8 9 8 7 8
71802024-02-28T22:15:57.492Z18/30 Read block 89 len 28672
71812024-02-28T22:15:57.492Z19/30 Read block 482 len 16384
71822024-02-28T22:15:57.492Z20/30 Write block 184 len 8192 data: 8 7
71832024-02-28T22:15:57.498Z21/30 Flush
71842024-02-28T22:15:57.498Z22/30 Write block 74 len 36864 data: 5 4 4 5 5 4 8 6 9
71852024-02-28T22:15:57.498Z23/30 Write block 109 len 24576 data: 10 9 8 8 5 5
71862024-02-28T22:15:57.498Z24/30 Read block 303 len 8192
71872024-02-28T22:15:57.498Z25/30 Write block 331 len 24576 data: 6 5 5 6 6 5
71882024-02-28T22:15:57.498Z26/30 Write block 326 len 24576 data: 6 7 8 8 9 7
71892024-02-28T22:15:57.498Z27/30 Read block 128 len 36864
71902024-02-28T22:15:57.498Z28/30 Read block 355 len 20480
71912024-02-28T22:15:57.498Z29/30 Write block 450 len 40960 data: 4 5 5 5 4 4 6 5 7 6
71922024-02-28T22:15:57.498Z30/30 Write block 316 len 20480 data: 4 3 4 4 4
71932024-02-28T22:15:57.498Z----------------------------------------------------------------
71942024-02-28T22:15:57.498Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71952024-02-28T22:15:57.498ZStates: Active Active Active
71962024-02-28T22:15:57.498ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71972024-02-28T22:15:57.498Z 1 Acked 1000 Write 7 Done Done Sent false
71982024-02-28T22:15:57.498Z 2 Acked 1001 Flush 0 Done Done Sent false
71992024-02-28T22:15:57.498Z 3 Acked 1002 Write 6 Done Done Sent false
72002024-02-28T22:15:57.498Z 4 Acked 1003 Flush 0 Done Done Sent false
72012024-02-28T22:15:57.498Z 5 Acked 1004 Write 9 Done Done Sent false
72022024-02-28T22:15:57.499Z 6 Acked 1005 Read 9 Done Done Sent false
72032024-02-28T22:15:57.499Z 7 Acked 1006 Read 10 Done Done Sent false
72042024-02-28T22:15:57.499Z 8 Acked 1007 Write 10 Done Done Sent false
72052024-02-28T22:15:57.499Z 9 Acked 1008 Flush 0 Done Done Sent false
72062024-02-28T22:15:57.499Z 10 Acked 1009 Flush 0 Done Done Sent false
72072024-02-28T22:15:57.499Z 11 Acked 1010 Write 5 Done Done Sent false
72082024-02-28T22:15:57.499Z 12 Acked 1011 Read 4 Done Done Sent false
72092024-02-28T22:15:57.499Z 13 Acked 1012 Write 5 Done Done Sent false
72102024-02-28T22:15:57.499Z 14 Acked 1013 Write 5 Done Done Sent false
72112024-02-28T22:15:57.499Z 15 Acked 1014 Read 8 Done Done Sent false
72122024-02-28T22:15:57.499Z 16 Acked 1015 Read 4 Done Done Sent false
72132024-02-28T22:15:57.499Z 17 Acked 1016 Write 10 Done Done Sent false
72142024-02-28T22:15:57.499Z 18 Acked 1017 Read 7 Done Done Sent false
72152024-02-28T22:15:57.499Z 19 Acked 1018 Read 4 Done Done Sent false
72162024-02-28T22:15:57.499Z 20 Acked 1019 Write 2 Done Done Sent false
72172024-02-28T22:15:57.499Z 21 Acked 1020 Flush 0 Done Done Sent false
72182024-02-28T22:15:57.499Z 22 Acked 1021 Write 9 Done Done Sent false
72192024-02-28T22:15:57.499Z 23 Acked 1022 Write 6 Done Done Sent false
72202024-02-28T22:15:57.499Z 24 Acked 1023 Read 2 Done Done Sent false
72212024-02-28T22:15:57.499Z 25 Acked 1024 Write 6 Done Done Sent false
72222024-02-28T22:15:57.499Z 26 Acked 1025 Write 6 Done Done Sent false
72232024-02-28T22:15:57.499Z 27 Acked 1026 Read 9 Done Done Sent false
72242024-02-28T22:15:57.500Z 28 Acked 1027 Read 5 Done Done Sent false
72252024-02-28T22:15:57.500Z 29 Acked 1028 Write 10 Done Sent Sent false
72262024-02-28T22:15:57.500Z 30 Acked 1029 Write 5 Sent Sent Sent false
72272024-02-28T22:15:57.500Z STATES DS:0 DS:1 DS:2 TOTAL
72282024-02-28T22:15:57.500Z New 0 0 0 0
72292024-02-28T22:15:57.500Z Sent 1 2 30 33
72302024-02-28T22:15:57.500Z Done 29 28 0 57
72312024-02-28T22:15:57.500Z Skipped 0 0 0 0
72322024-02-28T22:15:57.500Z Error 0 0 0 0
72332024-02-28T22:15:57.500ZLast Flush: 1020 1020 0
72342024-02-28T22:15:57.500ZDownstairs last five completed:
72352024-02-28T22:15:57.500ZUpstairs last five completed: 30 29 28 27 26
72362024-02-28T22:15:57.500Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:15:56.386406896Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1399}
72372024-02-28T22:15:57.500ZWrote out file "/tmp/test_repair_verify.data"
72382024-02-28T22:15:57.500Z
72392024-02-28T22:15:57.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72402024-02-28T22:15:57.520ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72412024-02-28T22:15:57.520Z 0 000-019 65 65 65 120 120 120 F F F
72422024-02-28T22:15:57.520Z 1 020-039 66 66 65 125 125 120 F F F <---
72432024-02-28T22:15:57.520Z 2 040-059 66 66 65 125 125 120 F F F <---
72442024-02-28T22:15:57.520Z 3 060-079 62 62 62 112 112 112 T T F <---
72452024-02-28T22:15:57.520Z 4 080-099 62 62 62 115 115 115 T T F <---
72462024-02-28T22:15:57.520Z 5 100-119 63 63 63 117 117 117 T T F <---
72472024-02-28T22:15:57.520Z 6 120-139 65 65 65 120 120 120 F F F
72482024-02-28T22:15:57.520Z 7 140-159 66 66 65 125 125 120 F F F <---
72492024-02-28T22:15:57.520Z 8 160-179 61 61 61 111 111 111 F F F
72502024-02-28T22:15:57.520Z 9 180-199 66 66 63 125 125 117 F F F <---
72512024-02-28T22:15:57.523Z 10 200-219 62 62 62 114 114 114 F F F
72522024-02-28T22:15:57.523Z 11 220-239 66 66 65 125 125 120 F F F <---
72532024-02-28T22:15:57.524Z 12 240-259 62 62 62 115 115 115 F F F
72542024-02-28T22:15:57.524Z 13 260-279 62 62 62 115 115 115 F F F
72552024-02-28T22:15:57.524Z 14 280-299 66 66 58 125 125 108 F F F <---
72562024-02-28T22:15:57.524Z 15 300-319 62 62 62 112 112 112 T T F <---
72572024-02-28T22:15:57.524Z 16 320-339 55 55 55 101 101 101 T T F <---
72582024-02-28T22:15:57.524Z 17 340-359 66 66 62 122 122 116 F F F <---
72592024-02-28T22:15:57.524Z 18 360-379 64 64 64 118 118 118 F F F
72602024-02-28T22:15:57.524Z 19 380-399 65 65 65 120 120 120 F F F
72612024-02-28T22:15:57.524Z 20 400-419 65 65 65 120 120 120 F F F
72622024-02-28T22:15:57.524Z 21 420-439 64 64 64 118 118 118 F F F
72632024-02-28T22:15:57.524Z 22 440-459 62 62 62 112 112 112 T T F <---
72642024-02-28T22:15:57.524Z 23 460-479 66 66 59 121 121 109 F F F <---
72652024-02-28T22:15:57.524Z 24 480-499 65 65 65 120 120 120 F F F
72662024-02-28T22:15:57.524Z 25 500-519 62 62 62 114 114 114 F F F
72672024-02-28T22:15:57.524Z 26 520-539 59 59 59 109 109 109 F F F
72682024-02-28T22:15:57.524Z 27 540-559 66 66 58 123 123 106 F F F <---
72692024-02-28T22:15:57.524Z 28 560-579 66 66 65 123 123 120 F F F <---
72702024-02-28T22:15:57.524Z 29 580-599 65 65 65 120 120 120 F F F
72712024-02-28T22:15:57.524ZMax gen: 66, Max flush: 125
72722024-02-28T22:15:57.524ZError: Difference in extent metadata found!
72732024-02-28T22:15:57.524ZA Difference in extent metadata is expected here
72742024-02-28T22:15:57.524ZOn loop 33
72752024-02-28T22:15:57.525Z
72762024-02-28T22:15:57.525ZVerifying data now
72772024-02-28T22:16:02.303ZNo change after verify
72782024-02-28T22:16:02.303ZLoop: 33 Downstairs dump after verify (and repair):
72792024-02-28T22:16:02.318ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72802024-02-28T22:16:02.318Z 0 000-019 65 65 65 120 120 120 F F F
72812024-02-28T22:16:02.318Z 1 020-039 66 66 66 125 125 125 F F F
72822024-02-28T22:16:02.318Z 2 040-059 66 66 66 125 125 125 F F F
72832024-02-28T22:16:02.318Z 3 060-079 67 67 67 126 126 126 F F F
72842024-02-28T22:16:02.319Z 4 080-099 67 67 67 126 126 126 F F F
72852024-02-28T22:16:02.319Z 5 100-119 67 67 67 126 126 126 F F F
72862024-02-28T22:16:02.319Z 6 120-139 65 65 65 120 120 120 F F F
72872024-02-28T22:16:02.319Z 7 140-159 66 66 66 125 125 125 F F F
72882024-02-28T22:16:02.319Z 8 160-179 61 61 61 111 111 111 F F F
72892024-02-28T22:16:02.319Z 9 180-199 66 66 66 125 125 125 F F F
72902024-02-28T22:16:02.319Z 10 200-219 62 62 62 114 114 114 F F F
72912024-02-28T22:16:02.319Z 11 220-239 66 66 66 125 125 125 F F F
72922024-02-28T22:16:02.322Z 12 240-259 62 62 62 115 115 115 F F F
72932024-02-28T22:16:02.322Z 13 260-279 62 62 62 115 115 115 F F F
72942024-02-28T22:16:02.323Z 14 280-299 66 66 66 125 125 125 F F F
72952024-02-28T22:16:02.323Z 15 300-319 67 67 67 126 126 126 F F F
72962024-02-28T22:16:02.323Z 16 320-339 67 67 67 126 126 126 F F F
72972024-02-28T22:16:02.323Z 17 340-359 66 66 66 122 122 122 F F F
72982024-02-28T22:16:02.323Z 18 360-379 64 64 64 118 118 118 F F F
72992024-02-28T22:16:02.323Z 19 380-399 65 65 65 120 120 120 F F F
73002024-02-28T22:16:02.323Z 20 400-419 65 65 65 120 120 120 F F F
73012024-02-28T22:16:02.323Z 21 420-439 64 64 64 118 118 118 F F F
73022024-02-28T22:16:02.323Z 22 440-459 67 67 67 126 126 126 F F F
73032024-02-28T22:16:02.323Z 23 460-479 66 66 66 121 121 121 F F F
73042024-02-28T22:16:02.323Z 24 480-499 65 65 65 120 120 120 F F F
73052024-02-28T22:16:02.323Z 25 500-519 62 62 62 114 114 114 F F F
73062024-02-28T22:16:02.323Z 26 520-539 59 59 59 109 109 109 F F F
73072024-02-28T22:16:02.323Z 27 540-559 66 66 66 123 123 123 F F F
73082024-02-28T22:16:02.323Z 28 560-579 66 66 66 123 123 123 F F F
73092024-02-28T22:16:02.323Z 29 580-599 65 65 65 120 120 120 F F F
73102024-02-28T22:16:02.323ZMax gen: 67, Max flush: 126
73112024-02-28T22:16:02.323Z
73122024-02-28T22:16:02.323ZBegin loop 34
73132024-02-28T22:16:02.323ZDownstairs to restart: 0
73142024-02-28T22:16:02.341Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234089856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
73152024-02-28T22:16:02.341Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
73162024-02-28T22:16:02.341Z,"time":"2024-02-28T22:16:01.234418638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
73172024-02-28T22:16:02.351Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234463556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
73182024-02-28T22:16:02.351Z{"msg":"Crucible 8da8850d-f7ed-43cd-bf5a-308f891a32e9 has session id: 4a7a0342-c037-48b8-a825-c90d3c5e95cf","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234508885Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73192024-02-28T22:16:02.351Z{"msg":"Upstairs opts: Upstairs UUID: 8da8850d-f7ed-43cd-bf5a-308f891a32e9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234549744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73202024-02-28T22:16:02.351Z{"msg":"Crucible stats registered with UUID: 8da8850d-f7ed-43cd-bf5a-308f891a32e9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234584303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73212024-02-28T22:16:02.351Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234603132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
73222024-02-28T22:16:02.351Z{"msg":"8da8850d-f7ed-43cd-bf5a-308f891a32e9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234627822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73232024-02-28T22:16:02.351Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234662651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73242024-02-28T22:16:02.351Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.23469643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73252024-02-28T22:16:02.351Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234744689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73262024-02-28T22:16:02.351Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234778488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73272024-02-28T22:16:02.351Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234822186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73282024-02-28T22:16:02.352Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234865245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73292024-02-28T22:16:02.352Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234897054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73302024-02-28T22:16:02.352Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234952163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73312024-02-28T22:16:02.352Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.234992382Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73322024-02-28T22:16:02.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235018501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73332024-02-28T22:16:02.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.23506312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73342024-02-28T22:16:02.352Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235111568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73352024-02-28T22:16:02.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235746641Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73362024-02-28T22:16:02.352Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235796569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73372024-02-28T22:16:02.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235827599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73382024-02-28T22:16:02.352Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.235865878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73392024-02-28T22:16:02.352Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236031293Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73402024-02-28T22:16:02.352Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236063982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73412024-02-28T22:16:02.352Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236427872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73422024-02-28T22:16:02.352Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236557139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73432024-02-28T22:16:02.353Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236807542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73442024-02-28T22:16:02.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.236989097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73452024-02-28T22:16:02.353Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237041495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73462024-02-28T22:16:02.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237086184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73472024-02-28T22:16:02.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237111553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73482024-02-28T22:16:02.353Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237136103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73492024-02-28T22:16:02.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237151972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73502024-02-28T22:16:02.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237370036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73512024-02-28T22:16:02.353Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237415505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73522024-02-28T22:16:02.353Z{"msg":"[0]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 126, 120, 125, 111, 125, 114, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237441814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73532024-02-28T22:16:02.353Z{"msg":"[0]R generation[0..12]: [65, 66, 66, 67, 67, 67, 65, 66, 61, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237477423Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73542024-02-28T22:16:02.353Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237513192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73552024-02-28T22:16:02.353Z{"msg":"[1]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 126, 120, 125, 111, 125, 114, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237550661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73562024-02-28T22:16:02.353ZWait for a query_work_queue command to finish before sending IO
73572024-02-28T22:16:02.353Z{"msg":"[1]R generation[0..12]: [65, 66, 66, 67, 67, 67, 65, 66, 61, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.23759262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73582024-02-28T22:16:02.353Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237635349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73592024-02-28T22:16:02.353Z{"msg":"[2]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 126, 120, 125, 111, 125, 114, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237673698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73602024-02-28T22:16:02.353Z{"msg":"[2]R generation[0..12]: [65, 66, 66, 67, 67, 67, 65, 66, 61, 66, 62, 66]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237719456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73612024-02-28T22:16:02.353Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237763325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73622024-02-28T22:16:02.353Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237800614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73632024-02-28T22:16:02.353Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237836263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73642024-02-28T22:16:02.353Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237871492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73652024-02-28T22:16:02.353Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237892142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"":"downstairs","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73662024-02-28T22:16:02.354Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237906191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73672024-02-28T22:16:02.354ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73682024-02-28T22:16:02.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237923171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"0","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73692024-02-28T22:16:02.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.23794486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"1","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73702024-02-28T22:16:02.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.23795872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"client":"2","":"downstairs","session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73712024-02-28T22:16:02.354Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237973649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73722024-02-28T22:16:02.354Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.237993169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73732024-02-28T22:16:02.354Z{"msg":"8da8850d-f7ed-43cd-bf5a-308f891a32e9 is now active with session: 4a7a0342-c037-48b8-a825-c90d3c5e95cf","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.238006789Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73742024-02-28T22:16:02.354Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.238019888Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407,"session_id":"4a7a0342-c037-48b8-a825-c90d3c5e95cf"}
73752024-02-28T22:16:02.354Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:01.238039358Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
73762024-02-28T22:16:02.354ZLoading write count information from file "/tmp/test_repair_verify.data"
73772024-02-28T22:16:02.354ZRun Repair workload
73782024-02-28T22:16:02.354Z01/30 Write block 597 len 12288 data: 6 6 5
73792024-02-28T22:16:02.361Z02/30 Write block 508 len 28672 data: 9 9 8 6 5 4 5
73802024-02-28T22:16:02.361Z03/30 Write block 503 len 16384 data: 9 10 9 9
73812024-02-28T22:16:02.361Z04/30 Read block 514 len 40960
73822024-02-28T22:16:02.361Z05/30 Read block 63 len 16384
73832024-02-28T22:16:02.361Z06/30 Read block 97 len 32768
73842024-02-28T22:16:02.361Z07/30 Flush
73852024-02-28T22:16:02.361Z08/30 Write block 238 len 20480 data: 6 5 5 5 4
73862024-02-28T22:16:02.361Z09/30 Flush
73872024-02-28T22:16:02.361Z10/30 Write block 311 len 8192 data: 7 7
73882024-02-28T22:16:02.365Z11/30 Write block 327 len 40960 data: 8 9 9 10 8 6 6 7 7 6
73892024-02-28T22:16:02.365Z12/30 Read block 166 len 36864
73902024-02-28T22:16:02.366Z13/30 Read block 463 len 32768
73912024-02-28T22:16:02.366Z14/30 Read block 477 len 20480
73922024-02-28T22:16:02.366Z15/30 Write block 497 len 32768 data: 8 7 7 8 8 8 10 11
73932024-02-28T22:16:02.366Z16/30 Read block 399 len 12288
73942024-02-28T22:16:02.366Z17/30 Read block 21 len 40960
73952024-02-28T22:16:02.375Z18/30 Write block 400 len 40960 data: 6 6 7 9 8 7 7 8 8 7
73962024-02-28T22:16:02.375Z19/30 Write block 342 len 8192 data: 7 6
73972024-02-28T22:16:02.375Z20/30 Write block 231 len 16384 data: 11 11 10 8
73982024-02-28T22:16:02.375Z21/30 Read block 0 len 12288
73992024-02-28T22:16:02.375Z22/30 Read block 36 len 28672
74002024-02-28T22:16:02.375Z23/30 Read block 563 len 20480
74012024-02-28T22:16:02.375Z24/30 Write block 332 len 36864 data: 7 7 8 8 7 5 7 7 7
74022024-02-28T22:16:02.375Z25/30 Write block 276 len 28672 data: 4 4 4 5 5 4 5
74032024-02-28T22:16:02.383Z26/30 Flush
74042024-02-28T22:16:02.383Z27/30 Write block 108 len 40960 data: 9 11 10 9 9 6 6 6 5 5
74052024-02-28T22:16:02.384Z28/30 Read block 528 len 28672
74062024-02-28T22:16:02.384Z29/30 Write block 326 len 32768 data: 7 9 10 10 11 9 8 8
74072024-02-28T22:16:02.384Z30/30 Read block 152 len 40960
74082024-02-28T22:16:02.399Z----------------------------------------------------------------
74092024-02-28T22:16:02.399Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74102024-02-28T22:16:02.399ZStates: Active Active Active
74112024-02-28T22:16:02.399ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74122024-02-28T22:16:02.399Z 1 Acked 1000 Write 3 Sent Done Done false
74132024-02-28T22:16:02.399Z 2 Acked 1001 Write 7 Sent Done Done false
74142024-02-28T22:16:02.399Z 3 Acked 1002 Write 4 Sent Done Done false
74152024-02-28T22:16:02.399Z 4 Acked 1003 Read 10 Sent Done Done false
74162024-02-28T22:16:02.399Z 5 Acked 1004 Read 4 Sent Done Done false
74172024-02-28T22:16:02.400Z 6 Acked 1005 Read 8 Sent Done Done false
74182024-02-28T22:16:02.400Z 7 Acked 1006 Flush 0 Sent Done Done false
74192024-02-28T22:16:02.400Z 8 Acked 1007 Write 5 Sent Done Done false
74202024-02-28T22:16:02.400Z 9 Acked 1008 Flush 0 Sent Done Done false
74212024-02-28T22:16:02.400Z 10 Acked 1009 Write 2 Sent Done Done false
74222024-02-28T22:16:02.400Z 11 Acked 1010 Write 10 Sent Done Done false
74232024-02-28T22:16:02.400Z 12 Acked 1011 Read 9 Sent Done Done false
74242024-02-28T22:16:02.400Z 13 Acked 1012 Read 8 Sent Done Done false
74252024-02-28T22:16:02.400Z 14 Acked 1013 Read 5 Sent Done Done false
74262024-02-28T22:16:02.400Z 15 Acked 1014 Write 8 Sent Done Done false
74272024-02-28T22:16:02.400Z 16 Acked 1015 Read 3 Sent Done Done false
74282024-02-28T22:16:02.400Z 17 Acked 1016 Read 10 Sent Done Done false
74292024-02-28T22:16:02.400Z 18 Acked 1017 Write 10 Sent Done Done false
74302024-02-28T22:16:02.400Z 19 Acked 1018 Write 2 Sent Done Done false
74312024-02-28T22:16:02.400Z 20 Acked 1019 Write 4 Sent Done Done false
74322024-02-28T22:16:02.400Z 21 Acked 1020 Read 3 Sent Done Done false
74332024-02-28T22:16:02.400Z 22 Acked 1021 Read 7 Sent Done Done false
74342024-02-28T22:16:02.400Z 23 Acked 1022 Read 5 Sent Done Done false
74352024-02-28T22:16:02.400Z 24 Acked 1023 Write 9 Sent Done Done false
74362024-02-28T22:16:02.400Z 25 Acked 1024 Write 7 Sent Done Done false
74372024-02-28T22:16:02.400Z 26 Acked 1025 Flush 0 Sent Done Done false
74382024-02-28T22:16:02.400Z 27 Acked 1026 Write 10 Sent Done Done false
74392024-02-28T22:16:02.400Z 28 Acked 1027 Read 7 Sent Done Done false
74402024-02-28T22:16:02.401Z 29 Acked 1028 Write 8 Sent Done Done false
74412024-02-28T22:16:02.401Z 30 Acked 1029 Read 10 Sent Done Sent false
74422024-02-28T22:16:02.401Z STATES DS:0 DS:1 DS:2 TOTAL
74432024-02-28T22:16:02.401Z New 0 0 0 0
74442024-02-28T22:16:02.401Z Sent 30 0 1 31
74452024-02-28T22:16:02.401Z Done 0 30 29 59
74462024-02-28T22:16:02.401Z Skipped 0 0 0 0
74472024-02-28T22:16:02.401Z Error 0 0 0 0
74482024-02-28T22:16:02.401ZLast Flush: 0 1025 1025
74492024-02-28T22:16:02.401ZDownstairs last five completed:
74502024-02-28T22:16:02.401ZUpstairs last five completed: 30 29 28 27 26
74512024-02-28T22:16:02.401Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:01.27535116Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1407}
74522024-02-28T22:16:02.401ZWrote out file "/tmp/test_repair_verify.data"
74532024-02-28T22:16:02.401Z
74542024-02-28T22:16:02.401ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74552024-02-28T22:16:02.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74562024-02-28T22:16:02.415Z 0 000-019 65 65 65 120 120 120 F F F
74572024-02-28T22:16:02.415Z 1 020-039 66 66 66 125 125 125 F F F
74582024-02-28T22:16:02.416Z 2 040-059 66 66 66 125 125 125 F F F
74592024-02-28T22:16:02.416Z 3 060-079 67 67 67 126 126 126 F F F
74602024-02-28T22:16:02.416Z 4 080-099 67 67 67 126 126 126 F F F
74612024-02-28T22:16:02.416Z 5 100-119 67 67 67 126 126 126 F T T <---
74622024-02-28T22:16:02.416Z 6 120-139 65 65 65 120 120 120 F F F
74632024-02-28T22:16:02.416Z 7 140-159 66 66 66 125 125 125 F F F
74642024-02-28T22:16:02.416Z 8 160-179 61 61 61 111 111 111 F F F
74652024-02-28T22:16:02.416Z 9 180-199 66 66 66 125 125 125 F F F
74662024-02-28T22:16:02.416Z 10 200-219 62 62 62 114 114 114 F F F
74672024-02-28T22:16:02.416Z 11 220-239 66 68 68 125 129 129 F F F <---
74682024-02-28T22:16:02.416Z 12 240-259 62 68 68 115 128 128 F F F <---
74692024-02-28T22:16:02.416Z 13 260-279 62 68 68 115 129 129 F F F <---
74702024-02-28T22:16:02.416Z 14 280-299 66 68 68 125 129 129 F F F <---
74712024-02-28T22:16:02.416Z 15 300-319 67 68 68 126 129 129 F F F <---
74722024-02-28T22:16:02.416Z 16 320-339 67 68 68 126 129 129 F T T <---
74732024-02-28T22:16:02.416Z 17 340-359 66 68 68 122 129 129 F F F <---
74742024-02-28T22:16:02.416Z 18 360-379 64 64 64 118 118 118 F F F
74752024-02-28T22:16:02.416Z 19 380-399 65 65 65 120 120 120 F F F
74762024-02-28T22:16:02.416Z 20 400-419 65 68 68 120 129 129 F F F <---
74772024-02-28T22:16:02.416Z 21 420-439 64 64 64 118 118 118 F F F
74782024-02-28T22:16:02.416Z 22 440-459 67 67 67 126 126 126 F F F
74792024-02-28T22:16:02.416Z 23 460-479 66 66 66 121 121 121 F F F
74802024-02-28T22:16:02.416Z 24 480-499 65 68 68 120 129 129 F F F <---
74812024-02-28T22:16:02.417Z 25 500-519 62 68 68 114 129 129 F F F <---
74822024-02-28T22:16:02.417Z 26 520-539 59 59 59 109 109 109 F F F
74832024-02-28T22:16:02.417Z 27 540-559 66 66 66 123 123 123 F F F
74842024-02-28T22:16:02.417Z 28 560-579 66 66 66 123 123 123 F F F
74852024-02-28T22:16:02.417Z 29 580-599 65 68 68 120 127 127 F F F <---
74862024-02-28T22:16:02.417ZMax gen: 68, Max flush: 129
74872024-02-28T22:16:02.417ZError: Difference in extent metadata found!
74882024-02-28T22:16:02.417ZA Difference in extent metadata is expected here
74892024-02-28T22:16:02.417ZOn loop 34
74902024-02-28T22:16:02.417Z
74912024-02-28T22:16:02.435ZVerifying data now
74922024-02-28T22:16:07.027ZNo change after verify
74932024-02-28T22:16:07.027ZLoop: 34 Downstairs dump after verify (and repair):
74942024-02-28T22:16:07.040ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74952024-02-28T22:16:07.041Z 0 000-019 65 65 65 120 120 120 F F F
74962024-02-28T22:16:07.041Z 1 020-039 66 66 66 125 125 125 F F F
74972024-02-28T22:16:07.041Z 2 040-059 66 66 66 125 125 125 F F F
74982024-02-28T22:16:07.041Z 3 060-079 67 67 67 126 126 126 F F F
74992024-02-28T22:16:07.041Z 4 080-099 67 67 67 126 126 126 F F F
75002024-02-28T22:16:07.041Z 5 100-119 69 69 69 130 130 130 F F F
75012024-02-28T22:16:07.041Z 6 120-139 65 65 65 120 120 120 F F F
75022024-02-28T22:16:07.041Z 7 140-159 66 66 66 125 125 125 F F F
75032024-02-28T22:16:07.041Z 8 160-179 61 61 61 111 111 111 F F F
75042024-02-28T22:16:07.041Z 9 180-199 66 66 66 125 125 125 F F F
75052024-02-28T22:16:07.047Z 10 200-219 62 62 62 114 114 114 F F F
75062024-02-28T22:16:07.048Z 11 220-239 68 68 68 129 129 129 F F F
75072024-02-28T22:16:07.048Z 12 240-259 68 68 68 128 128 128 F F F
75082024-02-28T22:16:07.048Z 13 260-279 68 68 68 129 129 129 F F F
75092024-02-28T22:16:07.048Z 14 280-299 68 68 68 129 129 129 F F F
75102024-02-28T22:16:07.048Z 15 300-319 68 68 68 129 129 129 F F F
75112024-02-28T22:16:07.048Z 16 320-339 69 69 69 130 130 130 F F F
75122024-02-28T22:16:07.048Z 17 340-359 68 68 68 129 129 129 F F F
75132024-02-28T22:16:07.048Z 18 360-379 64 64 64 118 118 118 F F F
75142024-02-28T22:16:07.048Z 19 380-399 65 65 65 120 120 120 F F F
75152024-02-28T22:16:07.048Z 20 400-419 68 68 68 129 129 129 F F F
75162024-02-28T22:16:07.048Z 21 420-439 64 64 64 118 118 118 F F F
75172024-02-28T22:16:07.048Z 22 440-459 67 67 67 126 126 126 F F F
75182024-02-28T22:16:07.048Z 23 460-479 66 66 66 121 121 121 F F F
75192024-02-28T22:16:07.048Z 24 480-499 68 68 68 129 129 129 F F F
75202024-02-28T22:16:07.048Z 25 500-519 68 68 68 129 129 129 F F F
75212024-02-28T22:16:07.048Z 26 520-539 59 59 59 109 109 109 F F F
75222024-02-28T22:16:07.048Z 27 540-559 66 66 66 123 123 123 F F F
75232024-02-28T22:16:07.048Z 28 560-579 66 66 66 123 123 123 F F F
75242024-02-28T22:16:07.048Z 29 580-599 68 68 68 127 127 127 F F F
75252024-02-28T22:16:07.048ZMax gen: 69, Max flush: 130
75262024-02-28T22:16:07.048Z
75272024-02-28T22:16:07.049ZBegin loop 35
75282024-02-28T22:16:07.049ZDownstairs to restart: 1
75292024-02-28T22:16:07.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95536218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
75302024-02-28T22:16:07.062Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.955627073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
75312024-02-28T22:16:07.062Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.955654272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
75322024-02-28T22:16:07.062Z{"msg":"Crucible a09c8276-a16b-44a9-854d-19c776952645 has session id: 3dbea8f6-d4c6-43af-8371-70a3d7af6819","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.955687771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75332024-02-28T22:16:07.069Z{"msg":"Upstairs opts: Upstairs UUID: a09c8276-a16b-44a9-854d-19c776952645, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95573175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75342024-02-28T22:16:07.069Z{"msg":"Crucible stats registered with UUID: a09c8276-a16b-44a9-854d-19c776952645","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.955756439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"Crucible runtime is spawned
75352024-02-28T22:16:07.069Zsession_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75362024-02-28T22:16:07.069Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.955883176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
75372024-02-28T22:16:07.069Z{"msg":"a09c8276-a16b-44a9-854d-19c776952645 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956000883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75382024-02-28T22:16:07.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956041071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75392024-02-28T22:16:07.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956071721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75402024-02-28T22:16:07.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95610601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75412024-02-28T22:16:07.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956130209Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75422024-02-28T22:16:07.069Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956153328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75432024-02-28T22:16:07.069Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956186418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75442024-02-28T22:16:07.069Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956210087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75452024-02-28T22:16:07.069Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956245626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75462024-02-28T22:16:07.069Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956271445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75472024-02-28T22:16:07.069Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956438431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75482024-02-28T22:16:07.069Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956489249Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75492024-02-28T22:16:07.070Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.956518038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75502024-02-28T22:16:07.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95717537Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75512024-02-28T22:16:07.070Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957219799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75522024-02-28T22:16:07.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957258718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75532024-02-28T22:16:07.070Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957289947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75542024-02-28T22:16:07.070Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957333796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75552024-02-28T22:16:07.070Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957374915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75562024-02-28T22:16:07.070Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957805543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75572024-02-28T22:16:07.070Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.957842872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75582024-02-28T22:16:07.070Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958042496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75592024-02-28T22:16:07.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958361347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75602024-02-28T22:16:07.070Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958411616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75612024-02-28T22:16:07.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958439956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75622024-02-28T22:16:07.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958461725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75632024-02-28T22:16:07.070Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958483234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75642024-02-28T22:16:07.070Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958516403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75652024-02-28T22:16:07.070Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958540073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75662024-02-28T22:16:07.070Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958568272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75672024-02-28T22:16:07.070Z{"msg":"[0]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 130, 120, 125, 111, 125, 114, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958593971Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75682024-02-28T22:16:07.070Z{"msg":"[0]R generation[0..12]: [65, 66, 66, 67, 67, 69, 65, 66, 61, 66, 62, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95862012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75692024-02-28T22:16:07.070Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95865405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75702024-02-28T22:16:07.070Z{"msg":"[1]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 130, 120, 125, 111, 125, 114, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958680559Z","hostname":"Wait for a query_work_queue command to finish before sending IO
75712024-02-28T22:16:07.070Zip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75722024-02-28T22:16:07.071Z{"msg":"[1]R generation[0..12]: [65, 66, 66, 67, 67, 69, 65, 66, 61, 66, 62, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958722288Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75732024-02-28T22:16:07.071Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958756717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75742024-02-28T22:16:07.071Z{"msg":"[2]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 130, 120, 125, 111, 125, 114, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958783256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75752024-02-28T22:16:07.071Z{"msg":"[2]R generation[0..12]: [65, 66, 66, 67, 67, 69, 65, 66, 61, 66, 62, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958815145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75762024-02-28T22:16:07.071Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958841574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75772024-02-28T22:16:07.071Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958864504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75782024-02-28T22:16:07.071Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958893643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75792024-02-28T22:16:07.071Z{"msg":"Next flush: 131","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958910063Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75802024-02-28T22:16:07.071Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958940712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"":"downstairs","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75812024-02-28T22:16:07.071Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.958959481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75822024-02-28T22:16:07.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.95898659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"0","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75832024-02-28T22:16:07.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959030229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"1","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75842024-02-28T22:16:07.071Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959051968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"client":"2","":"downstairs","session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75852024-02-28T22:16:07.071Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959080568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75862024-02-28T22:16:07.071Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959097487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75872024-02-28T22:16:07.071Z{"msg":"a09c8276-a16b-44a9-854d-19c776952645 is now active with session: 3dbea8f6-d4c6-43af-8371-70a3d7af6819","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959125366Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75882024-02-28T22:16:07.071Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959141506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415,"session_id":"3dbea8f6-d4c6-43af-8371-70a3d7af6819"}
75892024-02-28T22:16:07.071ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75902024-02-28T22:16:07.071Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:05.959172315Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
75912024-02-28T22:16:07.071ZLoading write count information from file "/tmp/test_repair_verify.data"
75922024-02-28T22:16:07.071ZRun Repair workload
75932024-02-28T22:16:07.098Z01/30 Write block 489 len 16384 data: 5 6 7 6
75942024-02-28T22:16:07.098Z02/30 Read block 139 len 4096
75952024-02-28T22:16:07.098Z03/30 Read block 117 len 28672
75962024-02-28T22:16:07.098Z04/30 Write block 201 len 28672 data: 3 4 3 3 5 4 4
75972024-02-28T22:16:07.098Z05/30 Flush
75982024-02-28T22:16:07.098Z06/30 Write block 435 len 40960 data: 8 9 9 10 9 7 8 8 5 6
75992024-02-28T22:16:07.098Z07/30 Write block 193 len 40960 data: 9 8 8 6 6 6 4 2 4 5
76002024-02-28T22:16:07.098Z08/30 Write block 593 len 28672 data: 9 8 8 7 7 7 6
76012024-02-28T22:16:07.098Z09/30 Read block 430 len 40960
76022024-02-28T22:16:07.098Z10/30 Write block 369 len 16384 data: 6 6 8 10
76032024-02-28T22:16:07.098Z11/30 Write block 127 len 8192 data: 9 10
76042024-02-28T22:16:07.098Z12/30 Read block 358 len 28672
76052024-02-28T22:16:07.098Z13/30 Read block 51 len 16384
76062024-02-28T22:16:07.098Z14/30 Write block 311 len 24576 data: 8 8 6 6 4 5
76072024-02-28T22:16:07.099Z15/30 Read block 352 len 24576
76082024-02-28T22:16:07.099Z16/30 Write block 256 len 8192 data: 8 9
76092024-02-28T22:16:07.099Z17/30 Read block 171 len 36864
76102024-02-28T22:16:07.099Z18/30 Read block 74 len 16384
76112024-02-28T22:16:07.099Z19/30 Write block 208 len 20480 data: 4 4 5 6 5
76122024-02-28T22:16:07.099Z20/30 Flush
76132024-02-28T22:16:07.099Z21/30 Write block 263 len 20480 data: 6 6 5 6 5
76142024-02-28T22:16:07.099Z22/30 Write block 303 len 4096 data: 5
76152024-02-28T22:16:07.099Z23/30 Read block 118 len 16384
76162024-02-28T22:16:07.099Z24/30 Write block 100 len 12288 data: 4 4 4
76172024-02-28T22:16:07.099Z25/30 Read block 12 len 8192
76182024-02-28T22:16:07.099Z26/30 Write block 411 len 36864 data: 6 6 7 9 9 8 5 8 8
76192024-02-28T22:16:07.099Z27/30 Read block 548 len 20480
76202024-02-28T22:16:07.099Z28/30 Write block 189 len 20480 data: 6 6 7 10 10
76212024-02-28T22:16:07.099Z29/30 Write block 386 len 12288 data: 8 6 5
76222024-02-28T22:16:07.099Z30/30 Write block 253 len 4096 data: 8
76232024-02-28T22:16:07.099Z----------------------------------------------------------------
76242024-02-28T22:16:07.099Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:25
76252024-02-28T22:16:07.099ZStates: Active Active Active
76262024-02-28T22:16:07.099ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76272024-02-28T22:16:07.099Z 6 Acked 1005 Write 10 Done Done Done false
76282024-02-28T22:16:07.099Z 7 Acked 1006 Write 10 Done Sent Done false
76292024-02-28T22:16:07.099Z 8 Acked 1007 Write 7 Done Sent Done false
76302024-02-28T22:16:07.100Z 9 Acked 1008 Read 10 Done Sent Done false
76312024-02-28T22:16:07.100Z 10 Acked 1009 Write 4 Done Sent Done false
76322024-02-28T22:16:07.100Z 11 Acked 1010 Write 2 Done Sent Done false
76332024-02-28T22:16:07.100Z 12 Acked 1011 Read 7 Done Sent Done false
76342024-02-28T22:16:07.100Z 13 Acked 1012 Read 4 Done Sent Done false
76352024-02-28T22:16:07.100Z 14 Acked 1013 Write 6 Done Sent Done false
76362024-02-28T22:16:07.100Z 15 Acked 1014 Read 6 Done Sent Done false
76372024-02-28T22:16:07.100Z 16 Acked 1015 Write 2 Done Sent Done false
76382024-02-28T22:16:07.100Z 17 Acked 1016 Read 9 Done Sent Done false
76392024-02-28T22:16:07.100Z 18 Acked 1017 Read 4 Done Sent Done false
76402024-02-28T22:16:07.100Z 19 Acked 1018 Write 5 Done Sent Done false
76412024-02-28T22:16:07.100Z 20 Acked 1019 Flush 0 Done Sent Done false
76422024-02-28T22:16:07.100Z 21 Acked 1020 Write 5 Done Sent Done false
76432024-02-28T22:16:07.100Z 22 Acked 1021 Write 1 Done Sent Done false
76442024-02-28T22:16:07.100Z 23 Acked 1022 Read 4 Done Sent Done false
76452024-02-28T22:16:07.100Z 24 Acked 1023 Write 3 Done Sent Done false
76462024-02-28T22:16:07.100Z 25 Acked 1024 Read 2 Done Sent Done false
76472024-02-28T22:16:07.100Z 26 Acked 1025 Write 9 Done Sent Done false
76482024-02-28T22:16:07.100Z 27 Acked 1026 Read 5 Done Sent Done false
76492024-02-28T22:16:07.100Z 28 Acked 1027 Write 5 Done Sent Done false
76502024-02-28T22:16:07.100Z 29 Acked 1028 Write 3 Sent Sent Sent false
76512024-02-28T22:16:07.100Z 30 Acked 1029 Write 1 Sent Sent Sent false
76522024-02-28T22:16:07.100Z STATES DS:0 DS:1 DS:2 TOTAL
76532024-02-28T22:16:07.101Z New 0 0 0 0
76542024-02-28T22:16:07.101Z Sent 2 24 2 28
76552024-02-28T22:16:07.101Z Done 23 1 23 47
76562024-02-28T22:16:07.101Z Skipped 0 0 0 0
76572024-02-28T22:16:07.101Z Error 0 0 0 0
76582024-02-28T22:16:07.101ZLast Flush: 1019 1004 1019
76592024-02-28T22:16:07.101ZDownstairs last five completed: 1004 1003 1002 1001 1000
76602024-02-28T22:16:07.101ZUpstairs last five completed: 30 29 28 27 26
76612024-02-28T22:16:07.101Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:05.990359026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1415}
76622024-02-28T22:16:07.101ZWrote out file "/tmp/test_repair_verify.data"
76632024-02-28T22:16:07.105Z
76642024-02-28T22:16:07.105ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76652024-02-28T22:16:07.121ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76662024-02-28T22:16:07.121Z 0 000-019 65 65 65 120 120 120 F F F
76672024-02-28T22:16:07.121Z 1 020-039 66 66 66 125 125 125 F F F
76682024-02-28T22:16:07.121Z 2 040-059 66 66 66 125 125 125 F F F
76692024-02-28T22:16:07.121Z 3 060-079 67 67 67 126 126 126 F F F
76702024-02-28T22:16:07.122Z 4 080-099 67 67 67 126 126 126 F F F
76712024-02-28T22:16:07.122Z 5 100-119 69 69 69 130 130 130 T F T <---
76722024-02-28T22:16:07.122Z 6 120-139 70 65 70 132 120 132 F F F <---
76732024-02-28T22:16:07.122Z 7 140-159 66 66 66 125 125 125 F F F
76742024-02-28T22:16:07.125Z 8 160-179 61 61 61 111 111 111 F F F
76752024-02-28T22:16:07.125Z 9 180-199 70 66 70 132 125 132 T F T <---
76762024-02-28T22:16:07.125Z 10 200-219 70 70 70 132 131 132 F F F <---
76772024-02-28T22:16:07.125Z 11 220-239 68 68 68 129 129 129 F F F
76782024-02-28T22:16:07.125Z 12 240-259 70 68 70 132 128 132 T F T <---
76792024-02-28T22:16:07.125Z 13 260-279 68 68 68 129 129 129 T F T <---
76802024-02-28T22:16:07.125Z 14 280-299 68 68 68 129 129 129 F F F
76812024-02-28T22:16:07.125Z 15 300-319 70 68 70 132 129 132 T F T <---
76822024-02-28T22:16:07.125Z 16 320-339 69 69 69 130 130 130 F F F
76832024-02-28T22:16:07.125Z 17 340-359 68 68 68 129 129 129 F F F
76842024-02-28T22:16:07.126Z 18 360-379 70 64 70 132 118 132 F F F <---
76852024-02-28T22:16:07.126Z 19 380-399 65 65 65 120 120 120 T F T <---
76862024-02-28T22:16:07.126Z 20 400-419 68 68 68 129 129 129 T F T <---
76872024-02-28T22:16:07.126Z 21 420-439 70 64 70 132 118 132 F T F <---
76882024-02-28T22:16:07.126Z 22 440-459 70 67 70 132 126 132 F T F <---
76892024-02-28T22:16:07.126Z 23 460-479 66 66 66 121 121 121 F F F
76902024-02-28T22:16:07.126Z 24 480-499 70 70 70 131 131 131 F F F
76912024-02-28T22:16:07.126Z 25 500-519 68 68 68 129 129 129 F F F
76922024-02-28T22:16:07.126Z 26 520-539 59 59 59 109 109 109 F F F
76932024-02-28T22:16:07.126Z 27 540-559 66 66 66 123 123 123 F F F
76942024-02-28T22:16:07.126Z 28 560-579 66 66 66 123 123 123 F F F
76952024-02-28T22:16:07.126Z 29 580-599 70 68 70 132 127 132 F F F <---
76962024-02-28T22:16:07.126ZMax gen: 70, Max flush: 132
76972024-02-28T22:16:07.126ZError: Difference in extent metadata found!
76982024-02-28T22:16:07.126ZA Difference in extent metadata is expected here
76992024-02-28T22:16:07.126ZOn loop 35
77002024-02-28T22:16:07.126Z
77012024-02-28T22:16:07.129ZVerifying data now
77022024-02-28T22:16:12.791ZNo change after verify
77032024-02-28T22:16:12.791ZLoop: 35 Downstairs dump after verify (and repair):
77042024-02-28T22:16:12.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77052024-02-28T22:16:12.805Z 0 000-019 65 65 65 120 120 120 F F F
77062024-02-28T22:16:12.805Z 1 020-039 66 66 66 125 125 125 F F F
77072024-02-28T22:16:12.805Z 2 040-059 66 66 66 125 125 125 F F F
77082024-02-28T22:16:12.805Z 3 060-079 67 67 67 126 126 126 F F F
77092024-02-28T22:16:12.805Z 4 080-099 67 67 67 126 126 126 F F F
77102024-02-28T22:16:12.805Z 5 100-119 71 71 71 133 133 133 F F F
77112024-02-28T22:16:12.805Z 6 120-139 70 70 70 132 132 132 F F F
77122024-02-28T22:16:12.805Z 7 140-159 66 66 66 125 125 125 F F F
77132024-02-28T22:16:12.805Z 8 160-179 61 61 61 111 111 111 F F F
77142024-02-28T22:16:12.808Z 9 180-199 71 71 71 133 133 133 F F F
77152024-02-28T22:16:12.808Z 10 200-219 70 70 70 132 132 132 F F F
77162024-02-28T22:16:12.808Z 11 220-239 68 68 68 129 129 129 F F F
77172024-02-28T22:16:12.808Z 12 240-259 71 71 71 133 133 133 F F F
77182024-02-28T22:16:12.808Z 13 260-279 71 71 71 133 133 133 F F F
77192024-02-28T22:16:12.809Z 14 280-299 68 68 68 129 129 129 F F F
77202024-02-28T22:16:12.809Z 15 300-319 71 71 71 133 133 133 F F F
77212024-02-28T22:16:12.809Z 16 320-339 69 69 69 130 130 130 F F F
77222024-02-28T22:16:12.809Z 17 340-359 68 68 68 129 129 129 F F F
77232024-02-28T22:16:12.809Z 18 360-379 70 70 70 132 132 132 F F F
77242024-02-28T22:16:12.809Z 19 380-399 71 71 71 133 133 133 F F F
77252024-02-28T22:16:12.809Z 20 400-419 71 71 71 133 133 133 F F F
77262024-02-28T22:16:12.809Z 21 420-439 70 70 70 132 132 132 F F F
77272024-02-28T22:16:12.809Z 22 440-459 70 70 70 132 132 132 F F F
77282024-02-28T22:16:12.809Z 23 460-479 66 66 66 121 121 121 F F F
77292024-02-28T22:16:12.809Z 24 480-499 70 70 70 131 131 131 F F F
77302024-02-28T22:16:12.809Z 25 500-519 68 68 68 129 129 129 F F F
77312024-02-28T22:16:12.809Z 26 520-539 59 59 59 109 109 109 F F F
77322024-02-28T22:16:12.809Z 27 540-559 66 66 66 123 123 123 F F F
77332024-02-28T22:16:12.809Z 28 560-579 66 66 66 123 123 123 F F F
77342024-02-28T22:16:12.809Z 29 580-599 70 70 70 132 132 132 F F F
77352024-02-28T22:16:12.809ZMax gen: 71, Max flush: 133
77362024-02-28T22:16:12.809Z
77372024-02-28T22:16:12.809ZBegin loop 36
77382024-02-28T22:16:12.809ZDownstairs to restart: 2
77392024-02-28T22:16:12.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719015627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
77402024-02-28T22:16:12.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
77412024-02-28T22:16:12.826Z,"time":"2024-02-28T22:16:11.719336958Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
77422024-02-28T22:16:12.826Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719373257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
77432024-02-28T22:16:12.832Z{"msg":"Crucible 61513192-7836-4b85-b17e-d030a0872eda has session id: 90e1200d-4a79-4b95-a99c-8b7b50f7809f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719401176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77442024-02-28T22:16:12.832Z{"msg":"Upstairs opts: Upstairs UUID: 61513192-7836-4b85-b17e-d030a0872eda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719454555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77452024-02-28T22:16:12.832Z{"msg":"Crucible stats registered with UUID: 61513192-7836-4b85-b17e-d030a0872eda","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719497284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77462024-02-28T22:16:12.832Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719526503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
77472024-02-28T22:16:12.832Z{"msg":"61513192-7836-4b85-b17e-d030a0872eda active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719590041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77482024-02-28T22:16:12.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.71962616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77492024-02-28T22:16:12.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719669009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77502024-02-28T22:16:12.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719712908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77512024-02-28T22:16:12.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719761026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77522024-02-28T22:16:12.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719789406Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77532024-02-28T22:16:12.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719814245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77542024-02-28T22:16:12.832Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719852274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77552024-02-28T22:16:12.832Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719894713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77562024-02-28T22:16:12.832Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.719928522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77572024-02-28T22:16:12.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.72001183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77582024-02-28T22:16:12.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720043579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77592024-02-28T22:16:12.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720071448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77602024-02-28T22:16:12.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.72072833Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77612024-02-28T22:16:12.833Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720770089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77622024-02-28T22:16:12.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720786038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77632024-02-28T22:16:12.833Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720801968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77642024-02-28T22:16:12.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.720951054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77652024-02-28T22:16:12.833Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721004542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77662024-02-28T22:16:12.833Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721385442Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77672024-02-28T22:16:12.833Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.72144334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77682024-02-28T22:16:12.833Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721661874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77692024-02-28T22:16:12.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721899218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77702024-02-28T22:16:12.833Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721948696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77712024-02-28T22:16:12.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721974996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77722024-02-28T22:16:12.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.721997865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77732024-02-28T22:16:12.833Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722032404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77742024-02-28T22:16:12.833Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722056263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77752024-02-28T22:16:12.833Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.72219357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77762024-02-28T22:16:12.833Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722230278Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77772024-02-28T22:16:12.834Z{"msg":"[0]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 133, 132, 125, 111, 133, 132, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722247668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77782024-02-28T22:16:12.834Z{"msg":"[0]R generation[0..12]: [65, 66, 66, 67, 67, 71, 70, 66, 61, 71, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722270357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77792024-02-28T22:16:12.834Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722300856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77802024-02-28T22:16:12.834Z{"msg":"[1]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 133, 132, 125, 111, 133, 132, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722316556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77812024-02-28T22:16:12.834Z{"msg"Wait for a query_work_queue command to finish before sending IO
77822024-02-28T22:16:12.834Z:"[1]R generation[0..12]: [65, 66, 66, 67, 67, 71, 70, 66, 61, 71, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722350715Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77832024-02-28T22:16:12.834Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722368765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77842024-02-28T22:16:12.834Z{"msg":"[2]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 133, 132, 125, 111, 133, 132, 129]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722400234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77852024-02-28T22:16:12.834Z{"msg":"[2]R generation[0..12]: [65, 66, 66, 67, 67, 71, 70, 66, 61, 71, 70, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722415923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77862024-02-28T22:16:12.834Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722449272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77872024-02-28T22:16:12.834Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722474792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77882024-02-28T22:16:12.834Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722508821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77892024-02-28T22:16:12.834Z{"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.72253342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77902024-02-28T22:16:12.834Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722564069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"":"downstairs","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77912024-02-28T22:16:12.834Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722588269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77922024-02-28T22:16:12.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722610248Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"0","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77932024-02-28T22:16:12.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722645837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"1","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77942024-02-28T22:16:12.834Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722671526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"client":"2","":"downstairs","session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77952024-02-28T22:16:12.834Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722702356Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77962024-02-28T22:16:12.847Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722723765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77972024-02-28T22:16:12.847Z{"msg":"61513192-7836-4b85-b17e-d030a0872eda is now active with session: 90e1200d-4a79-4b95-a99c-8b7b50f7809f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722748154Z","hostnameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77982024-02-28T22:16:12.847Z":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
77992024-02-28T22:16:12.847Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722775483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423,"session_id":"90e1200d-4a79-4b95-a99c-8b7b50f7809f"}
78002024-02-28T22:16:12.847Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:11.722803223Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
78012024-02-28T22:16:12.847ZLoading write count information from file "/tmp/test_repair_verify.data"
78022024-02-28T22:16:12.847ZRun Repair workload
78032024-02-28T22:16:12.847Z01/30 Write block 377 len 40960 data: 6 7 7 7 8 12 11 12 10 9
78042024-02-28T22:16:12.847Z02/30 Write block 345 len 4096 data: 5
78052024-02-28T22:16:12.847Z03/30 Write block 413 len 36864 data: 8 10 10 9 6 9 9 9 9
78062024-02-28T22:16:12.848Z04/30 Read block 578 len 32768
78072024-02-28T22:16:12.848Z05/30 Write block 472 len 12288 data: 8 8 9
78082024-02-28T22:16:12.848Z06/30 Write block 114 len 12288 data: 7 7 6
78092024-02-28T22:16:12.848Z07/30 Read block 310 len 4096
78102024-02-28T22:16:12.848Z08/30 Write block 328 len 16384 data: 11 11 12 10
78112024-02-28T22:16:12.848Z09/30 Write block 167 len 28672 data: 11 9 10 10 9 9 9
78122024-02-28T22:16:12.848Z10/30 Write block 117 len 12288 data: 6 4 4
78132024-02-28T22:16:12.851Z11/30 Read block 145 len 8192
78142024-02-28T22:16:12.851Z12/30 Read block 528 len 16384
78152024-02-28T22:16:12.851Z13/30 Write block 286 len 16384 data: 4 3 4 4
78162024-02-28T22:16:12.852Z14/30 Read block 573 len 4096
78172024-02-28T22:16:12.852Z15/30 Write block 345 len 8192 data: 6 5
78182024-02-28T22:16:12.852Z16/30 Read block 209 len 24576
78192024-02-28T22:16:12.852Z17/30 Write block 397 len 24576 data: 6 6 5 7 7 8
78202024-02-28T22:16:12.852Z18/30 Read block 225 len 12288
78212024-02-28T22:16:12.852Z19/30 Read block 530 len 8192
78222024-02-28T22:16:12.852Z20/30 Read block 189 len 40960
78232024-02-28T22:16:12.852Z21/30 Write block 144 len 36864 data: 7 9 11 10 11 11 8 6 5
78242024-02-28T22:16:12.852Z22/30 Read block 62 len 32768
78252024-02-28T22:16:12.852Z23/30 Read block 415 len 40960
78262024-02-28T22:16:12.852Z24/30 Read block 321 len 36864
78272024-02-28T22:16:12.852Z25/30 Read block 325 len 36864
78282024-02-28T22:16:12.852Z26/30 Read block 196 len 4096
78292024-02-28T22:16:12.856Z27/30 Write block 237 len 8192 data: 4 7
78302024-02-28T22:16:12.856Z28/30 Read block 308 len 28672
78312024-02-28T22:16:12.856Z29/30 Read block 373 len 24576
78322024-02-28T22:16:12.856Z30/30 Write block 166 len 24576 data: 8 12 10 11 11 10
78332024-02-28T22:16:12.856Z----------------------------------------------------------------
78342024-02-28T22:16:12.856Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78352024-02-28T22:16:12.856ZStates: Active Active Active
78362024-02-28T22:16:12.856ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78372024-02-28T22:16:12.856Z 1 Acked 1000 Write 10 Done Done Done false
78382024-02-28T22:16:12.856Z 2 Acked 1001 Write 1 Done Done Done false
78392024-02-28T22:16:12.856Z 3 Acked 1002 Write 9 Done Done Sent false
78402024-02-28T22:16:12.856Z 4 Acked 1003 Read 8 Done Done Sent false
78412024-02-28T22:16:12.856Z 5 Acked 1004 Write 3 Done Done Sent false
78422024-02-28T22:16:12.856Z 6 Acked 1005 Write 3 Done Done Sent false
78432024-02-28T22:16:12.856Z 7 Acked 1006 Read 1 Done Done Sent false
78442024-02-28T22:16:12.856Z 8 Acked 1007 Write 4 Done Done Sent false
78452024-02-28T22:16:12.856Z 9 Acked 1008 Write 7 Done Done Sent false
78462024-02-28T22:16:12.856Z 10 Acked 1009 Write 3 Done Done Sent false
78472024-02-28T22:16:12.857Z 11 Acked 1010 Read 2 Done Done Sent false
78482024-02-28T22:16:12.857Z 12 Acked 1011 Read 4 Done Done Sent false
78492024-02-28T22:16:12.857Z 13 Acked 1012 Write 4 Done Done Sent false
78502024-02-28T22:16:12.857Z 14 Acked 1013 Read 1 Done Done Sent false
78512024-02-28T22:16:12.857Z 15 Acked 1014 Write 2 Done Done Sent false
78522024-02-28T22:16:12.857Z 16 Acked 1015 Read 6 Done Done Sent false
78532024-02-28T22:16:12.857Z 17 Acked 1016 Write 6 Done Done Sent false
78542024-02-28T22:16:12.857Z 18 Acked 1017 Read 3 Done Done Sent false
78552024-02-28T22:16:12.857Z 19 Acked 1018 Read 2 Done Done Sent false
78562024-02-28T22:16:12.857Z 20 Acked 1019 Read 10 Done Done Sent false
78572024-02-28T22:16:12.857Z 21 Acked 1020 Write 9 Done Done Sent false
78582024-02-28T22:16:12.857Z 22 Acked 1021 Read 8 Done Done Sent false
78592024-02-28T22:16:12.857Z 23 Acked 1022 Read 10 Done Done Sent false
78602024-02-28T22:16:12.857Z 24 Acked 1023 Read 9 Done Done Sent false
78612024-02-28T22:16:12.857Z 25 Acked 1024 Read 9 Done Done Sent false
78622024-02-28T22:16:12.857Z 26 Acked 1025 Read 1 Done Done Sent false
78632024-02-28T22:16:12.857Z 27 Acked 1026 Write 2 Done Done Sent false
78642024-02-28T22:16:12.857Z 28 Acked 1027 Read 7 Done Done Sent false
78652024-02-28T22:16:12.857Z 29 Acked 1028 Read 6 Done Done Sent false
78662024-02-28T22:16:12.857Z 30 Acked 1029 Write 6 Sent Sent Sent false
78672024-02-28T22:16:12.857Z STATES DS:0 DS:1 DS:2 TOTAL
78682024-02-28T22:16:12.857Z New 0 0 0 0
78692024-02-28T22:16:12.857Z Sent 1 1 28 30
78702024-02-28T22:16:12.857Z Done 29 29 2 60
78712024-02-28T22:16:12.858Z Skipped 0 0 0 0
78722024-02-28T22:16:12.858Z Error 0 0 0 0
78732024-02-28T22:16:12.858ZLast Flush: 0 0 0
78742024-02-28T22:16:12.858ZDownstairs last five completed:
78752024-02-28T22:16:12.858ZUpstairs last five completed: 30 29 28 27 26
78762024-02-28T22:16:12.861Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:11.749340662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1423}
78772024-02-28T22:16:12.861ZWrote out file "/tmp/test_repair_verify.data"
78782024-02-28T22:16:12.861Z
78792024-02-28T22:16:12.864ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78802024-02-28T22:16:12.883ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78812024-02-28T22:16:12.883Z 0 000-019 65 65 65 120 120 120 F F F
78822024-02-28T22:16:12.883Z 1 020-039 66 66 66 125 125 125 F F F
78832024-02-28T22:16:12.883Z 2 040-059 66 66 66 125 125 125 F F F
78842024-02-28T22:16:12.884Z 3 060-079 67 67 67 126 126 126 F F F
78852024-02-28T22:16:12.884Z 4 080-099 67 67 67 126 126 126 F F F
78862024-02-28T22:16:12.884Z 5 100-119 71 71 71 133 133 133 T T F <---
78872024-02-28T22:16:12.884Z 6 120-139 70 70 70 132 132 132 F F F
78882024-02-28T22:16:12.884Z 7 140-159 66 66 66 125 125 125 T T F <---
78892024-02-28T22:16:12.884Z 8 160-179 61 61 61 111 111 111 T T F <---
78902024-02-28T22:16:12.884Z 9 180-199 71 71 71 133 133 133 F F F
78912024-02-28T22:16:12.884Z 10 200-219 70 70 70 132 132 132 F F F
78922024-02-28T22:16:12.884Z 11 220-239 68 68 68 129 129 129 T T F <---
78932024-02-28T22:16:12.884Z 12 240-259 71 71 71 133 133 133 F F F
78942024-02-28T22:16:12.887Z 13 260-279 71 71 71 133 133 133 F F F
78952024-02-28T22:16:12.887Z 14 280-299 68 68 68 129 129 129 T T F <---
78962024-02-28T22:16:12.887Z 15 300-319 71 71 71 133 133 133 F F F
78972024-02-28T22:16:12.887Z 16 320-339 69 69 69 130 130 130 T T F <---
78982024-02-28T22:16:12.887Z 17 340-359 68 68 68 129 129 129 T T T
78992024-02-28T22:16:12.887Z 18 360-379 70 70 70 132 132 132 T T T
79002024-02-28T22:16:12.887Z 19 380-399 71 71 71 133 133 133 T T T
79012024-02-28T22:16:12.888Z 20 400-419 71 71 71 133 133 133 T T F <---
79022024-02-28T22:16:12.888Z 21 420-439 70 70 70 132 132 132 T T F <---
79032024-02-28T22:16:12.888Z 22 440-459 70 70 70 132 132 132 F F F
79042024-02-28T22:16:12.888Z 23 460-479 66 66 66 121 121 121 T T F <---
79052024-02-28T22:16:12.888Z 24 480-499 70 70 70 131 131 131 F F F
79062024-02-28T22:16:12.888Z 25 500-519 68 68 68 129 129 129 F F F
79072024-02-28T22:16:12.888Z 26 520-539 59 59 59 109 109 109 F F F
79082024-02-28T22:16:12.888Z 27 540-559 66 66 66 123 123 123 F F F
79092024-02-28T22:16:12.888Z 28 560-579 66 66 66 123 123 123 F F F
79102024-02-28T22:16:12.888Z 29 580-599 70 70 70 132 132 132 F F F
79112024-02-28T22:16:12.888ZMax gen: 71, Max flush: 133
79122024-02-28T22:16:12.888ZError: Difference in extent metadata found!
79132024-02-28T22:16:12.888ZA Difference in extent metadata is expected here
79142024-02-28T22:16:12.888ZOn loop 36
79152024-02-28T22:16:12.888Z
79162024-02-28T22:16:12.891ZVerifying data now
79172024-02-28T22:16:18.537ZNo change after verify
79182024-02-28T22:16:18.537ZLoop: 36 Downstairs dump after verify (and repair):
79192024-02-28T22:16:18.550ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79202024-02-28T22:16:18.550Z 0 000-019 65 65 65 120 120 120 F F F
79212024-02-28T22:16:18.550Z 1 020-039 66 66 66 125 125 125 F F F
79222024-02-28T22:16:18.550Z 2 040-059 66 66 66 125 125 125 F F F
79232024-02-28T22:16:18.550Z 3 060-079 67 67 67 126 126 126 F F F
79242024-02-28T22:16:18.550Z 4 080-099 67 67 67 126 126 126 F F F
79252024-02-28T22:16:18.550Z 5 100-119 72 72 72 134 134 134 F F F
79262024-02-28T22:16:18.553Z 6 120-139 70 70 70 132 132 132 F F F
79272024-02-28T22:16:18.553Z 7 140-159 72 72 72 134 134 134 F F F
79282024-02-28T22:16:18.553Z 8 160-179 72 72 72 134 134 134 F F F
79292024-02-28T22:16:18.554Z 9 180-199 71 71 71 133 133 133 F F F
79302024-02-28T22:16:18.554Z 10 200-219 70 70 70 132 132 132 F F F
79312024-02-28T22:16:18.554Z 11 220-239 72 72 72 134 134 134 F F F
79322024-02-28T22:16:18.554Z 12 240-259 71 71 71 133 133 133 F F F
79332024-02-28T22:16:18.554Z 13 260-279 71 71 71 133 133 133 F F F
79342024-02-28T22:16:18.554Z 14 280-299 72 72 72 134 134 134 F F F
79352024-02-28T22:16:18.554Z 15 300-319 71 71 71 133 133 133 F F F
79362024-02-28T22:16:18.554Z 16 320-339 72 72 72 134 134 134 F F F
79372024-02-28T22:16:18.554Z 17 340-359 72 72 72 134 134 134 F F F
79382024-02-28T22:16:18.554Z 18 360-379 72 72 72 134 134 134 F F F
79392024-02-28T22:16:18.554Z 19 380-399 72 72 72 134 134 134 F F F
79402024-02-28T22:16:18.554Z 20 400-419 72 72 72 134 134 134 F F F
79412024-02-28T22:16:18.554Z 21 420-439 72 72 72 134 134 134 F F F
79422024-02-28T22:16:18.554Z 22 440-459 70 70 70 132 132 132 F F F
79432024-02-28T22:16:18.554Z 23 460-479 72 72 72 134 134 134 F F F
79442024-02-28T22:16:18.554Z 24 480-499 70 70 70 131 131 131 F F F
79452024-02-28T22:16:18.554Z 25 500-519 68 68 68 129 129 129 F F F
79462024-02-28T22:16:18.554Z 26 520-539 59 59 59 109 109 109 F F F
79472024-02-28T22:16:18.554Z 27 540-559 66 66 66 123 123 123 F F F
79482024-02-28T22:16:18.554Z 28 560-579 66 66 66 123 123 123 F F F
79492024-02-28T22:16:18.554Z 29 580-599 70 70 70 132 132 132 F F F
79502024-02-28T22:16:18.554ZMax gen: 72, Max flush: 134
79512024-02-28T22:16:18.554Z
79522024-02-28T22:16:18.555ZBegin loop 37
79532024-02-28T22:16:18.555ZDownstairs to restart: 1
79542024-02-28T22:16:18.573Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466406312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
79552024-02-28T22:16:18.574Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466706464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
79562024-02-28T22:16:18.990Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466757073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
79572024-02-28T22:16:18.990Z{"msg":"Crucible aa2a53b2-104f-4003-93bb-6954db684c4b has session id: e928e2dd-3a26-4cca-a4f2-323ee181f933","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466800282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79582024-02-28T22:16:18.990Z{"msg":"Upstairs opts: Upstairs UUID: aa2a53b2-104f-4003-93bb-6954db684c4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.46684601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79592024-02-28T22:16:18.990ZCrucible runtime is spawned
79602024-02-28T22:16:18.990Z{"msg":"Crucible stats registered with UUID: aa2a53b2-104f-4003-93bb-6954db684c4b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466902979Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79612024-02-28T22:16:18.990Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.466978517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
79622024-02-28T22:16:18.990Z{"msg":"aa2a53b2-104f-4003-93bb-6954db684c4b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467075074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79632024-02-28T22:16:18.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467108213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79642024-02-28T22:16:18.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467128203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79652024-02-28T22:16:18.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467166851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79662024-02-28T22:16:18.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467197941Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79672024-02-28T22:16:18.990Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.4672281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79682024-02-28T22:16:18.990Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467278608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79692024-02-28T22:16:18.990Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467313218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79702024-02-28T22:16:18.990Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467367856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79712024-02-28T22:16:18.990Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467412015Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79722024-02-28T22:16:18.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467447404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79732024-02-28T22:16:18.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467494902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79742024-02-28T22:16:18.991Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.467614639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79752024-02-28T22:16:18.991Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468080576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79762024-02-28T22:16:18.991Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468138965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79772024-02-28T22:16:18.991Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468231512Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79782024-02-28T22:16:18.991Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468284741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79792024-02-28T22:16:18.991Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.46831318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79802024-02-28T22:16:18.991Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468332199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79812024-02-28T22:16:18.991Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468703079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79822024-02-28T22:16:18.991Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468746968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79832024-02-28T22:16:18.991Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.468892804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79842024-02-28T22:16:18.991Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469288023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79852024-02-28T22:16:18.991Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469336352Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79862024-02-28T22:16:18.991Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469368241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79872024-02-28T22:16:18.991Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469417569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79882024-02-28T22:16:18.991Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469440609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79892024-02-28T22:16:18.991Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469463828Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79902024-02-28T22:16:18.991Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469477818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79912024-02-28T22:16:18.991Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469498627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79922024-02-28T22:16:18.991Z{"msg":"[0]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 134, 132, 134, 134, 133, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469513197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79932024-02-28T22:16:18.991Z{"msg":"[0]R generation[0..12]: [65, 66, 66, 67, 67, 72, 70, 72, 72, 71, 70, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469533866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79942024-02-28T22:16:18.991Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469548636Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79952024-02-28T22:16:18.991Z{"msg":"[1]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 134, 132, 134, 134, 133, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469569365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79962024-02-28T22:16:18.991Z{"msg":"[1]R generation[0..12]: [65, 66, 66, 67, 67, 72, 70, 72, 72, 71, 70, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469583785Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79972024-02-28T22:16:18.992Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469604734Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
79982024-02-28T22:16:18.992Z{"msg":"[2]R flush_numbers[0..12]: [120, 125, 125, 126, 126, 134, 132, 134, 134, 133, 132, 134]Wait for a query_work_queue command to finish before sending IO
79992024-02-28T22:16:18.992Z","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469635794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80002024-02-28T22:16:18.992Z{"msg":"[2]R generation[0..12]: [65, 66, 66, 67, 67, 72, 70, 72, 72, 71, 70, 72]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469650893Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80012024-02-28T22:16:18.992Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469665643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80022024-02-28T22:16:18.992Z{"msg":"Max found gen is 73","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469686032Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80032024-02-28T22:16:18.992Z{"msg":"Generation requested: 74 >= found:73","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469699972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80042024-02-28T22:16:18.992Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469712911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80052024-02-28T22:16:18.992Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469733171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"":"downstairs","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80062024-02-28T22:16:18.992Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.46974728Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80072024-02-28T22:16:18.992Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.46975991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"0","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80082024-02-28T22:16:18.992Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.46977329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"1","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80092024-02-28T22:16:18.992Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469794399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"client":"2","":"downstairs","session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80102024-02-28T22:16:18.992Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469808209Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80112024-02-28T22:16:18.992Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469827228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80122024-02-28T22:16:18.992Z{"msg":"aa2a53b2-104f-4003-93bb-6954db684c4b is now active with session: e928e2dd-3a26-4cca-a4f2-323ee181f933","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469840288Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80132024-02-28T22:16:18.992Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469858927Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431,"session_id":"e928e2dd-3a26-4cca-a4f2-323ee181f933"}
80142024-02-28T22:16:18.992Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:17.469871567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
80152024-02-28T22:16:18.992ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80162024-02-28T22:16:18.992ZLoading write count information from file "/tmp/test_repair_verify.data"
80172024-02-28T22:16:18.992ZRun Repair workload
80182024-02-28T22:16:18.992Z01/30 Write block 16 len 4096 data: 7
80192024-02-28T22:16:18.998Z02/30 Read block 480 len 28672
80202024-02-28T22:16:18.998Z03/30 Read block 580 len 28672
80212024-02-28T22:16:18.999Z04/30 Write block 12 len 32768 data: 9 8 7 7 8 7 8 8
80222024-02-28T22:16:18.999Z05/30 Read block 516 len 32768
80232024-02-28T22:16:18.999Z06/30 Flush
80242024-02-28T22:16:18.999Z07/30 Write block 27 len 12288 data: 5 5 6
80252024-02-28T22:16:18.999Z08/30 Read block 575 len 16384
80262024-02-28T22:16:18.999Z09/30 Read block 410 len 4096
80272024-02-28T22:16:18.999Z10/30 Read block 140 len 36864
80282024-02-28T22:16:18.999Z11/30 Read block 183 len 12288
80292024-02-28T22:16:18.999Z12/30 Read block 159 len 28672
80302024-02-28T22:16:18.999Z13/30 Write block 234 len 12288 data: 9 5 4
80312024-02-28T22:16:18.999Z14/30 Read block 155 len 16384
80322024-02-28T22:16:18.999Z15/30 Read block 509 len 12288
80332024-02-28T22:16:18.999Z16/30 Read block 410 len 36864
80342024-02-28T22:16:18.999Z17/30 Read block 575 len 4096
80352024-02-28T22:16:18.999Z18/30 Read block 443 len 24576
80362024-02-28T22:16:18.999Z19/30 Flush
80372024-02-28T22:16:18.999Z20/30 Flush
80382024-02-28T22:16:18.999Z21/30 Write block 7 len 28672 data: 10 10 9 8 8 10 9
80392024-02-28T22:16:18.999Z22/30 Read block 146 len 28672
80402024-02-28T22:16:18.999Z23/30 Write block 509 len 20480 data: 10 9 7 6 5
80412024-02-28T22:16:18.999Z24/30 Read block 269 len 20480
80422024-02-28T22:16:18.999Z25/30 Read block 560 len 36864
80432024-02-28T22:16:18.999Z26/30 Write block 101 len 8192 data: 5 5
80442024-02-28T22:16:19.000Z27/30 Write block 452 len 8192 data: 6 6
80452024-02-28T22:16:19.000Z28/30 Read block 264 len 32768
80462024-02-28T22:16:19.000Z29/30 Write block 534 len 28672 data: 7 5 6 5 5 5 5
80472024-02-28T22:16:19.000Z30/30 Write block 555 len 40960 data: 5 5 5 5 5 5 7 6 6 7
80482024-02-28T22:16:19.000Z----------------------------------------------------------------
80492024-02-28T22:16:19.000Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
80502024-02-28T22:16:19.000ZStates: Active Active Active
80512024-02-28T22:16:19.000ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80522024-02-28T22:16:19.000Z 1 Acked 1000 Write 1 Done Sent Done false
80532024-02-28T22:16:19.000Z 2 Acked 1001 Read 7 Done Sent Done false
80542024-02-28T22:16:19.000Z 3 Acked 1002 Read 7 Done Sent Done false
80552024-02-28T22:16:19.000Z 4 Acked 1003 Write 8 Done Sent Done false
80562024-02-28T22:16:19.000Z 5 Acked 1004 Read 8 Done Sent Done false
80572024-02-28T22:16:19.000Z 6 Acked 1005 Flush 0 Done Sent Done false
80582024-02-28T22:16:19.000Z 7 Acked 1006 Write 3 Done Sent Done false
80592024-02-28T22:16:19.000Z 8 Acked 1007 Read 4 Done Sent Done false
80602024-02-28T22:16:19.000Z 9 Acked 1008 Read 1 Done Sent Done false
80612024-02-28T22:16:19.000Z 10 Acked 1009 Read 9 Done Sent Done false
80622024-02-28T22:16:19.001Z 11 Acked 1010 Read 3 Done Sent Done false
80632024-02-28T22:16:19.001Z 12 Acked 1011 Read 7 Done Sent Done false
80642024-02-28T22:16:19.001Z 13 Acked 1012 Write 3 Done Sent Done false
80652024-02-28T22:16:19.001Z 14 Acked 1013 Read 4 Done Sent Done false
80662024-02-28T22:16:19.001Z 15 Acked 1014 Read 3 Done Sent Done false
80672024-02-28T22:16:19.001Z 16 Acked 1015 Read 9 Done Sent Done false
80682024-02-28T22:16:19.001Z 17 Acked 1016 Read 1 Done Sent Done false
80692024-02-28T22:16:19.001Z 18 Acked 1017 Read 6 Done Sent Done false
80702024-02-28T22:16:19.001Z 19 Acked 1018 Flush 0 Done Sent Done false
80712024-02-28T22:16:19.001Z 20 Acked 1019 Flush 0 Done Sent Done false
80722024-02-28T22:16:19.001Z 21 Acked 1020 Write 7 Done Sent Done false
80732024-02-28T22:16:19.001Z 22 Acked 1021 Read 7 Done Sent Done false
80742024-02-28T22:16:19.001Z 23 Acked 1022 Write 5 Done Sent Done false
80752024-02-28T22:16:19.002Z 24 Acked 1023 Read 5 Done Sent Done false
80762024-02-28T22:16:19.002Z 25 Acked 1024 Read 9 Done Sent Done false
80772024-02-28T22:16:19.002Z 26 Acked 1025 Write 2 Done Sent Done false
80782024-02-28T22:16:19.002Z 27 Acked 1026 Write 2 Done Sent Done false
80792024-02-28T22:16:19.002Z 28 Acked 1027 Read 8 Done Sent Done false
80802024-02-28T22:16:19.002Z 29 Acked 1028 Write 7 Done Sent Done false
80812024-02-28T22:16:19.002Z 30 Acked 1029 Write 10 Sent Sent Sent false
80822024-02-28T22:16:19.002Z STATES DS:0 DS:1 DS:2 TOTAL
80832024-02-28T22:16:19.002Z New 0 0 0 0
80842024-02-28T22:16:19.002Z Sent 1 30 1 32
80852024-02-28T22:16:19.002Z Done 29 0 29 58
80862024-02-28T22:16:19.002Z Skipped 0 0 0 0
80872024-02-28T22:16:19.002Z Error 0 0 0 0
80882024-02-28T22:16:19.002ZLast Flush: 1019 0 1019
80892024-02-28T22:16:19.002ZDownstairs last five completed:
80902024-02-28T22:16:19.002ZUpstairs last five completed: 30 29 28 27 26
80912024-02-28T22:16:19.002Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:17.498652724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1431}
80922024-02-28T22:16:19.002ZWrote out file "/tmp/test_repair_verify.data"
80932024-02-28T22:16:19.003Z
80942024-02-28T22:16:19.003ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80952024-02-28T22:16:19.003ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80962024-02-28T22:16:19.003Z 0 000-019 74 65 74 135 120 135 T F T <---
80972024-02-28T22:16:19.003Z 1 020-039 74 66 74 136 125 136 F F F <---
80982024-02-28T22:16:19.003Z 2 040-059 66 66 66 125 125 125 F F F
80992024-02-28T22:16:19.003Z 3 060-079 67 67 67 126 126 126 F F F
81002024-02-28T22:16:19.003Z 4 080-099 67 67 67 126 126 126 F F F
81012024-02-28T22:16:19.003Z 5 100-119 72 72 72 134 134 134 T F T <---
81022024-02-28T22:16:19.003Z 6 120-139 70 70 70 132 132 132 F F F
81032024-02-28T22:16:19.003Z 7 140-159 72 72 72 134 134 134 F F F
81042024-02-28T22:16:19.003Z 8 160-179 72 72 72 134 134 134 F F F
81052024-02-28T22:16:19.003Z 9 180-199 71 71 71 133 133 133 F F F
81062024-02-28T22:16:19.003Z 10 200-219 70 70 70 132 132 132 F F F
81072024-02-28T22:16:19.003Z 11 220-239 74 72 74 136 134 136 F F F <---
81082024-02-28T22:16:19.003Z 12 240-259 71 71 71 133 133 133 F F F
81092024-02-28T22:16:19.003Z 13 260-279 71 71 71 133 133 133 F F F
81102024-02-28T22:16:19.003Z 14 280-299 72 72 72 134 134 134 F F F
81112024-02-28T22:16:19.003Z 15 300-319 71 71 71 133 133 133 F F F
81122024-02-28T22:16:19.003Z 16 320-339 72 72 72 134 134 134 F F F
81132024-02-28T22:16:19.003Z 17 340-359 72 72 72 134 134 134 F F F
81142024-02-28T22:16:19.003Z 18 360-379 72 72 72 134 134 134 F F F
81152024-02-28T22:16:19.003Z 19 380-399 72 72 72 134 134 134 F F F
81162024-02-28T22:16:19.003Z 20 400-419 72 72 72 134 134 134 F F F
81172024-02-28T22:16:19.004Z 21 420-439 72 72 72 134 134 134 F F F
81182024-02-28T22:16:19.004Z 22 440-459 70 70 70 132 132 132 T F T <---
81192024-02-28T22:16:19.009Z 23 460-479 72 72 72 134 134 134 F F F
81202024-02-28T22:16:19.009Z 24 480-499 70 70 70 131 131 131 F F F
81212024-02-28T22:16:19.009Z 25 500-519 68 68 68 129 129 129 T F T <---
81222024-02-28T22:16:19.009Z 26 520-539 59 59 59 109 109 109 T F T <---
81232024-02-28T22:16:19.009Z 27 540-559 66 66 66 123 123 123 T F T <---
81242024-02-28T22:16:19.009Z 28 560-579 66 66 66 123 123 123 T F T <---
81252024-02-28T22:16:19.009Z 29 580-599 70 70 70 132 132 132 F F F
81262024-02-28T22:16:19.009ZMax gen: 74, Max flush: 136
81272024-02-28T22:16:19.009ZError: Difference in extent metadata found!
81282024-02-28T22:16:19.009ZA Difference in extent metadata is expected here
81292024-02-28T22:16:19.009ZOn loop 37
81302024-02-28T22:16:19.009Z
81312024-02-28T22:16:19.009ZVerifying data now
81322024-02-28T22:16:23.148ZNo change after verify
81332024-02-28T22:16:23.149ZLoop: 37 Downstairs dump after verify (and repair):
81342024-02-28T22:16:23.162ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81352024-02-28T22:16:23.162Z 0 000-019 75 75 75 137 137 137 F F F
81362024-02-28T22:16:23.162Z 1 020-039 74 74 74 136 136 136 F F F
81372024-02-28T22:16:23.162Z 2 040-059 66 66 66 125 125 125 F F F
81382024-02-28T22:16:23.163Z 3 060-079 67 67 67 126 126 126 F F F
81392024-02-28T22:16:23.163Z 4 080-099 67 67 67 126 126 126 F F F
81402024-02-28T22:16:23.163Z 5 100-119 75 75 75 137 137 137 F F F
81412024-02-28T22:16:23.163Z 6 120-139 70 70 70 132 132 132 F F F
81422024-02-28T22:16:23.163Z 7 140-159 72 72 72 134 134 134 F F F
81432024-02-28T22:16:23.163Z 8 160-179 72 72 72 134 134 134 F F F
81442024-02-28T22:16:23.163Z 9 180-199 71 71 71 133 133 133 F F F
81452024-02-28T22:16:23.163Z 10 200-219 70 70 70 132 132 132 F F F
81462024-02-28T22:16:23.163Z 11 220-239 74 74 74 136 136 136 F F F
81472024-02-28T22:16:23.168Z 12 240-259 71 71 71 133 133 133 F F F
81482024-02-28T22:16:23.168Z 13 260-279 71 71 71 133 133 133 F F F
81492024-02-28T22:16:23.168Z 14 280-299 72 72 72 134 134 134 F F F
81502024-02-28T22:16:23.168Z 15 300-319 71 71 71 133 133 133 F F F
81512024-02-28T22:16:23.168Z 16 320-339 72 72 72 134 134 134 F F F
81522024-02-28T22:16:23.168Z 17 340-359 72 72 72 134 134 134 F F F
81532024-02-28T22:16:23.168Z 18 360-379 72 72 72 134 134 134 F F F
81542024-02-28T22:16:23.168Z 19 380-399 72 72 72 134 134 134 F F F
81552024-02-28T22:16:23.168Z 20 400-419 72 72 72 134 134 134 F F F
81562024-02-28T22:16:23.168Z 21 420-439 72 72 72 134 134 134 F F F
81572024-02-28T22:16:23.168Z 22 440-459 75 75 75 137 137 137 F F F
81582024-02-28T22:16:23.168Z 23 460-479 72 72 72 134 134 134 F F F
81592024-02-28T22:16:23.168Z 24 480-499 70 70 70 131 131 131 F F F
81602024-02-28T22:16:23.168Z 25 500-519 75 75 75 137 137 137 F F F
81612024-02-28T22:16:23.168Z 26 520-539 75 75 75 137 137 137 F F F
81622024-02-28T22:16:23.168Z 27 540-559 75 75 75 137 137 137 F F F
81632024-02-28T22:16:23.168Z 28 560-579 75 75 75 137 137 137 F F F
81642024-02-28T22:16:23.168Z 29 580-599 70 70 70 132 132 132 F F F
81652024-02-28T22:16:23.168ZMax gen: 75, Max flush: 137
81662024-02-28T22:16:23.169Z
81672024-02-28T22:16:23.169ZBegin loop 38
81682024-02-28T22:16:23.169ZDownstairs to restart: 2
81692024-02-28T22:16:23.182Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.074508244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
81702024-02-28T22:16:23.182Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.074794136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
81712024-02-28T22:16:23.182Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,Crucible runtime is spawned
81722024-02-28T22:16:23.182Z"time":"2024-02-28T22:16:22.074819425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
81732024-02-28T22:16:23.194Z{"msg":"Crucible 2e62c755-10b3-43aa-bdbb-e1699e06dd0e has session id: 01dee3c8-5da2-426d-aa01-cbeee3ad1c02","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.074864724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81742024-02-28T22:16:23.194Z{"msg":"Upstairs opts: Upstairs UUID: 2e62c755-10b3-43aa-bdbb-e1699e06dd0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.074924812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81752024-02-28T22:16:23.194Z{"msg":"Crucible stats registered with UUID: 2e62c755-10b3-43aa-bdbb-e1699e06dd0e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.074966971Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81762024-02-28T22:16:23.195Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.07501386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
81772024-02-28T22:16:23.195Z{"msg":"2e62c755-10b3-43aa-bdbb-e1699e06dd0e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075179575Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81782024-02-28T22:16:23.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075224124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81792024-02-28T22:16:23.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075255773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81802024-02-28T22:16:23.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075307182Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81812024-02-28T22:16:23.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075337271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81822024-02-28T22:16:23.195Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.07537318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81832024-02-28T22:16:23.195Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075391379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81842024-02-28T22:16:23.195Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075411909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81852024-02-28T22:16:23.195Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075449628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81862024-02-28T22:16:23.195Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075485357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81872024-02-28T22:16:23.195Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075555525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81882024-02-28T22:16:23.195Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075585444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81892024-02-28T22:16:23.195Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.075609963Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81902024-02-28T22:16:23.195Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076239746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81912024-02-28T22:16:23.195Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076307834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81922024-02-28T22:16:23.195Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076345803Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81932024-02-28T22:16:23.195Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076376712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81942024-02-28T22:16:23.195Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076508319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81952024-02-28T22:16:23.195Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076558417Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81962024-02-28T22:16:23.195Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.076954566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81972024-02-28T22:16:23.195Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077075223Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81982024-02-28T22:16:23.195Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077267318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
81992024-02-28T22:16:23.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077572829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82002024-02-28T22:16:23.196Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077605808Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82012024-02-28T22:16:23.196Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077621298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82022024-02-28T22:16:23.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077634628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82032024-02-28T22:16:23.196Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077658887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82042024-02-28T22:16:23.196Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077673767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82052024-02-28T22:16:23.196Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077766094Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82062024-02-28T22:16:23.196Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077785884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82072024-02-28T22:16:23.196Z{"msg":"[0]R flush_numbers[0..12]: [137, 136, 125, 126, 126, 137, 132, 134, 134, 133, 132, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077831602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82082024-02-28T22:16:23.196Z{"msg":"[0]R generation[0..12]: [75, 74, 66, 67, 67, 75, 70, 72, 72, 71, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077870931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82092024-02-28T22:16:23.196Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.07789956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82102024-02-28T22:16:23.196Z{"msg":"[1]R flush_numbers[0..12]: [137, 136, 125, 126, 126, 137, 132, 134, 134, 133, 132, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.07793068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82112024-02-28T22:16:23.196ZWait for a query_work_queue command to finish before sending IO
82122024-02-28T22:16:23.196Z{"msg":"[1]R generation[0..12]: [75, 74, 66, 67, 67, 75, 70, 72, 72, 71, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077967119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82132024-02-28T22:16:23.196Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.077992418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82142024-02-28T22:16:23.196Z{"msg":"[2]R flush_numbers[0..12]: [137, 136, 125, 126, 126, 137, 132, 134, 134, 133, 132, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078024907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82152024-02-28T22:16:23.196Z{"msg":"[2]R generation[0..12]: [75, 74, 66, 67, 67, 75, 70, 72, 72, 71, 70, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078041466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82162024-02-28T22:16:23.196Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078061156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82172024-02-28T22:16:23.196Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078092085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82182024-02-28T22:16:23.196Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078116214Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82192024-02-28T22:16:23.196Z{"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078140494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82202024-02-28T22:16:23.196Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078182062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"":"downstairs","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82212024-02-28T22:16:23.196Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078224551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82222024-02-28T22:16:23.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078248161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"0","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82232024-02-28T22:16:23.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.07827812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"1","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82242024-02-28T22:16:23.197Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078309759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"client":"2","":"downstairs","session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82252024-02-28T22:16:23.197ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82262024-02-28T22:16:23.197Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078344718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82272024-02-28T22:16:23.197Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078366088Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82282024-02-28T22:16:23.197Z{"msg":"2e62c755-10b3-43aa-bdbb-e1699e06dd0e is now active with session: 01dee3c8-5da2-426d-aa01-cbeee3ad1c02","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078395967Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82292024-02-28T22:16:23.197Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078418256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439,"session_id":"01dee3c8-5da2-426d-aa01-cbeee3ad1c02"}
82302024-02-28T22:16:23.197Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:22.078444095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
82312024-02-28T22:16:23.197ZLoading write count information from file "/tmp/test_repair_verify.data"
82322024-02-28T22:16:23.197ZRun Repair workload
82332024-02-28T22:16:23.204Z01/30 Flush
82342024-02-28T22:16:23.205Z02/30 Write block 489 len 36864 data: 6 7 8 7 6 5 7 7 9
82352024-02-28T22:16:23.205Z03/30 Read block 471 len 28672
82362024-02-28T22:16:23.205Z04/30 Read block 338 len 36864
82372024-02-28T22:16:23.206Z05/30 Read block 456 len 12288
82382024-02-28T22:16:23.206Z06/30 Read block 498 len 8192
82392024-02-28T22:16:23.206Z07/30 Write block 64 len 36864 data: 8 8 8 10 9 6 7 7 6
82402024-02-28T22:16:23.207Z08/30 Read block 304 len 8192
82412024-02-28T22:16:23.207Z09/30 Write block 573 len 16384 data: 8 6 7 6
82422024-02-28T22:16:23.207Z10/30 Write block 433 len 32768 data: 7 8 9 10 10 11 10 8
82432024-02-28T22:16:23.207Z11/30 Read block 357 len 4096
82442024-02-28T22:16:23.207Z12/30 Read block 457 len 16384
82452024-02-28T22:16:23.208Z13/30 Read block 278 len 4096
82462024-02-28T22:16:23.208Z14/30 Write block 501 len 36864 data: 9 9 11 12 10 10 9 10 11
82472024-02-28T22:16:23.208Z15/30 Write block 208 len 24576 data: 5 5 6 7 6 6
82482024-02-28T22:16:23.209Z16/30 Write block 307 len 12288 data: 8 7 5
82492024-02-28T22:16:23.209Z17/30 Flush
82502024-02-28T22:16:23.645Z18/30 Write block 491 len 24576 data: 9 8 7 6 8 8
82512024-02-28T22:16:23.645Z19/30 Write block 404 len 12288 data: 9 8 8
82522024-02-28T22:16:23.645Z20/30 Read block 182 len 28672
82532024-02-28T22:16:23.646Z21/30 Read block 120 len 24576
82542024-02-28T22:16:23.646Z22/30 Read block 238 len 36864
82552024-02-28T22:16:23.646Z23/30 Write block 581 len 28672 data: 6 3 3 3 5 5 4
82562024-02-28T22:16:23.646Z24/30 Read block 5 len 36864
82572024-02-28T22:16:23.646Z25/30 Write block 362 len 36864 data: 6 6 4 4 3 4 6 7 7
82582024-02-28T22:16:23.646Z26/30 Write block 447 len 20480 data: 6 5 5 5 6
82592024-02-28T22:16:23.646Z27/30 Write block 262 len 8192 data: 9 7
82602024-02-28T22:16:23.646Z28/30 Read block 388 len 20480
82612024-02-28T22:16:23.646Z29/30 Read block 288 len 4096
82622024-02-28T22:16:23.647Z30/30 Read block 185 len 40960
82632024-02-28T22:16:23.647Z----------------------------------------------------------------
82642024-02-28T22:16:23.647Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82652024-02-28T22:16:23.647ZStates: Active Active Active
82662024-02-28T22:16:23.647ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82672024-02-28T22:16:23.647Z 1 Acked 1000 Flush 0 Done Done Sent false
82682024-02-28T22:16:23.647Z 2 Acked 1001 Write 9 Done Done Sent false
82692024-02-28T22:16:23.647Z 3 Acked 1002 Read 7 Done Done Sent false
82702024-02-28T22:16:23.648Z 4 Acked 1003 Read 9 Done Done Sent false
82712024-02-28T22:16:23.648Z 5 Acked 1004 Read 3 Done Done Sent false
82722024-02-28T22:16:23.648Z 6 Acked 1005 Read 2 Done Done Sent false
82732024-02-28T22:16:23.648Z 7 Acked 1006 Write 9 Done Done Sent false
82742024-02-28T22:16:23.648Z 8 Acked 1007 Read 2 Done Done Sent false
82752024-02-28T22:16:23.648Z 9 Acked 1008 Write 4 Done Done Sent false
82762024-02-28T22:16:23.648Z 10 Acked 1009 Write 8 Done Done Sent false
82772024-02-28T22:16:23.648Z 11 Acked 1010 Read 1 Done Done Sent false
82782024-02-28T22:16:23.648Z 12 Acked 1011 Read 4 Done Done Sent false
82792024-02-28T22:16:23.648Z 13 Acked 1012 Read 1 Done Done Sent false
82802024-02-28T22:16:23.649Z 14 Acked 1013 Write 9 Done Done Sent false
82812024-02-28T22:16:23.649Z 15 Acked 1014 Write 6 Done Done Sent false
82822024-02-28T22:16:23.649Z 16 Acked 1015 Write 3 Done Done Sent false
82832024-02-28T22:16:23.649Z 17 Acked 1016 Flush 0 Done Done Sent false
82842024-02-28T22:16:23.649Z 18 Acked 1017 Write 6 Done Done Sent false
82852024-02-28T22:16:23.649Z 19 Acked 1018 Write 3 Done Done Sent false
82862024-02-28T22:16:23.649Z 20 Acked 1019 Read 7 Done Done Sent false
82872024-02-28T22:16:23.649Z 21 Acked 1020 Read 6 Done Done Sent false
82882024-02-28T22:16:23.649Z 22 Acked 1021 Read 9 Done Done Sent false
82892024-02-28T22:16:23.650Z 23 Acked 1022 Write 7 Done Done Sent false
82902024-02-28T22:16:23.650Z 24 Acked 1023 Read 9 Done Done Sent false
82912024-02-28T22:16:23.650Z 25 Acked 1024 Write 9 Done Done Sent false
82922024-02-28T22:16:23.650Z 26 Acked 1025 Write 5 Done Done Sent false
82932024-02-28T22:16:23.650Z 27 Acked 1026 Write 2 Done Done Sent false
82942024-02-28T22:16:23.650Z 28 Acked 1027 Read 5 Done Done Sent false
82952024-02-28T22:16:23.650Z 29 Acked 1028 Read 1 Done Done Sent false
82962024-02-28T22:16:23.651Z 30 Acked 1029 Read 10 Done Done Sent false
82972024-02-28T22:16:23.651Z STATES DS:0 DS:1 DS:2 TOTAL
82982024-02-28T22:16:23.651Z New 0 0 0 0
82992024-02-28T22:16:23.651Z Sent 0 0 30 30
83002024-02-28T22:16:23.651Z Done 30 30 0 60
83012024-02-28T22:16:23.651Z Skipped 0 0 0 0
83022024-02-28T22:16:23.651Z Error 0 0 0 0
83032024-02-28T22:16:23.651ZLast Flush: 1016 1016 0
83042024-02-28T22:16:23.652ZDownstairs last five completed:
83052024-02-28T22:16:23.652ZUpstairs last five completed: 30 29 28 27 26
83062024-02-28T22:16:23.652Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:22.107480985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1439}
83072024-02-28T22:16:23.652ZWrote out file "/tmp/test_repair_verify.data"
83082024-02-28T22:16:23.652Z
83092024-02-28T22:16:23.652ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83102024-02-28T22:16:24.074ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83112024-02-28T22:16:24.075Z 0 000-019 75 75 75 137 137 137 F F F
83122024-02-28T22:16:24.075Z 1 020-039 74 74 74 136 136 136 F F F
83132024-02-28T22:16:24.075Z 2 040-059 66 66 66 125 125 125 F F F
83142024-02-28T22:16:24.075Z 3 060-079 76 76 67 139 139 126 F F F <---
83152024-02-28T22:16:24.075Z 4 080-099 67 67 67 126 126 126 F F F
83162024-02-28T22:16:24.075Z 5 100-119 75 75 75 137 137 137 F F F
83172024-02-28T22:16:24.075Z 6 120-139 70 70 70 132 132 132 F F F
83182024-02-28T22:16:24.075Z 7 140-159 72 72 72 134 134 134 F F F
83192024-02-28T22:16:24.075Z 8 160-179 72 72 72 134 134 134 F F F
83202024-02-28T22:16:24.075Z 9 180-199 71 71 71 133 133 133 F F F
83212024-02-28T22:16:24.075Z 10 200-219 76 76 70 139 139 132 F F F <---
83222024-02-28T22:16:24.075Z 11 220-239 74 74 74 136 136 136 F F F
83232024-02-28T22:16:24.075Z 12 240-259 71 71 71 133 133 133 F F F
83242024-02-28T22:16:24.075Z 13 260-279 71 71 71 133 133 133 T T F <---
83252024-02-28T22:16:24.075Z 14 280-299 72 72 72 134 134 134 F F F
83262024-02-28T22:16:24.075Z 15 300-319 76 76 71 139 139 133 F F F <---
83272024-02-28T22:16:24.075Z 16 320-339 72 72 72 134 134 134 F F F
83282024-02-28T22:16:24.075Z 17 340-359 72 72 72 134 134 134 F F F
83292024-02-28T22:16:24.075Z 18 360-379 72 72 72 134 134 134 T T F <---
83302024-02-28T22:16:24.075Z 19 380-399 72 72 72 134 134 134 F F F
83312024-02-28T22:16:24.076Z 20 400-419 72 72 72 134 134 134 T T F <---
83322024-02-28T22:16:24.076Z 21 420-439 76 76 72 139 139 134 F F F <---
83332024-02-28T22:16:24.076Z 22 440-459 76 76 75 139 139 137 T T F <---
83342024-02-28T22:16:24.076Z 23 460-479 72 72 72 134 134 134 F F F
83352024-02-28T22:16:24.076Z 24 480-499 76 76 70 139 139 131 T T F <---
83362024-02-28T22:16:24.076Z 25 500-519 76 76 75 139 139 137 F F F <---
83372024-02-28T22:16:24.076Z 26 520-539 75 75 75 137 137 137 F F F
83382024-02-28T22:16:24.076Z 27 540-559 75 75 75 137 137 137 F F F
83392024-02-28T22:16:24.076Z 28 560-579 76 76 75 139 139 137 F F F <---
83402024-02-28T22:16:24.076Z 29 580-599 70 70 70 132 132 132 T T F <---
83412024-02-28T22:16:24.076ZMax gen: 76, Max flush: 139
83422024-02-28T22:16:24.076ZError: Difference in extent metadata found!
83432024-02-28T22:16:24.076ZA Difference in extent metadata is expected here
83442024-02-28T22:16:24.076ZOn loop 38
83452024-02-28T22:16:24.076Z
83462024-02-28T22:16:24.076ZVerifying data now
83472024-02-28T22:16:27.877ZNo change after verify
83482024-02-28T22:16:27.877ZLoop: 38 Downstairs dump after verify (and repair):
83492024-02-28T22:16:27.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83502024-02-28T22:16:27.893Z 0 000-019 75 75 75 137 137 137 F F F
83512024-02-28T22:16:27.893Z 1 020-039 74 74 74 136 136 136 F F F
83522024-02-28T22:16:27.893Z 2 040-059 66 66 66 125 125 125 F F F
83532024-02-28T22:16:27.893Z 3 060-079 76 76 76 139 139 139 F F F
83542024-02-28T22:16:27.893Z 4 080-099 67 67 67 126 126 126 F F F
83552024-02-28T22:16:27.893Z 5 100-119 75 75 75 137 137 137 F F F
83562024-02-28T22:16:27.893Z 6 120-139 70 70 70 132 132 132 F F F
83572024-02-28T22:16:27.893Z 7 140-159 72 72 72 134 134 134 F F F
83582024-02-28T22:16:27.893Z 8 160-179 72 72 72 134 134 134 F F F
83592024-02-28T22:16:27.893Z 9 180-199 71 71 71 133 133 133 F F F
83602024-02-28T22:16:27.893Z 10 200-219 76 76 76 139 139 139 F F F
83612024-02-28T22:16:27.894Z 11 220-239 74 74 74 136 136 136 F F F
83622024-02-28T22:16:27.897Z 12 240-259 71 71 71 133 133 133 F F F
83632024-02-28T22:16:27.897Z 13 260-279 77 77 77 140 140 140 F F F
83642024-02-28T22:16:27.897Z 14 280-299 72 72 72 134 134 134 F F F
83652024-02-28T22:16:27.898Z 15 300-319 76 76 76 139 139 139 F F F
83662024-02-28T22:16:27.898Z 16 320-339 72 72 72 134 134 134 F F F
83672024-02-28T22:16:27.898Z 17 340-359 72 72 72 134 134 134 F F F
83682024-02-28T22:16:27.898Z 18 360-379 77 77 77 140 140 140 F F F
83692024-02-28T22:16:27.898Z 19 380-399 72 72 72 134 134 134 F F F
83702024-02-28T22:16:27.898Z 20 400-419 77 77 77 140 140 140 F F F
83712024-02-28T22:16:27.898Z 21 420-439 76 76 76 139 139 139 F F F
83722024-02-28T22:16:27.898Z 22 440-459 77 77 77 140 140 140 F F F
83732024-02-28T22:16:27.898Z 23 460-479 72 72 72 134 134 134 F F F
83742024-02-28T22:16:27.898Z 24 480-499 77 77 77 140 140 140 F F F
83752024-02-28T22:16:27.898Z 25 500-519 76 76 76 139 139 139 F F F
83762024-02-28T22:16:27.898Z 26 520-539 75 75 75 137 137 137 F F F
83772024-02-28T22:16:27.898Z 27 540-559 75 75 75 137 137 137 F F F
83782024-02-28T22:16:27.899Z 28 560-579 76 76 76 139 139 139 F F F
83792024-02-28T22:16:27.899Z 29 580-599 77 77 77 140 140 140 F F F
83802024-02-28T22:16:27.899ZMax gen: 77, Max flush: 140
83812024-02-28T22:16:27.899Z
83822024-02-28T22:16:27.899ZBegin loop 39
83832024-02-28T22:16:27.899ZDownstairs to restart: 0
83842024-02-28T22:16:27.916Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.80803022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
83852024-02-28T22:16:27.916Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
83862024-02-28T22:16:27.916Z,"time":"2024-02-28T22:16:26.808308082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
83872024-02-28T22:16:27.923Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808341391Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
83882024-02-28T22:16:27.923Z{"msg":"Crucible 7a3bb875-f5bc-41d4-8de2-9b3981410c1d has session id: 77616868-3444-4f3b-b820-261c64a8755a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.80838156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83892024-02-28T22:16:27.923Z{"msg":"Upstairs opts: Upstairs UUID: 7a3bb875-f5bc-41d4-8de2-9b3981410c1d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808420669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83902024-02-28T22:16:27.923Z{"msg":"Crucible stats registered with UUID: 7a3bb875-f5bc-41d4-8de2-9b3981410c1d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808439739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83912024-02-28T22:16:27.923Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808463948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
83922024-02-28T22:16:27.923Z{"msg":"7a3bb875-f5bc-41d4-8de2-9b3981410c1d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808476567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83932024-02-28T22:16:27.923Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808504087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83942024-02-28T22:16:27.923Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808530406Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83952024-02-28T22:16:27.923Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808568375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83962024-02-28T22:16:27.923Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808611364Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83972024-02-28T22:16:27.924Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808638673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83982024-02-28T22:16:27.924Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808729531Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
83992024-02-28T22:16:27.924Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808798249Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84002024-02-28T22:16:27.924Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808848947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84012024-02-28T22:16:27.924Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808875887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84022024-02-28T22:16:27.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808922525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84032024-02-28T22:16:27.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808952514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84042024-02-28T22:16:27.924Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.808997813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84052024-02-28T22:16:27.924Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.809602827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84062024-02-28T22:16:27.924Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.809652455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84072024-02-28T22:16:27.924Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.809679524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84082024-02-28T22:16:27.924Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.809730633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84092024-02-28T22:16:27.924Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810125622Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84102024-02-28T22:16:27.924Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810319437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84112024-02-28T22:16:27.924Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810372285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84122024-02-28T22:16:27.924Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810410504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84132024-02-28T22:16:27.924Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810599329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84142024-02-28T22:16:27.924Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810644588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84152024-02-28T22:16:27.924Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810670087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84162024-02-28T22:16:27.924Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810961279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84172024-02-28T22:16:27.924Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.810998108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84182024-02-28T22:16:27.924Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811016508Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84192024-02-28T22:16:27.925Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811227502Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84202024-02-28T22:16:27.925Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.8116468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84212024-02-28T22:16:27.925Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811687359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84222024-02-28T22:16:27.925Z{"msg":"[0]R flush_numbers[0..12]: [137, 136, 125, 139, 126, 137, 132, 134, 134, 133, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811715358Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84232024-02-28T22:16:27.925Z{"msg":"[0]R generation[0..12]: [75, 74, 66, 76, 67, 75, 70, 72, 72, 71, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811730408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84242024-02-28T22:16:27.925Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811751977Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84252024-02-28T22:16:27.925Z{"msg":"[1]R flush_numbers[0..12]: [137, 136, 125, 139, 126, 137, 132, 134, 134, 133, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811766607Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84262024-02-28T22:16:27.925Z{"msg":"[1]R generation[0..12]: [75, 74, 66, 76, 67, 75, 70, 72, 72, 71, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811810796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84272024-02-28T22:16:27.925ZWait for a query_work_queue command to finish before sending IO
84282024-02-28T22:16:27.925Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811852585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84292024-02-28T22:16:27.925Z{"msg":"[2]R flush_numbers[0..12]: [137, 136, 125, 139, 126, 137, 132, 134, 134, 133, 139, 136]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811890153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84302024-02-28T22:16:27.925Z{"msg":"[2]R generation[0..12]: [75, 74, 66, 76, 67, 75, 70, 72, 72, 71, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811920493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84312024-02-28T22:16:27.925Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811935602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84322024-02-28T22:16:27.925Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811950582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84332024-02-28T22:16:27.925Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.811984611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84342024-02-28T22:16:27.925Z{"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.81202068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84352024-02-28T22:16:27.925Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812045169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"":"downstairs","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84362024-02-28T22:16:27.925Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812061349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84372024-02-28T22:16:27.925Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812093058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"0","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84382024-02-28T22:16:27.925Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812129017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"1","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84392024-02-28T22:16:27.925Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812153446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"client":"2","":"downstairs","session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84402024-02-28T22:16:27.925Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812175676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84412024-02-28T22:16:27.926Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812210495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84422024-02-28T22:16:27.926Z{"msg":"7a3bb875-f5bc-41d4-8de2-9b3981410c1d is now active with session: 77616868-3444-4f3b-b820-261c64a8755a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812239324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84432024-02-28T22:16:27.926Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812255843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447,"session_id":"77616868-3444-4f3b-b820-261c64a8755a"}
84442024-02-28T22:16:27.926Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:26.812288062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
84452024-02-28T22:16:27.926ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84462024-02-28T22:16:27.930ZLoading write count information from file "/tmp/test_repair_verify.data"
84472024-02-28T22:16:27.930ZRun Repair workload
84482024-02-28T22:16:27.930Z01/30 Write block 400 len 12288 data: 8 8 9
84492024-02-28T22:16:27.930Z02/30 Read block 96 len 24576
84502024-02-28T22:16:27.930Z03/30 Read block 274 len 16384
84512024-02-28T22:16:27.930Z04/30 Flush
84522024-02-28T22:16:27.930Z05/30 Write block 59 len 28672 data: 8 9 9 10 9 9 9
84532024-02-28T22:16:27.930Z06/30 Read block 25 len 16384
84542024-02-28T22:16:27.930Z07/30 Write block 72 len 16384 data: 7 5 6 5
84552024-02-28T22:16:27.931Z08/30 Write block 168 len 24576 data: 11 12 12 11 10 10
84562024-02-28T22:16:27.931Z09/30 Read block 276 len 4096
84572024-02-28T22:16:27.931Z10/30 Read block 596 len 8192
84582024-02-28T22:16:27.935Z11/30 Read block 462 len 12288
84592024-02-28T22:16:27.935Z12/30 Read block 517 len 20480
84602024-02-28T22:16:27.935Z13/30 Read block 89 len 32768
84612024-02-28T22:16:27.935Z14/30 Write block 336 len 32768 data: 8 6 8 8 8 7 8 7
84622024-02-28T22:16:27.935Z15/30 Write block 449 len 28672 data: 6 6 7 7 7 5 5
84632024-02-28T22:16:27.939Z16/30 Flush
84642024-02-28T22:16:27.939Z17/30 Flush
84652024-02-28T22:16:27.943Z18/30 Write block 446 len 12288 data: 7 7 6
84662024-02-28T22:16:27.943Z19/30 Read block 445 len 40960
84672024-02-28T22:16:27.943Z20/30 Write block 203 len 40960 data: 4 4 6 5 5 6 6 7 8 7
84682024-02-28T22:16:27.943Z21/30 Write block 468 len 20480 data: 6 5 6 7 9
84692024-02-28T22:16:27.943Z22/30 Read block 581 len 36864
84702024-02-28T22:16:27.947Z23/30 Read block 92 len 32768
84712024-02-28T22:16:27.947Z24/30 Write block 420 len 40960 data: 10 10 9 8 7 7 4 3 3 4
84722024-02-28T22:16:27.947Z25/30 Write block 224 len 12288 data: 5 6 7
84732024-02-28T22:16:27.947Z26/30 Read block 489 len 28672
84742024-02-28T22:16:27.950Z27/30 Write block 303 len 16384 data: 6 7 6 6
84752024-02-28T22:16:27.951Z28/30 Read block 338 len 28672
84762024-02-28T22:16:27.951Z29/30 Read block 276 len 36864
84772024-02-28T22:16:27.951Z30/30 Write block 487 len 32768 data: 7 5 7 8 10 9 8 7
84782024-02-28T22:16:27.955Z----------------------------------------------------------------
84792024-02-28T22:16:27.955Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84802024-02-28T22:16:27.955ZStates: Active Active Active
84812024-02-28T22:16:27.955ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84822024-02-28T22:16:27.955Z 1 Acked 1000 Write 3 Sent Done Done false
84832024-02-28T22:16:27.955Z 2 Acked 1001 Read 6 Sent Done Done false
84842024-02-28T22:16:27.955Z 3 Acked 1002 Read 4 Sent Done Done false
84852024-02-28T22:16:27.955Z 4 Acked 1003 Flush 0 Sent Done Done false
84862024-02-28T22:16:27.956Z 5 Acked 1004 Write 7 Sent Done Done false
84872024-02-28T22:16:27.956Z 6 Acked 1005 Read 4 Sent Done Done false
84882024-02-28T22:16:27.956Z 7 Acked 1006 Write 4 Sent Done Done false
84892024-02-28T22:16:27.956Z 8 Acked 1007 Write 6 Sent Done Done false
84902024-02-28T22:16:27.956Z 9 Acked 1008 Read 1 Sent Done Done false
84912024-02-28T22:16:27.956Z 10 Acked 1009 Read 2 Sent Done Done false
84922024-02-28T22:16:27.956Z 11 Acked 1010 Read 3 Sent Done Done false
84932024-02-28T22:16:27.956Z 12 Acked 1011 Read 5 Sent Done Done false
84942024-02-28T22:16:27.956Z 13 Acked 1012 Read 8 Sent Done Done false
84952024-02-28T22:16:27.956Z 14 Acked 1013 Write 8 Sent Done Done false
84962024-02-28T22:16:27.956Z 15 Acked 1014 Write 7 Sent Done Done false
84972024-02-28T22:16:27.956Z 16 Acked 1015 Flush 0 Sent Done Done false
84982024-02-28T22:16:27.957Z 17 Acked 1016 Flush 0 Sent Done Done false
84992024-02-28T22:16:27.957Z 18 Acked 1017 Write 3 Sent Done Done false
85002024-02-28T22:16:27.957Z 19 Acked 1018 Read 10 Sent Done Done false
85012024-02-28T22:16:27.957Z 20 Acked 1019 Write 10 Sent Done Done false
85022024-02-28T22:16:27.957Z 21 Acked 1020 Write 5 Sent Done Done false
85032024-02-28T22:16:27.957Z 22 Acked 1021 Read 9 Sent Done Done false
85042024-02-28T22:16:27.957Z 23 Acked 1022 Read 8 Sent Done Done false
85052024-02-28T22:16:27.957Z 24 Acked 1023 Write 10 Sent Done Done false
85062024-02-28T22:16:27.957Z 25 Acked 1024 Write 3 Sent Done Done false
85072024-02-28T22:16:27.957Z 26 Acked 1025 Read 7 Sent Done Done false
85082024-02-28T22:16:27.957Z 27 Acked 1026 Write 4 Sent Done Done false
85092024-02-28T22:16:27.957Z 28 Acked 1027 Read 7 Sent Done Done false
85102024-02-28T22:16:27.957Z 29 Acked 1028 Read 9 Sent Done Done false
85112024-02-28T22:16:27.957Z 30 Acked 1029 Write 8 Sent Sent Sent false
85122024-02-28T22:16:27.957Z STATES DS:0 DS:1 DS:2 TOTAL
85132024-02-28T22:16:27.957Z New 0 0 0 0
85142024-02-28T22:16:27.957Z Sent 30 1 1 32
85152024-02-28T22:16:27.957Z Done 0 29 29 58
85162024-02-28T22:16:27.957Z Skipped 0 0 0 0
85172024-02-28T22:16:27.957Z Error 0 0 0 0
85182024-02-28T22:16:27.958ZLast Flush: 0 1016 1016
85192024-02-28T22:16:27.958ZDownstairs last five completed:
85202024-02-28T22:16:27.958ZUpstairs last five completed: 30 29 28 27 26
85212024-02-28T22:16:27.958Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:26.843818384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1447}
85222024-02-28T22:16:27.958ZWrote out file "/tmp/test_repair_verify.data"
85232024-02-28T22:16:27.963Z
85242024-02-28T22:16:27.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85252024-02-28T22:16:27.988ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85262024-02-28T22:16:27.989Z 0 000-019 75 75 75 137 137 137 F F F
85272024-02-28T22:16:27.989Z 1 020-039 74 74 74 136 136 136 F F F
85282024-02-28T22:16:27.989Z 2 040-059 66 78 78 125 142 142 F F F <---
85292024-02-28T22:16:27.989Z 3 060-079 76 78 78 139 142 142 F F F <---
85302024-02-28T22:16:27.989Z 4 080-099 67 67 67 126 126 126 F F F
85312024-02-28T22:16:27.989Z 5 100-119 75 75 75 137 137 137 F F F
85322024-02-28T22:16:27.989Z 6 120-139 70 70 70 132 132 132 F F F
85332024-02-28T22:16:27.989Z 7 140-159 72 72 72 134 134 134 F F F
85342024-02-28T22:16:27.989Z 8 160-179 72 78 78 134 142 142 F F F <---
85352024-02-28T22:16:27.989Z 9 180-199 71 71 71 133 133 133 F F F
85362024-02-28T22:16:27.989Z 10 200-219 76 76 76 139 139 139 F T T <---
85372024-02-28T22:16:27.993Z 11 220-239 74 74 74 136 136 136 F T T <---
85382024-02-28T22:16:27.994Z 12 240-259 71 71 71 133 133 133 F F F
85392024-02-28T22:16:27.994Z 13 260-279 77 77 77 140 140 140 F F F
85402024-02-28T22:16:27.994Z 14 280-299 72 72 72 134 134 134 F F F
85412024-02-28T22:16:27.994Z 15 300-319 76 76 76 139 139 139 F T T <---
85422024-02-28T22:16:27.994Z 16 320-339 72 78 78 134 142 142 F F F <---
85432024-02-28T22:16:27.994Z 17 340-359 72 78 78 134 142 142 F F F <---
85442024-02-28T22:16:27.994Z 18 360-379 77 77 77 140 140 140 F F F
85452024-02-28T22:16:27.994Z 19 380-399 72 72 72 134 134 134 F F F
85462024-02-28T22:16:27.994Z 20 400-419 77 78 78 140 141 141 F F F <---
85472024-02-28T22:16:27.994Z 21 420-439 76 76 76 139 139 139 F T T <---
85482024-02-28T22:16:27.994Z 22 440-459 77 78 78 140 142 142 F T T <---
85492024-02-28T22:16:27.994Z 23 460-479 72 72 72 134 134 134 F T T <---
85502024-02-28T22:16:27.995Z 24 480-499 77 77 77 140 140 140 F T T <---
85512024-02-28T22:16:27.995Z 25 500-519 76 76 76 139 139 139 F F F
85522024-02-28T22:16:27.995Z 26 520-539 75 75 75 137 137 137 F F F
85532024-02-28T22:16:27.995Z 27 540-559 75 75 75 137 137 137 F F F
85542024-02-28T22:16:27.995Z 28 560-579 76 76 76 139 139 139 F F F
85552024-02-28T22:16:27.995Z 29 580-599 77 77 77 140 140 140 F F F
85562024-02-28T22:16:27.995ZMax gen: 78, Max flush: 142
85572024-02-28T22:16:27.995ZError: Difference in extent metadata found!
85582024-02-28T22:16:27.995ZA Difference in extent metadata is expected here
85592024-02-28T22:16:27.995ZOn loop 39
85602024-02-28T22:16:27.995Z
85612024-02-28T22:16:27.995ZVerifying data now
85622024-02-28T22:16:32.650ZNo change after verify
85632024-02-28T22:16:32.650ZLoop: 39 Downstairs dump after verify (and repair):
85642024-02-28T22:16:32.663ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85652024-02-28T22:16:32.664Z 0 000-019 75 75 75 137 137 137 F F F
85662024-02-28T22:16:32.664Z 1 020-039 74 74 74 136 136 136 F F F
85672024-02-28T22:16:32.664Z 2 040-059 78 78 78 142 142 142 F F F
85682024-02-28T22:16:32.664Z 3 060-079 78 78 78 142 142 142 F F F
85692024-02-28T22:16:32.664Z 4 080-099 67 67 67 126 126 126 F F F
85702024-02-28T22:16:32.664Z 5 100-119 75 75 75 137 137 137 F F F
85712024-02-28T22:16:32.664Z 6 120-139 70 70 70 132 132 132 F F F
85722024-02-28T22:16:32.664Z 7 140-159 72 72 72 134 134 134 F F F
85732024-02-28T22:16:32.664Z 8 160-179 78 78 78 142 142 142 F F F
85742024-02-28T22:16:32.664Z 9 180-199 71 71 71 133 133 133 F F F
85752024-02-28T22:16:32.664Z 10 200-219 79 79 79 143 143 143 F F F
85762024-02-28T22:16:32.668Z 11 220-239 79 79 79 143 143 143 F F F
85772024-02-28T22:16:32.668Z 12 240-259 71 71 71 133 133 133 F F F
85782024-02-28T22:16:32.668Z 13 260-279 77 77 77 140 140 140 F F F
85792024-02-28T22:16:32.668Z 14 280-299 72 72 72 134 134 134 F F F
85802024-02-28T22:16:32.668Z 15 300-319 79 79 79 143 143 143 F F F
85812024-02-28T22:16:32.668Z 16 320-339 78 78 78 142 142 142 F F F
85822024-02-28T22:16:32.668Z 17 340-359 78 78 78 142 142 142 F F F
85832024-02-28T22:16:32.668Z 18 360-379 77 77 77 140 140 140 F F F
85842024-02-28T22:16:32.668Z 19 380-399 72 72 72 134 134 134 F F F
85852024-02-28T22:16:32.668Z 20 400-419 78 78 78 141 141 141 F F F
85862024-02-28T22:16:32.668Z 21 420-439 79 79 79 143 143 143 F F F
85872024-02-28T22:16:32.668Z 22 440-459 79 79 79 143 143 143 F F F
85882024-02-28T22:16:32.668Z 23 460-479 79 79 79 143 143 143 F F F
85892024-02-28T22:16:32.668Z 24 480-499 79 79 79 143 143 143 F F F
85902024-02-28T22:16:32.668Z 25 500-519 76 76 76 139 139 139 F F F
85912024-02-28T22:16:32.668Z 26 520-539 75 75 75 137 137 137 F F F
85922024-02-28T22:16:32.668Z 27 540-559 75 75 75 137 137 137 F F F
85932024-02-28T22:16:32.668Z 28 560-579 76 76 76 139 139 139 F F F
85942024-02-28T22:16:32.668Z 29 580-599 77 77 77 140 140 140 F F F
85952024-02-28T22:16:32.668ZMax gen: 79, Max flush: 143
85962024-02-28T22:16:32.669Z
85972024-02-28T22:16:32.669ZBegin loop 40
85982024-02-28T22:16:32.669ZDownstairs to restart: 0
85992024-02-28T22:16:32.685Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.576901992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
86002024-02-28T22:16:32.685ZCrucible runtime is spawned
86012024-02-28T22:16:32.692Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577284742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
86022024-02-28T22:16:32.692Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577320741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
86032024-02-28T22:16:32.692Z{"msg":"Crucible 6aedb4a6-bf7e-4036-a7d1-d4c547206d45 has session id: 2c045db9-bcfd-4ccf-a2fa-bd04049c98bb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.57734146Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86042024-02-28T22:16:32.692Z{"msg":"Upstairs opts: Upstairs UUID: 6aedb4a6-bf7e-4036-a7d1-d4c547206d45, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577434447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86052024-02-28T22:16:32.692Z{"msg":"Crucible stats registered with UUID: 6aedb4a6-bf7e-4036-a7d1-d4c547206d45","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577460557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86062024-02-28T22:16:32.692Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577475726Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
86072024-02-28T22:16:32.692Z{"msg":"6aedb4a6-bf7e-4036-a7d1-d4c547206d45 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577499936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86082024-02-28T22:16:32.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577532515Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86092024-02-28T22:16:32.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577560354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86102024-02-28T22:16:32.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577586193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86112024-02-28T22:16:32.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577623702Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86122024-02-28T22:16:32.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577659361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86132024-02-28T22:16:32.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577730559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86142024-02-28T22:16:32.692Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577847006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86152024-02-28T22:16:32.692Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577909034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86162024-02-28T22:16:32.692Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577950613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86172024-02-28T22:16:32.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.577976372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86182024-02-28T22:16:32.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578000562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86192024-02-28T22:16:32.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578032041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86202024-02-28T22:16:32.693Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578659164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86212024-02-28T22:16:32.693Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578695603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86222024-02-28T22:16:32.693Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578710952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86232024-02-28T22:16:32.693Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578750881Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86242024-02-28T22:16:32.693Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578774921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86252024-02-28T22:16:32.693Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.578806589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86262024-02-28T22:16:32.693Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579314755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86272024-02-28T22:16:32.693Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579361534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86282024-02-28T22:16:32.693Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579398293Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86292024-02-28T22:16:32.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579787562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86302024-02-28T22:16:32.693Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579842461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86312024-02-28T22:16:32.693Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.57988312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86322024-02-28T22:16:32.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.579969337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86332024-02-28T22:16:32.693Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580012516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86342024-02-28T22:16:32.693Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580030936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86352024-02-28T22:16:32.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580051175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86362024-02-28T22:16:32.693Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580065355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86372024-02-28T22:16:32.693Z{"msg":"[0]R flush_numbers[0..12]: [137, 136, 142, 142, 126, 137, 132, 134, 142, 133, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580086084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86382024-02-28T22:16:32.693Z{"msg":"[0]R generation[0..12]: [75, 74, 78, 78, 67, 75, 70, 72, 78, 71, 79, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580100274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86392024-02-28T22:16:32.693Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580121143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86402024-02-28T22:16:32.694Z{"msg":"[1]R flush_numbers[0..12]: [137, 136, 142, 142, 126, 137, 132, 134, 142, 133, 143, 143]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580135783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86412024-02-28T22:16:32.694Z{"msg":"[1]R generation[0..12]: [75, 74, 78, 78, 67, 75, 70, 72, 78, 71, 79, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580151863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86422024-02-28T22:16:32.694Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580186162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86432024-02-28T22:16:32.694Z{"msg":"[2]R flush_numbers[0..12]: [137, 136, 142, 142, 126, 137, 132, 134, 142, 133, 143, 143]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
86442024-02-28T22:16:32.694Z,"time":"2024-02-28T22:16:31.580212221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86452024-02-28T22:16:32.694Z{"msg":"[2]R generation[0..12]: [75, 74, 78, 78, 67, 75, 70, 72, 78, 71, 79, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.58024273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86462024-02-28T22:16:32.694Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580269679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86472024-02-28T22:16:32.694Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580299579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86482024-02-28T22:16:32.694Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580324108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86492024-02-28T22:16:32.694Z{"msg":"Next flush: 144","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580351027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86502024-02-28T22:16:32.694Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580374866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"":"downstairs","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86512024-02-28T22:16:32.694Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580401476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86522024-02-28T22:16:32.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580423215Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"0","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86532024-02-28T22:16:32.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580450054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"1","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86542024-02-28T22:16:32.694Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580475034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"client":"2","":"downstairs","session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86552024-02-28T22:16:32.694Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580504453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86562024-02-28T22:16:32.694Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580526172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86572024-02-28T22:16:32.694Z{"msg":"6aedb4a6-bf7e-4036-a7d1-d4c547206d45 is now active with session: 2c045db9-bcfd-4ccf-a2fa-bd04049c98bb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580549301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86582024-02-28T22:16:32.694ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86592024-02-28T22:16:32.694Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.580582141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455,"session_id":"2c045db9-bcfd-4ccf-a2fa-bd04049c98bb"}
86602024-02-28T22:16:32.694Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:31.5806052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
86612024-02-28T22:16:32.698ZLoading write count information from file "/tmp/test_repair_verify.data"
86622024-02-28T22:16:32.698ZRun Repair workload
86632024-02-28T22:16:32.698Z01/30 Write block 506 len 8192 data: 11 10
86642024-02-28T22:16:32.698Z02/30 Write block 505 len 8192 data: 11 12
86652024-02-28T22:16:32.698Z03/30 Read block 381 len 40960
86662024-02-28T22:16:32.698Z04/30 Read block 521 len 40960
86672024-02-28T22:16:32.698Z05/30 Read block 410 len 24576
86682024-02-28T22:16:32.699Z06/30 Write block 8 len 4096 data: 11
86692024-02-28T22:16:32.699Z07/30 Read block 142 len 40960
86702024-02-28T22:16:32.699Z08/30 Write block 393 len 4096 data: 5
86712024-02-28T22:16:32.699Z09/30 Write block 178 len 4096 data: 6
86722024-02-28T22:16:32.699Z10/30 Read block 370 len 28672
86732024-02-28T22:16:32.699Z11/30 Read block 332 len 4096
86742024-02-28T22:16:32.699Z12/30 Read block 206 len 4096
86752024-02-28T22:16:32.699Z13/30 Write block 155 len 4096 data: 6
86762024-02-28T22:16:32.702Z14/30 Write block 170 len 32768 data: 13 12 11 11 10 8 7 6
86772024-02-28T22:16:32.702Z15/30 Read block 24 len 4096
86782024-02-28T22:16:32.702Z16/30 Read block 195 len 32768
86792024-02-28T22:16:32.702Z17/30 Flush
86802024-02-28T22:16:32.708Z18/30 Write block 231 len 24576 data: 12 12 11 10 6 5
86812024-02-28T22:16:32.708Z19/30 Read block 493 len 8192
86822024-02-28T22:16:32.714Z20/30 Read block 548 len 4096
86832024-02-28T22:16:32.714Z21/30 Read block 155 len 40960
86842024-02-28T22:16:32.714Z22/30 Write block 370 len 24576 data: 8 9 11 8 6 5
86852024-02-28T22:16:32.714Z23/30 Write block 40 len 8192 data: 8 9
86862024-02-28T22:16:32.714Z24/30 Read block 264 len 28672
86872024-02-28T22:16:32.714Z25/30 Write block 445 len 24576 data: 6 8 8 7 7 7
86882024-02-28T22:16:32.714Z26/30 Write block 374 len 32768 data: 7 6 6 7 8 8 8 9
86892024-02-28T22:16:32.718Z27/30 Read block 538 len 16384
86902024-02-28T22:16:32.718Z28/30 Write block 196 len 24576 data: 7 7 7 5 3 5
86912024-02-28T22:16:32.718Z29/30 Write block 412 len 8192 data: 7 9
86922024-02-28T22:16:32.718Z30/30 Read block 206 len 16384
86932024-02-28T22:16:32.718Z----------------------------------------------------------------
86942024-02-28T22:16:32.718Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
86952024-02-28T22:16:32.718ZStates: Active Active Active
86962024-02-28T22:16:32.718ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86972024-02-28T22:16:32.718Z 1 Acked 1000 Write 2 Done Done Done false
86982024-02-28T22:16:32.718Z 2 Acked 1001 Write 2 Sent Done Done false
86992024-02-28T22:16:32.718Z 3 Acked 1002 Read 10 Sent Done Done false
87002024-02-28T22:16:32.718Z 4 Acked 1003 Read 10 Sent Done Done false
87012024-02-28T22:16:32.719Z 5 Acked 1004 Read 6 Sent Done Done false
87022024-02-28T22:16:32.719Z 6 Acked 1005 Write 1 Sent Done Done false
87032024-02-28T22:16:32.719Z 7 Acked 1006 Read 10 Sent Done Done false
87042024-02-28T22:16:32.719Z 8 Acked 1007 Write 1 Sent Done Done false
87052024-02-28T22:16:32.719Z 9 Acked 1008 Write 1 Sent Done Done false
87062024-02-28T22:16:32.719Z 10 Acked 1009 Read 7 Sent Done Done false
87072024-02-28T22:16:32.719Z 11 Acked 1010 Read 1 Sent Done Done false
87082024-02-28T22:16:32.719Z 12 Acked 1011 Read 1 Sent Done Done false
87092024-02-28T22:16:32.719Z 13 Acked 1012 Write 1 Sent Done Done false
87102024-02-28T22:16:32.719Z 14 Acked 1013 Write 8 Sent Done Done false
87112024-02-28T22:16:32.719Z 15 Acked 1014 Read 1 Sent Done Done false
87122024-02-28T22:16:32.719Z 16 Acked 1015 Read 8 Sent Done Done false
87132024-02-28T22:16:32.719Z 17 Acked 1016 Flush 0 Sent Done Done false
87142024-02-28T22:16:32.719Z 18 Acked 1017 Write 6 Sent Done Done false
87152024-02-28T22:16:32.719Z 19 Acked 1018 Read 2 Sent Done Done false
87162024-02-28T22:16:32.719Z 20 Acked 1019 Read 1 Sent Done Done false
87172024-02-28T22:16:32.719Z 21 Acked 1020 Read 10 Sent Done Done false
87182024-02-28T22:16:32.719Z 22 Acked 1021 Write 6 Sent Done Done false
87192024-02-28T22:16:32.719Z 23 Acked 1022 Write 2 Sent Done Done false
87202024-02-28T22:16:32.719Z 24 Acked 1023 Read 7 Sent Done Done false
87212024-02-28T22:16:32.719Z 25 Acked 1024 Write 6 Sent Done Done false
87222024-02-28T22:16:32.719Z 26 Acked 1025 Write 8 Sent Done Done false
87232024-02-28T22:16:32.719Z 27 Acked 1026 Read 4 Sent Done Done false
87242024-02-28T22:16:32.720Z 28 Acked 1027 Write 6 Sent Done Done false
87252024-02-28T22:16:32.720Z 29 Acked 1028 Write 2 Sent Done Done false
87262024-02-28T22:16:32.720Z 30 Acked 1029 Read 4 Sent Sent Done false
87272024-02-28T22:16:32.720Z STATES DS:0 DS:1 DS:2 TOTAL
87282024-02-28T22:16:32.720Z New 0 0 0 0
87292024-02-28T22:16:32.720Z Sent 29 1 0 30
87302024-02-28T22:16:32.720Z Done 1 29 30 60
87312024-02-28T22:16:32.720Z Skipped 0 0 0 0
87322024-02-28T22:16:32.720Z Error 0 0 0 0
87332024-02-28T22:16:32.720ZLast Flush: 0 1016 1016
87342024-02-28T22:16:32.720ZDownstairs last five completed:
87352024-02-28T22:16:32.720ZUpstairs last five completed: 30 29 28 27 26
87362024-02-28T22:16:32.720Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:31.607517459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1455}
87372024-02-28T22:16:32.720ZWrote out file "/tmp/test_repair_verify.data"
87382024-02-28T22:16:32.724Z
87392024-02-28T22:16:32.724ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87402024-02-28T22:16:32.739ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87412024-02-28T22:16:32.739Z 0 000-019 75 80 80 137 144 144 F F F <---
87422024-02-28T22:16:32.739Z 1 020-039 74 74 74 136 136 136 F F F
87432024-02-28T22:16:32.739Z 2 040-059 78 78 78 142 142 142 F T T <---
87442024-02-28T22:16:32.740Z 3 060-079 78 78 78 142 142 142 F F F
87452024-02-28T22:16:32.740Z 4 080-099 67 67 67 126 126 126 F F F
87462024-02-28T22:16:32.740Z 5 100-119 75 75 75 137 137 137 F F F
87472024-02-28T22:16:32.740Z 6 120-139 70 70 70 132 132 132 F F F
87482024-02-28T22:16:32.740Z 7 140-159 72 80 80 134 144 144 F F F <---
87492024-02-28T22:16:32.740Z 8 160-179 78 80 80 142 144 144 F F F <---
87502024-02-28T22:16:32.740Z 9 180-199 71 71 71 133 133 133 F T T <---
87512024-02-28T22:16:32.740Z 10 200-219 79 79 79 143 143 143 F T T <---
87522024-02-28T22:16:32.740Z 11 220-239 79 79 79 143 143 143 F T T <---
87532024-02-28T22:16:32.740Z 12 240-259 71 71 71 133 133 133 F F F
87542024-02-28T22:16:32.743Z 13 260-279 77 77 77 140 140 140 F F F
87552024-02-28T22:16:32.744Z 14 280-299 72 72 72 134 134 134 F F F
87562024-02-28T22:16:32.744Z 15 300-319 79 79 79 143 143 143 F F F
87572024-02-28T22:16:32.744Z 16 320-339 78 78 78 142 142 142 F F F
87582024-02-28T22:16:32.744Z 17 340-359 78 78 78 142 142 142 F F F
87592024-02-28T22:16:32.744Z 18 360-379 77 77 77 140 140 140 F T T <---
87602024-02-28T22:16:32.744Z 19 380-399 72 80 80 134 144 144 F T T <---
87612024-02-28T22:16:32.744Z 20 400-419 78 78 78 141 141 141 F T T <---
87622024-02-28T22:16:32.744Z 21 420-439 79 79 79 143 143 143 F F F
87632024-02-28T22:16:32.744Z 22 440-459 79 79 79 143 143 143 F T T <---
87642024-02-28T22:16:32.744Z 23 460-479 79 79 79 143 143 143 F F F
87652024-02-28T22:16:32.744Z 24 480-499 79 79 79 143 143 143 F F F
87662024-02-28T22:16:32.744Z 25 500-519 76 80 80 139 144 144 T F F <---
87672024-02-28T22:16:32.744Z 26 520-539 75 75 75 137 137 137 F F F
87682024-02-28T22:16:32.744Z 27 540-559 75 75 75 137 137 137 F F F
87692024-02-28T22:16:32.744Z 28 560-579 76 76 76 139 139 139 F F F
87702024-02-28T22:16:32.744Z 29 580-599 77 77 77 140 140 140 F F F
87712024-02-28T22:16:32.744ZMax gen: 80, Max flush: 144
87722024-02-28T22:16:32.744ZError: Difference in extent metadata found!
87732024-02-28T22:16:32.744ZA Difference in extent metadata is expected here
87742024-02-28T22:16:32.744ZOn loop 40
87752024-02-28T22:16:32.744Z
87762024-02-28T22:16:32.748ZVerifying data now
87772024-02-28T22:16:37.378ZNo change after verify
87782024-02-28T22:16:37.378ZLoop: 40 Downstairs dump after verify (and repair):
87792024-02-28T22:16:37.394ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87802024-02-28T22:16:37.394Z 0 000-019 80 80 80 144 144 144 F F F
87812024-02-28T22:16:37.394Z 1 020-039 74 74 74 136 136 136 F F F
87822024-02-28T22:16:37.394Z 2 040-059 81 81 81 145 145 145 F F F
87832024-02-28T22:16:37.394Z 3 060-079 78 78 78 142 142 142 F F F
87842024-02-28T22:16:37.394Z 4 080-099 67 67 67 126 126 126 F F F
87852024-02-28T22:16:37.394Z 5 100-119 75 75 75 137 137 137 F F F
87862024-02-28T22:16:37.394Z 6 120-139 70 70 70 132 132 132 F F F
87872024-02-28T22:16:37.394Z 7 140-159 80 80 80 144 144 144 F F F
87882024-02-28T22:16:37.394Z 8 160-179 80 80 80 144 144 144 F F F
87892024-02-28T22:16:37.394Z 9 180-199 81 81 81 145 145 145 F F F
87902024-02-28T22:16:37.394Z 10 200-219 81 81 81 145 145 145 F F F
87912024-02-28T22:16:37.394Z 11 220-239 81 81 81 145 145 145 F F F
87922024-02-28T22:16:37.394Z 12 240-259 71 71 71 133 133 133 F F F
87932024-02-28T22:16:37.394Z 13 260-279 77 77 77 140 140 140 F F F
87942024-02-28T22:16:37.394Z 14 280-299 72 72 72 134 134 134 F F F
87952024-02-28T22:16:37.394Z 15 300-319 79 79 79 143 143 143 F F F
87962024-02-28T22:16:37.395Z 16 320-339 78 78 78 142 142 142 F F F
87972024-02-28T22:16:37.395Z 17 340-359 78 78 78 142 142 142 F F F
87982024-02-28T22:16:37.395Z 18 360-379 81 81 81 145 145 145 F F F
87992024-02-28T22:16:37.399Z 19 380-399 81 81 81 145 145 145 F F F
88002024-02-28T22:16:37.399Z 20 400-419 81 81 81 145 145 145 F F F
88012024-02-28T22:16:37.399Z 21 420-439 79 79 79 143 143 143 F F F
88022024-02-28T22:16:37.399Z 22 440-459 81 81 81 145 145 145 F F F
88032024-02-28T22:16:37.399Z 23 460-479 79 79 79 143 143 143 F F F
88042024-02-28T22:16:37.399Z 24 480-499 79 79 79 143 143 143 F F F
88052024-02-28T22:16:37.399Z 25 500-519 80 80 80 144 144 144 F F F
88062024-02-28T22:16:37.399Z 26 520-539 75 75 75 137 137 137 F F F
88072024-02-28T22:16:37.399Z 27 540-559 75 75 75 137 137 137 F F F
88082024-02-28T22:16:37.399Z 28 560-579 76 76 76 139 139 139 F F F
88092024-02-28T22:16:37.400Z 29 580-599 77 77 77 140 140 140 F F F
88102024-02-28T22:16:37.400ZMax gen: 81, Max flush: 145
88112024-02-28T22:16:37.400Z
88122024-02-28T22:16:37.400ZBegin loop 41
88132024-02-28T22:16:37.400ZDownstairs to restart: 0
88142024-02-28T22:16:37.417Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309081826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
88152024-02-28T22:16:37.417Z{"msg":"Crucible Version: BuildInfo {\nCrucible runtime is spawned
88162024-02-28T22:16:37.423Z version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309519594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
88172024-02-28T22:16:37.424Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309560942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
88182024-02-28T22:16:37.424Z{"msg":"Crucible 2d3f0693-ecc6-4108-80c7-5c2a6bc7ac63 has session id: 08601901-2538-4179-9f23-5272e0dc4ce9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309577852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88192024-02-28T22:16:37.424Z{"msg":"Upstairs opts: Upstairs UUID: 2d3f0693-ecc6-4108-80c7-5c2a6bc7ac63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309613271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88202024-02-28T22:16:37.424Z{"msg":"Crucible stats registered with UUID: 2d3f0693-ecc6-4108-80c7-5c2a6bc7ac63","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.30963311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88212024-02-28T22:16:37.424Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.30965101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
88222024-02-28T22:16:37.424Z{"msg":"2d3f0693-ecc6-4108-80c7-5c2a6bc7ac63 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309733308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88232024-02-28T22:16:37.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309759517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88242024-02-28T22:16:37.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309775607Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88252024-02-28T22:16:37.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309797186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88262024-02-28T22:16:37.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309811826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88272024-02-28T22:16:37.424Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309825595Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88282024-02-28T22:16:37.424Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309856944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88292024-02-28T22:16:37.424Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309883344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88302024-02-28T22:16:37.424Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.309922172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88312024-02-28T22:16:37.424Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31000688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88322024-02-28T22:16:37.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.310165006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88332024-02-28T22:16:37.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.310231234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88342024-02-28T22:16:37.424Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.310263853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88352024-02-28T22:16:37.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.310977133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88362024-02-28T22:16:37.424Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311026092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88372024-02-28T22:16:37.424Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311055371Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88382024-02-28T22:16:37.425Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31109567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88392024-02-28T22:16:37.425Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311617616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88402024-02-28T22:16:37.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311665324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88412024-02-28T22:16:37.425Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311706863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88422024-02-28T22:16:37.425Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.311750752Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88432024-02-28T22:16:37.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312264018Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88442024-02-28T22:16:37.425Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312300147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88452024-02-28T22:16:37.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312315806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88462024-02-28T22:16:37.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312338046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88472024-02-28T22:16:37.425Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312353205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88482024-02-28T22:16:37.425Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312373355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88492024-02-28T22:16:37.425Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312507471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88502024-02-28T22:16:37.425Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.312979778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88512024-02-28T22:16:37.425Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313021127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88522024-02-28T22:16:37.425Z{"msg":"[0]R flush_numbers[0..12]: [144, 136, 145, 142, 126, 137, 132, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313090075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88532024-02-28T22:16:37.425Z{"msg":"[0]R generation[0..12]: [80, 74, 81, 78, 67, 75, 70, 80, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313131984Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88542024-02-28T22:16:37.425Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313169583Z","hostname":"Wait for a query_work_queue command to finish before sending IO
88552024-02-28T22:16:37.425Zip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88562024-02-28T22:16:37.430Z{"msg":"[1]R flush_numbers[0..12]: [144, 136, 145, 142, 126, 137, 132, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313227351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88572024-02-28T22:16:37.430Z{"msg":"[1]R generation[0..12]: [80, 74, 81, 78, 67, 75, 70, 80, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31325466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88582024-02-28T22:16:37.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31327125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88592024-02-28T22:16:37.431Z{"msg":"[2]R flush_numbers[0..12]: [144, 136, 145, 142, 126, 137, 132, 144, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31328565Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88602024-02-28T22:16:37.431Z{"msg":"[2]R generation[0..12]: [80, 74, 81, 78, 67, 75, 70, 80, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313312829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88612024-02-28T22:16:37.431Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313338108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88622024-02-28T22:16:37.431Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313372597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88632024-02-28T22:16:37.431Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313394047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88642024-02-28T22:16:37.431Z{"msg":"Next flush: 146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313416626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88652024-02-28T22:16:37.431Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313432906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"":"downstairs","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88662024-02-28T22:16:37.431Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313457925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88672024-02-28T22:16:37.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313492244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"0","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88682024-02-28T22:16:37.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313518033Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"1","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88692024-02-28T22:16:37.431Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313536543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"client":"2","":"downstairs","session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88702024-02-28T22:16:37.431Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313572372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88712024-02-28T22:16:37.431Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313601871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88722024-02-28T22:16:37.431Z{"msg":"2d3f0693-ecc6-4108-80c7-5c2a6bc7ac63 is now active with session: 08601901-2538-4179-9f23-5272e0dc4ce9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.31363684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88732024-02-28T22:16:37.431Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313664489Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463,"session_id":"08601901-2538-4179-9f23-5272e0dc4ce9"}
88742024-02-28T22:16:37.431Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:36.313678799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
88752024-02-28T22:16:37.431ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88762024-02-28T22:16:37.431ZLoading write count information from file "/tmp/test_repair_verify.data"
88772024-02-28T22:16:37.431ZRun Repair workload
88782024-02-28T22:16:37.431Z01/30 Write block 383 len 4096 data: 12
88792024-02-28T22:16:37.431Z02/30 Write block 143 len 24576 data: 7 8 10 12 11 12
88802024-02-28T22:16:37.432Z03/30 Read block 55 len 16384
88812024-02-28T22:16:37.432Z04/30 Write block 124 len 28672 data: 7 7 6 10 11 8 8
88822024-02-28T22:16:37.432Z05/30 Write block 55 len 32768 data: 5 6 6 6 9 10 10 11
88832024-02-28T22:16:37.432Z06/30 Read block 326 len 16384
88842024-02-28T22:16:37.432Z07/30 Write block 247 len 12288 data: 7 9 6
88852024-02-28T22:16:37.432Z08/30 Read block 363 len 36864
88862024-02-28T22:16:37.435Z09/30 Read block 204 len 32768
88872024-02-28T22:16:37.435Z10/30 Flush
88882024-02-28T22:16:37.435Z11/30 Write block 11 len 12288 data: 9 11 10
88892024-02-28T22:16:37.435Z12/30 Write block 97 len 32768 data: 4 4 4 5 6 6 7 6
88902024-02-28T22:16:37.435Z13/30 Flush
88912024-02-28T22:16:37.439Z14/30 Flush
88922024-02-28T22:16:37.442Z15/30 Write block 489 len 32768 data: 8 9 11 10 9 8 9 9
88932024-02-28T22:16:37.442Z16/30 Read block 68 len 8192
88942024-02-28T22:16:37.442Z17/30 Read block 394 len 20480
88952024-02-28T22:16:37.445Z18/30 Read block 182 len 32768
88962024-02-28T22:16:37.445Z19/30 Write block 427 len 32768 data: 4 4 5 5 5 7 8 9
88972024-02-28T22:16:37.445Z20/30 Write block 525 len 40960 data: 6 6 6 5 6 6 6 5 6 8
88982024-02-28T22:16:37.446Z21/30 Read block 438 len 4096
88992024-02-28T22:16:37.449Z22/30 Read block 315 len 4096
89002024-02-28T22:16:37.449Z23/30 Write block 579 len 4096 data: 9
89012024-02-28T22:16:37.449Z24/30 Write block 485 len 8192 data: 9 6
89022024-02-28T22:16:37.449Z25/30 Flush
89032024-02-28T22:16:37.452Z26/30 Write block 264 len 32768 data: 7 6 7 6 5 4 3 3
89042024-02-28T22:16:37.452Z27/30 Read block 271 len 12288
89052024-02-28T22:16:37.457Z28/30 Read block 461 len 4096
89062024-02-28T22:16:37.457Z29/30 Read block 454 len 40960
89072024-02-28T22:16:37.458Z30/30 Read block 295 len 12288
89082024-02-28T22:16:37.458Z----------------------------------------------------------------
89092024-02-28T22:16:37.458Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89102024-02-28T22:16:37.458ZStates: Active Active Active
89112024-02-28T22:16:37.458ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89122024-02-28T22:16:37.458Z 1 Acked 1000 Write 1 Sent Done Done false
89132024-02-28T22:16:37.458Z 2 Acked 1001 Write 6 Sent Done Done false
89142024-02-28T22:16:37.458Z 3 Acked 1002 Read 4 Sent Done Done false
89152024-02-28T22:16:37.458Z 4 Acked 1003 Write 7 Sent Done Done false
89162024-02-28T22:16:37.458Z 5 Acked 1004 Write 8 Sent Done Done false
89172024-02-28T22:16:37.458Z 6 Acked 1005 Read 4 Sent Done Done false
89182024-02-28T22:16:37.458Z 7 Acked 1006 Write 3 Sent Done Done false
89192024-02-28T22:16:37.459Z 8 Acked 1007 Read 9 Sent Done Done false
89202024-02-28T22:16:37.459Z 9 Acked 1008 Read 8 Sent Done Done false
89212024-02-28T22:16:37.459Z 10 Acked 1009 Flush 0 Sent Done Done false
89222024-02-28T22:16:37.459Z 11 Acked 1010 Write 3 Sent Done Done false
89232024-02-28T22:16:37.459Z 12 Acked 1011 Write 8 Sent Done Done false
89242024-02-28T22:16:37.459Z 13 Acked 1012 Flush 0 Sent Done Done false
89252024-02-28T22:16:37.459Z 14 Acked 1013 Flush 0 Sent Done Done false
89262024-02-28T22:16:37.459Z 15 Acked 1014 Write 8 Sent Done Done false
89272024-02-28T22:16:37.459Z 16 Acked 1015 Read 2 Sent Done Done false
89282024-02-28T22:16:37.459Z 17 Acked 1016 Read 5 Sent Done Done false
89292024-02-28T22:16:37.459Z 18 Acked 1017 Read 8 Sent Done Done false
89302024-02-28T22:16:37.459Z 19 Acked 1018 Write 8 Sent Done Done false
89312024-02-28T22:16:37.459Z 20 Acked 1019 Write 10 Sent Done Done false
89322024-02-28T22:16:37.460Z 21 Acked 1020 Read 1 Sent Done Done false
89332024-02-28T22:16:37.460Z 22 Acked 1021 Read 1 Sent Done Done false
89342024-02-28T22:16:37.460Z 23 Acked 1022 Write 1 Sent Done Done false
89352024-02-28T22:16:37.460Z 24 Acked 1023 Write 2 Sent Done Done false
89362024-02-28T22:16:37.460Z 25 Acked 1024 Flush 0 Sent Done Done false
89372024-02-28T22:16:37.460Z 26 Acked 1025 Write 8 Sent Done Done false
89382024-02-28T22:16:37.460Z 27 Acked 1026 Read 3 Sent Done Done false
89392024-02-28T22:16:37.460Z 28 Acked 1027 Read 1 Sent Done Done false
89402024-02-28T22:16:37.460Z 29 Acked 1028 Read 10 Sent Done Done false
89412024-02-28T22:16:37.460Z 30 Acked 1029 Read 3 Sent Done Done false
89422024-02-28T22:16:37.460Z STATES DS:0 DS:1 DS:2 TOTAL
89432024-02-28T22:16:37.460Z New 0 0 0 0
89442024-02-28T22:16:37.461Z Sent 30 0 0 30
89452024-02-28T22:16:37.461Z Done 0 30 30 60
89462024-02-28T22:16:37.461Z Skipped 0 0 0 0
89472024-02-28T22:16:37.461Z Error 0 0 0 0
89482024-02-28T22:16:37.461ZLast Flush: 0 1024 1024
89492024-02-28T22:16:37.461ZDownstairs last five completed:
89502024-02-28T22:16:37.461ZUpstairs last five completed: 30 29 28 27 26
89512024-02-28T22:16:37.461Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:36.347585545Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1463}
89522024-02-28T22:16:37.461ZWrote out file "/tmp/test_repair_verify.data"
89532024-02-28T22:16:37.465Z
89542024-02-28T22:16:37.465ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89552024-02-28T22:16:37.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89562024-02-28T22:16:37.477Z 0 000-019 80 82 82 144 147 147 F F F <---
89572024-02-28T22:16:37.477Z 1 020-039 74 74 74 136 136 136 F F F
89582024-02-28T22:16:37.477Z 2 040-059 81 82 82 145 146 146 F F F <---
89592024-02-28T22:16:37.477Z 3 060-079 78 82 82 142 146 146 F F F <---
89602024-02-28T22:16:37.477Z 4 080-099 67 82 82 126 147 147 F F F <---
89612024-02-28T22:16:37.478Z 5 100-119 75 82 82 137 147 147 F F F <---
89622024-02-28T22:16:37.478Z 6 120-139 70 82 82 132 146 146 F F F <---
89632024-02-28T22:16:37.478Z 7 140-159 80 82 82 144 146 146 F F F <---
89642024-02-28T22:16:37.478Z 8 160-179 80 80 80 144 144 144 F F F
89652024-02-28T22:16:37.478Z 9 180-199 81 81 81 145 145 145 F F F
89662024-02-28T22:16:37.478Z 10 200-219 81 81 81 145 145 145 F F F
89672024-02-28T22:16:37.481Z 11 220-239 81 81 81 145 145 145 F F F
89682024-02-28T22:16:37.481Z 12 240-259 71 82 82 133 146 146 F F F <---
89692024-02-28T22:16:37.481Z 13 260-279 77 77 77 140 140 140 F T T <---
89702024-02-28T22:16:37.481Z 14 280-299 72 72 72 134 134 134 F F F
89712024-02-28T22:16:37.481Z 15 300-319 79 79 79 143 143 143 F F F
89722024-02-28T22:16:37.481Z 16 320-339 78 78 78 142 142 142 F F F
89732024-02-28T22:16:37.481Z 17 340-359 78 78 78 142 142 142 F F F
89742024-02-28T22:16:37.481Z 18 360-379 81 81 81 145 145 145 F F F
89752024-02-28T22:16:37.481Z 19 380-399 81 82 82 145 146 146 F F F <---
89762024-02-28T22:16:37.482Z 20 400-419 81 81 81 145 145 145 F F F
89772024-02-28T22:16:37.482Z 21 420-439 79 82 82 143 149 149 F F F <---
89782024-02-28T22:16:37.482Z 22 440-459 81 81 81 145 145 145 F F F
89792024-02-28T22:16:37.482Z 23 460-479 79 79 79 143 143 143 F F F
89802024-02-28T22:16:37.482Z 24 480-499 79 82 82 143 149 149 F F F <---
89812024-02-28T22:16:37.482Z 25 500-519 80 80 80 144 144 144 F F F
89822024-02-28T22:16:37.482Z 26 520-539 75 82 82 137 149 149 F F F <---
89832024-02-28T22:16:37.482Z 27 540-559 75 75 75 137 137 137 F F F
89842024-02-28T22:16:37.482Z 28 560-579 76 82 82 139 149 149 F F F <---
89852024-02-28T22:16:37.482Z 29 580-599 77 77 77 140 140 140 F F F
89862024-02-28T22:16:37.482ZMax gen: 82, Max flush: 149
89872024-02-28T22:16:37.482ZError: Difference in extent metadata found!
89882024-02-28T22:16:37.482ZA Difference in extent metadata is expected here
89892024-02-28T22:16:37.482ZOn loop 41
89902024-02-28T22:16:37.482Z
89912024-02-28T22:16:37.485ZVerifying data now
89922024-02-28T22:16:42.170ZNo change after verify
89932024-02-28T22:16:42.170ZLoop: 41 Downstairs dump after verify (and repair):
89942024-02-28T22:16:42.184ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89952024-02-28T22:16:42.184Z 0 000-019 82 82 82 147 147 147 F F F
89962024-02-28T22:16:42.184Z 1 020-039 74 74 74 136 136 136 F F F
89972024-02-28T22:16:42.184Z 2 040-059 82 82 82 146 146 146 F F F
89982024-02-28T22:16:42.184Z 3 060-079 82 82 82 146 146 146 F F F
89992024-02-28T22:16:42.184Z 4 080-099 82 82 82 147 147 147 F F F
90002024-02-28T22:16:42.184Z 5 100-119 82 82 82 147 147 147 F F F
90012024-02-28T22:16:42.184Z 6 120-139 82 82 82 146 146 146 F F F
90022024-02-28T22:16:42.185Z 7 140-159 82 82 82 146 146 146 F F F
90032024-02-28T22:16:42.185Z 8 160-179 80 80 80 144 144 144 F F F
90042024-02-28T22:16:42.185Z 9 180-199 81 81 81 145 145 145 F F F
90052024-02-28T22:16:42.188Z 10 200-219 81 81 81 145 145 145 F F F
90062024-02-28T22:16:42.188Z 11 220-239 81 81 81 145 145 145 F F F
90072024-02-28T22:16:42.188Z 12 240-259 82 82 82 146 146 146 F F F
90082024-02-28T22:16:42.188Z 13 260-279 83 83 83 150 150 150 F F F
90092024-02-28T22:16:42.189Z 14 280-299 72 72 72 134 134 134 F F F
90102024-02-28T22:16:42.189Z 15 300-319 79 79 79 143 143 143 F F F
90112024-02-28T22:16:42.189Z 16 320-339 78 78 78 142 142 142 F F F
90122024-02-28T22:16:42.189Z 17 340-359 78 78 78 142 142 142 F F F
90132024-02-28T22:16:42.189Z 18 360-379 81 81 81 145 145 145 F F F
90142024-02-28T22:16:42.189Z 19 380-399 82 82 82 146 146 146 F F F
90152024-02-28T22:16:42.189Z 20 400-419 81 81 81 145 145 145 F F F
90162024-02-28T22:16:42.189Z 21 420-439 82 82 82 149 149 149 F F F
90172024-02-28T22:16:42.189Z 22 440-459 81 81 81 145 145 145 F F F
90182024-02-28T22:16:42.189Z 23 460-479 79 79 79 143 143 143 F F F
90192024-02-28T22:16:42.189Z 24 480-499 82 82 82 149 149 149 F F F
90202024-02-28T22:16:42.189Z 25 500-519 80 80 80 144 144 144 F F F
90212024-02-28T22:16:42.189Z 26 520-539 82 82 82 149 149 149 F F F
90222024-02-28T22:16:42.189Z 27 540-559 75 75 75 137 137 137 F F F
90232024-02-28T22:16:42.189Z 28 560-579 82 82 82 149 149 149 F F F
90242024-02-28T22:16:42.189Z 29 580-599 77 77 77 140 140 140 F F F
90252024-02-28T22:16:42.189ZMax gen: 83, Max flush: 150
90262024-02-28T22:16:42.189Z
90272024-02-28T22:16:42.189ZBegin loop 42
90282024-02-28T22:16:42.189ZDownstairs to restart: 0
90292024-02-28T22:16:42.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.096989723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
90302024-02-28T22:16:42.205Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097343924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
90312024-02-28T22:16:42.212Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
90322024-02-28T22:16:42.212Z,"time":"2024-02-28T22:16:41.097378243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
90332024-02-28T22:16:42.212Z{"msg":"Crucible e2bb3e50-9e10-499b-bdcf-eaefe2ef5e1e has session id: 5dd1da68-7d78-4d7f-999d-cfb110daf121","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097416252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90342024-02-28T22:16:42.212Z{"msg":"Upstairs opts: Upstairs UUID: e2bb3e50-9e10-499b-bdcf-eaefe2ef5e1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097452071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90352024-02-28T22:16:42.212Z{"msg":"Crucible stats registered with UUID: e2bb3e50-9e10-499b-bdcf-eaefe2ef5e1e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.09747931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90362024-02-28T22:16:42.212Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097515899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
90372024-02-28T22:16:42.212Z{"msg":"e2bb3e50-9e10-499b-bdcf-eaefe2ef5e1e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097569217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90382024-02-28T22:16:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097605976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90392024-02-28T22:16:42.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097624736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90402024-02-28T22:16:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097643475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90412024-02-28T22:16:42.212Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097667575Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90422024-02-28T22:16:42.212Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097706924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90432024-02-28T22:16:42.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097796321Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90442024-02-28T22:16:42.213Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.09784451Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90452024-02-28T22:16:42.213Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097888219Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90462024-02-28T22:16:42.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097926638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90472024-02-28T22:16:42.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.097973186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90482024-02-28T22:16:42.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098009055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90492024-02-28T22:16:42.213Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098036004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90502024-02-28T22:16:42.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098756145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90512024-02-28T22:16:42.213Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098793274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90522024-02-28T22:16:42.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098810143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90532024-02-28T22:16:42.213Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098824393Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90542024-02-28T22:16:42.213Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.098903701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90552024-02-28T22:16:42.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.09893645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90562024-02-28T22:16:42.213Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099341208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90572024-02-28T22:16:42.213Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099378947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90582024-02-28T22:16:42.213Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099632311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90592024-02-28T22:16:42.213Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099822405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90602024-02-28T22:16:42.213Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099859534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90612024-02-28T22:16:42.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099875634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90622024-02-28T22:16:42.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099955012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90632024-02-28T22:16:42.214Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.099985571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90642024-02-28T22:16:42.214Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.10002283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90652024-02-28T22:16:42.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100132597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90662024-02-28T22:16:42.214Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100176216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90672024-02-28T22:16:42.214Z{"msg":"[0]R flush_numbers[0..12]: [147, 136, 146, 146, 147, 147, 146, 146, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100202025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90682024-02-28T22:16:42.214Z{"msg":"[0]R generation[0..12]: [82, 74, 82, 82, 82, 82, 82, 82, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100217575Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90692024-02-28T22:16:42.215Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100232324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90702024-02-28T22:16:42.215Z{"msg":"[1]R flush_numbers[0..12]: [147, 136, 146, 146, 147, 147, 146, 146, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100246564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90712024-02-28T22:16:42.215Z{"msg":"[1]R generation[0..12]: [82, 74, 82, 82, 82, 82, 82, 82, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100260363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90722024-02-28T22:16:42.215Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100275043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90732024-02-28T22:16:42.215Z{"msg":"[2]R flush_numbers[0..12]: [147, 136, 146, 146, 147, 147, 146, 146, 144, 145, 145, 145]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100296192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90742024-02-28T22:16:42.215ZWait for a query_work_queue command to finish before sending IO
90752024-02-28T22:16:42.216Z{"msg":"[2]R generation[0..12]: [82, 74, 82, 82, 82, 82, 82, 82, 80, 81, 81, 81]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100336381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90762024-02-28T22:16:42.216Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.10036474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90772024-02-28T22:16:42.216Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.10038919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90782024-02-28T22:16:42.216Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100404169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90792024-02-28T22:16:42.216Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100420209Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90802024-02-28T22:16:42.216Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100453868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"":"downstairs","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90812024-02-28T22:16:42.217Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100482447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90822024-02-28T22:16:42.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100499077Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"0","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90832024-02-28T22:16:42.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100512726Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"1","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90842024-02-28T22:16:42.217Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100526656Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"client":"2","":"downstairs","session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90852024-02-28T22:16:42.217Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100560045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90862024-02-28T22:16:42.217Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100585384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90872024-02-28T22:16:42.218Z{"msg":"e2bb3e50-9e10-499b-bdcf-eaefe2ef5e1e is now active with session: 5dd1da68-7d78-4d7f-999d-cfb110daf121","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100614084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90882024-02-28T22:16:42.218Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100627583Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471,"session_id":"5dd1da68-7d78-4d7f-999d-cfb110daf121"}
90892024-02-28T22:16:42.218Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:41.100643723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
90902024-02-28T22:16:42.227ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90912024-02-28T22:16:42.227ZLoading write count information from file "/tmp/test_repair_verify.data"
90922024-02-28T22:16:42.227ZRun Repair workload
90932024-02-28T22:16:42.228Z01/30 Write block 567 len 20480 data: 12 11 10 9 8
90942024-02-28T22:16:42.228Z02/30 Read block 246 len 32768
90952024-02-28T22:16:42.228Z03/30 Write block 386 len 28672 data: 10 7 6 5 6 7 6
90962024-02-28T22:16:42.228Z04/30 Write block 449 len 12288 data: 8 8 8
90972024-02-28T22:16:42.228Z05/30 Write block 515 len 12288 data: 6 6 6
90982024-02-28T22:16:42.228Z06/30 Write block 0 len 4096 data: 2
90992024-02-28T22:16:42.228Z07/30 Write block 266 len 32768 data: 8 7 6 5 4 4 3 2
91002024-02-28T22:16:42.228Z08/30 Read block 456 len 8192
91012024-02-28T22:16:42.228Z09/30 Read block 125 len 4096
91022024-02-28T22:16:42.228Z10/30 Write block 2 len 28672 data: 4 6 8 8 9 11 12
91032024-02-28T22:16:42.228Z11/30 Flush
91042024-02-28T22:16:42.228Z12/30 Write block 55 len 24576 data: 6 7 7 7 10 11
91052024-02-28T22:16:42.228Z13/30 Read block 432 len 36864
91062024-02-28T22:16:42.229Z14/30 Read block 406 len 4096
91072024-02-28T22:16:42.229Z15/30 Read block 221 len 12288
91082024-02-28T22:16:42.229Z16/30 Read block 19 len 12288
91092024-02-28T22:16:42.229Z17/30 Read block 329 len 16384
91102024-02-28T22:16:42.233Z18/30 Write block 237 len 4096 data: 5
91112024-02-28T22:16:42.233Z19/30 Flush
91122024-02-28T22:16:42.233Z20/30 Write block 350 len 20480 data: 6 6 7 8 8
91132024-02-28T22:16:42.233Z21/30 Read block 14 len 8192
91142024-02-28T22:16:42.233Z22/30 Flush
91152024-02-28T22:16:42.242Z23/30 Write block 355 len 12288 data: 7 7 7
91162024-02-28T22:16:42.242Z24/30 Read block 17 len 20480
91172024-02-28T22:16:42.242Z25/30 Flush
91182024-02-28T22:16:42.257Z26/30 Write block 334 len 16384 data: 9 9 9 7
91192024-02-28T22:16:42.258Z27/30 Read block 44 len 36864
91202024-02-28T22:16:42.258Z28/30 Write block 512 len 8192 data: 7 6
91212024-02-28T22:16:42.258Z29/30 Write block 509 len 36864 data: 12 10 8 8 7 6 7 7 7
91222024-02-28T22:16:42.258Z30/30 Write block 536 len 24576 data: 7 6 6 6 6 7
91232024-02-28T22:16:42.258Z----------------------------------------------------------------
91242024-02-28T22:16:42.258Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91252024-02-28T22:16:42.258ZStates: Active Active Active
91262024-02-28T22:16:42.258ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91272024-02-28T22:16:42.258Z 1 Acked 1000 Write 5 Done Done Done false
91282024-02-28T22:16:42.258Z 2 Acked 1001 Read 8 Done Done Done false
91292024-02-28T22:16:42.258Z 3 Acked 1002 Write 7 Done Done Done false
91302024-02-28T22:16:42.258Z 4 Acked 1003 Write 3 Done Done Done false
91312024-02-28T22:16:42.258Z 5 Acked 1004 Write 3 Sent Done Done false
91322024-02-28T22:16:42.259Z 6 Acked 1005 Write 1 Sent Done Done false
91332024-02-28T22:16:42.259Z 7 Acked 1006 Write 8 Sent Done Done false
91342024-02-28T22:16:42.259Z 8 Acked 1007 Read 2 Sent Done Done false
91352024-02-28T22:16:42.259Z 9 Acked 1008 Read 1 Sent Done Done false
91362024-02-28T22:16:42.259Z 10 Acked 1009 Write 7 Sent Done Done false
91372024-02-28T22:16:42.259Z 11 Acked 1010 Flush 0 Sent Done Done false
91382024-02-28T22:16:42.259Z 12 Acked 1011 Write 6 Sent Done Done false
91392024-02-28T22:16:42.259Z 13 Acked 1012 Read 9 Sent Done Done false
91402024-02-28T22:16:42.259Z 14 Acked 1013 Read 1 Sent Done Done false
91412024-02-28T22:16:42.259Z 15 Acked 1014 Read 3 Sent Done Done false
91422024-02-28T22:16:42.259Z 16 Acked 1015 Read 3 Sent Done Done false
91432024-02-28T22:16:42.259Z 17 Acked 1016 Read 4 Sent Done Done false
91442024-02-28T22:16:42.259Z 18 Acked 1017 Write 1 Sent Done Done false
91452024-02-28T22:16:42.259Z 19 Acked 1018 Flush 0 Sent Done Done false
91462024-02-28T22:16:42.259Z 20 Acked 1019 Write 5 Sent Done Done false
91472024-02-28T22:16:42.259Z 21 Acked 1020 Read 2 Sent Done Done false
91482024-02-28T22:16:42.259Z 22 Acked 1021 Flush 0 Sent Done Done false
91492024-02-28T22:16:42.259Z 23 Acked 1022 Write 3 Sent Done Done false
91502024-02-28T22:16:42.259Z 24 Acked 1023 Read 5 Sent Done Done false
91512024-02-28T22:16:42.259Z 25 Acked 1024 Flush 0 Sent Done Done false
91522024-02-28T22:16:42.259Z 26 Acked 1025 Write 4 Sent Done Done false
91532024-02-28T22:16:42.259Z 27 Acked 1026 Read 9 Sent Done Done false
91542024-02-28T22:16:42.259Z 28 Acked 1027 Write 2 Sent Done Done false
91552024-02-28T22:16:42.260Z 29 Acked 1028 Write 9 Sent Done Done false
91562024-02-28T22:16:42.260Z 30 Acked 1029 Write 6 Sent Sent Sent false
91572024-02-28T22:16:42.260Z STATES DS:0 DS:1 DS:2 TOTAL
91582024-02-28T22:16:42.260Z New 0 0 0 0
91592024-02-28T22:16:42.260Z Sent 26 1 1 28
91602024-02-28T22:16:42.260Z Done 4 29 29 62
91612024-02-28T22:16:42.260Z Skipped 0 0 0 0
91622024-02-28T22:16:42.260Z Error 0 0 0 0
91632024-02-28T22:16:42.260ZLast Flush: 0 1024 1024
91642024-02-28T22:16:42.260ZDownstairs last five completed:
91652024-02-28T22:16:42.260ZUpstairs last five completed: 30 29 28 27 26
91662024-02-28T22:16:42.260Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:41.133912576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1471}
91672024-02-28T22:16:42.260ZWrote out file "/tmp/test_repair_verify.data"
91682024-02-28T22:16:42.260Z
91692024-02-28T22:16:42.266ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91702024-02-28T22:16:42.270ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91712024-02-28T22:16:42.270Z 0 000-019 82 84 84 147 151 151 F F F <---
91722024-02-28T22:16:42.270Z 1 020-039 74 74 74 136 136 136 F F F
91732024-02-28T22:16:42.270Z 2 040-059 82 84 84 146 152 152 F F F <---
91742024-02-28T22:16:42.270Z 3 060-079 82 84 84 146 152 152 F F F <---
91752024-02-28T22:16:42.271Z 4 080-099 82 82 82 147 147 147 F F F
91762024-02-28T22:16:42.271Z 5 100-119 82 82 82 147 147 147 F F F
91772024-02-28T22:16:42.271Z 6 120-139 82 82 82 146 146 146 F F F
91782024-02-28T22:16:42.271Z 7 140-159 82 82 82 146 146 146 F F F
91792024-02-28T22:16:42.271Z 8 160-179 80 80 80 144 144 144 F F F
91802024-02-28T22:16:42.271Z 9 180-199 81 81 81 145 145 145 F F F
91812024-02-28T22:16:42.271Z 10 200-219 81 81 81 145 145 145 F F F
91822024-02-28T22:16:42.271Z 11 220-239 81 84 84 145 152 152 F F F <---
91832024-02-28T22:16:42.271Z 12 240-259 82 82 82 146 146 146 F F F
91842024-02-28T22:16:42.271Z 13 260-279 83 84 84 150 151 151 F F F <---
91852024-02-28T22:16:42.271Z 14 280-299 72 72 72 134 134 134 F F F
91862024-02-28T22:16:42.271Z 15 300-319 79 79 79 143 143 143 F F F
91872024-02-28T22:16:42.271Z 16 320-339 78 78 78 142 142 142 F T T <---
91882024-02-28T22:16:42.271Z 17 340-359 78 84 84 142 154 154 F F F <---
91892024-02-28T22:16:42.271Z 18 360-379 81 81 81 145 145 145 F F F
91902024-02-28T22:16:42.271Z 19 380-399 82 84 84 146 151 151 T F F <---
91912024-02-28T22:16:42.271Z 20 400-419 81 81 81 145 145 145 F F F
91922024-02-28T22:16:42.271Z 21 420-439 82 82 82 149 149 149 F F F
91932024-02-28T22:16:42.271Z 22 440-459 81 84 84 145 151 151 T F F <---
91942024-02-28T22:16:42.271Z 23 460-479 79 79 79 143 143 143 F F F
91952024-02-28T22:16:42.271Z 24 480-499 82 82 82 149 149 149 F F F
91962024-02-28T22:16:42.271Z 25 500-519 80 84 84 144 151 151 F T T <---
91972024-02-28T22:16:42.271Z 26 520-539 82 82 82 149 149 149 F T T <---
91982024-02-28T22:16:42.271Z 27 540-559 75 75 75 137 137 137 F T T <---
91992024-02-28T22:16:42.272Z 28 560-579 82 84 84 149 151 151 T F F <---
92002024-02-28T22:16:42.272Z 29 580-599 77 77 77 140 140 140 F F F
92012024-02-28T22:16:42.272ZMax gen: 84, Max flush: 154
92022024-02-28T22:16:42.272ZError: Difference in extent metadata found!
92032024-02-28T22:16:42.272ZA Difference in extent metadata is expected here
92042024-02-28T22:16:42.272ZOn loop 42
92052024-02-28T22:16:42.272Z
92062024-02-28T22:16:42.280ZVerifying data now
92072024-02-28T22:16:47.931ZNo change after verify
92082024-02-28T22:16:47.931ZLoop: 42 Downstairs dump after verify (and repair):
92092024-02-28T22:16:47.945ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92102024-02-28T22:16:47.946Z 0 000-019 84 84 84 151 151 151 F F F
92112024-02-28T22:16:47.946Z 1 020-039 74 74 74 136 136 136 F F F
92122024-02-28T22:16:47.946Z 2 040-059 84 84 84 152 152 152 F F F
92132024-02-28T22:16:47.946Z 3 060-079 84 84 84 152 152 152 F F F
92142024-02-28T22:16:47.946Z 4 080-099 82 82 82 147 147 147 F F F
92152024-02-28T22:16:47.946Z 5 100-119 82 82 82 147 147 147 F F F
92162024-02-28T22:16:47.946Z 6 120-139 82 82 82 146 146 146 F F F
92172024-02-28T22:16:47.949Z 7 140-159 82 82 82 146 146 146 F F F
92182024-02-28T22:16:47.949Z 8 160-179 80 80 80 144 144 144 F F F
92192024-02-28T22:16:47.950Z 9 180-199 81 81 81 145 145 145 F F F
92202024-02-28T22:16:47.950Z 10 200-219 81 81 81 145 145 145 F F F
92212024-02-28T22:16:47.950Z 11 220-239 84 84 84 152 152 152 F F F
92222024-02-28T22:16:47.950Z 12 240-259 82 82 82 146 146 146 F F F
92232024-02-28T22:16:47.950Z 13 260-279 84 84 84 151 151 151 F F F
92242024-02-28T22:16:47.950Z 14 280-299 72 72 72 134 134 134 F F F
92252024-02-28T22:16:47.950Z 15 300-319 79 79 79 143 143 143 F F F
92262024-02-28T22:16:47.950Z 16 320-339 85 85 85 155 155 155 F F F
92272024-02-28T22:16:47.950Z 17 340-359 84 84 84 154 154 154 F F F
92282024-02-28T22:16:47.950Z 18 360-379 81 81 81 145 145 145 F F F
92292024-02-28T22:16:47.950Z 19 380-399 84 84 84 151 151 151 F F F
92302024-02-28T22:16:47.950Z 20 400-419 81 81 81 145 145 145 F F F
92312024-02-28T22:16:47.950Z 21 420-439 82 82 82 149 149 149 F F F
92322024-02-28T22:16:47.950Z 22 440-459 84 84 84 151 151 151 F F F
92332024-02-28T22:16:47.950Z 23 460-479 79 79 79 143 143 143 F F F
92342024-02-28T22:16:47.950Z 24 480-499 82 82 82 149 149 149 F F F
92352024-02-28T22:16:47.950Z 25 500-519 85 85 85 155 155 155 F F F
92362024-02-28T22:16:47.950Z 26 520-539 85 85 85 155 155 155 F F F
92372024-02-28T22:16:47.950Z 27 540-559 85 85 85 155 155 155 F F F
92382024-02-28T22:16:47.950Z 28 560-579 84 84 84 151 151 151 F F F
92392024-02-28T22:16:47.950Z 29 580-599 77 77 77 140 140 140 F F F
92402024-02-28T22:16:47.950ZMax gen: 85, Max flush: 155
92412024-02-28T22:16:47.950Z
92422024-02-28T22:16:47.951ZBegin loop 43
92432024-02-28T22:16:47.951ZDownstairs to restart: 1
92442024-02-28T22:16:47.969Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860404197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479}
92452024-02-28T22:16:47.969Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860715528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479Crucible runtime is spawned
92462024-02-28T22:16:47.969Z}
92472024-02-28T22:16:47.975Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860762967Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479}
92482024-02-28T22:16:47.975Z{"msg":"Crucible b71a5c1e-894f-4978-88c7-031e8b7a3758 has session id: 3db8be9f-c1a5-45d1-982d-f84d774477e1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860801336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92492024-02-28T22:16:47.975Z{"msg":"Upstairs opts: Upstairs UUID: b71a5c1e-894f-4978-88c7-031e8b7a3758, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860852115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92502024-02-28T22:16:47.975Z{"msg":"Crucible stats registered with UUID: b71a5c1e-894f-4978-88c7-031e8b7a3758","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860894274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92512024-02-28T22:16:47.976Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860932892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479}
92522024-02-28T22:16:47.976Z{"msg":"b71a5c1e-894f-4978-88c7-031e8b7a3758 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860965601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92532024-02-28T22:16:47.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.860999731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92542024-02-28T22:16:47.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86101926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92552024-02-28T22:16:47.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861065809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92562024-02-28T22:16:47.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861111988Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92572024-02-28T22:16:47.976Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861139347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92582024-02-28T22:16:47.976Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861166986Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92592024-02-28T22:16:47.976Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861203555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92602024-02-28T22:16:47.976Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861255504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92612024-02-28T22:16:47.976Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861297102Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92622024-02-28T22:16:47.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861337111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92632024-02-28T22:16:47.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86137559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92642024-02-28T22:16:47.976Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.861504217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92652024-02-28T22:16:47.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862065631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92662024-02-28T22:16:47.976Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86210196Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92672024-02-28T22:16:47.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86211813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92682024-02-28T22:16:47.976Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862132709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92692024-02-28T22:16:47.976Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862169198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92702024-02-28T22:16:47.976Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862191588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92712024-02-28T22:16:47.976Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862602606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92722024-02-28T22:16:47.976Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862643385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92732024-02-28T22:16:47.977Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.862864649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92742024-02-28T22:16:47.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863054184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92752024-02-28T22:16:47.977Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863095363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92762024-02-28T22:16:47.977Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863121202Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92772024-02-28T22:16:47.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86321078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92782024-02-28T22:16:47.977Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863244239Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92792024-02-28T22:16:47.977Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863269518Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92802024-02-28T22:16:47.977Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863428714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92812024-02-28T22:16:47.977Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863486292Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92822024-02-28T22:16:47.977Z{"msg":"[0]R flush_numbers[0..12]: [151, 136, 152, 152, 147, 147, 146, 146, 144, 145, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863525741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92832024-02-28T22:16:47.977Z{"msg":"[0]R generation[0..12]: [84, 74, 84, 84, 82, 82, 82, 82, 80, 81, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86356085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92842024-02-28T22:16:47.977Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863595379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92852024-02-28T22:16:47.977Z{"msg":"[1]R flush_numbers[0..12]: [151, 136, 152, 152, 147, 147, 146, 146, 144, 145, 145, 152]","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO
92862024-02-28T22:16:47.977Z"time":"2024-02-28T22:16:46.863630338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92872024-02-28T22:16:47.977Z{"msg":"[1]R generation[0..12]: [84, 74, 84, 84, 82, 82, 82, 82, 80, 81, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863657317Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92882024-02-28T22:16:47.977Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863686577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92892024-02-28T22:16:47.977Z{"msg":"[2]R flush_numbers[0..12]: [151, 136, 152, 152, 147, 147, 146, 146, 144, 145, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863706636Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92902024-02-28T22:16:47.977Z{"msg":"[2]R generation[0..12]: [84, 74, 84, 84, 82, 82, 82, 82, 80, 81, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863732355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92912024-02-28T22:16:47.977Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863753015Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92922024-02-28T22:16:47.977Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863767574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92932024-02-28T22:16:47.977Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863789714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92942024-02-28T22:16:47.977Z{"msg":"Next flush: 156","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863803373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92952024-02-28T22:16:47.977Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863817453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"":"downstairs","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92962024-02-28T22:16:47.977Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863838592Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92972024-02-28T22:16:47.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863851372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"0","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92982024-02-28T22:16:47.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863865422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"1","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
92992024-02-28T22:16:47.978Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863885691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"client":"2","":"downstairs","session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
93002024-02-28T22:16:47.978Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863899541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
93012024-02-28T22:16:47.978Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.8639126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
93022024-02-28T22:16:47.978Z{"msg":"b71a5c1e-894f-4978-88c7-031e8b7a3758 is now active with session: 3db8be9f-c1a5-45d1-982d-f84d774477e1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.86393276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
93032024-02-28T22:16:47.978Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863945549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479,"session_id":"3db8be9f-c1a5-45d1-982d-f84d774477e1"}
93042024-02-28T22:16:47.978Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:46.863957969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479}
93052024-02-28T22:16:47.978ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93062024-02-28T22:16:47.983ZLoading write count information from file "/tmp/test_repair_verify.data"
93072024-02-28T22:16:47.983ZRun Repair workload
93082024-02-28T22:16:47.983Z01/30 Write block 324 len 12288 data: 6 7 8
93092024-02-28T22:16:47.983Z02/30 Flush
93102024-02-28T22:16:47.983Z03/30 Write block 390 len 4096 data: 7
93112024-02-28T22:16:47.983Z04/30 Read block 537 len 8192
93122024-02-28T22:16:47.983Z05/30 Read block 583 len 32768
93132024-02-28T22:16:47.983Z06/30 Write block 114 len 32768 data: 8 8 7 7 5 5 6 8
93142024-02-28T22:16:47.983Z07/30 Write block 319 len 40960 data: 5 5 4 5 5 7 8 9 10 12
93152024-02-28T22:16:47.983Z08/30 Write block 444 len 8192 data: 7 7
93162024-02-28T22:16:47.983Z09/30 Read block 270 len 40960
93172024-02-28T22:16:47.987Z10/30 Write block 87 len 28672 data: 7 7 6 7 6 7 5
93182024-02-28T22:16:47.987Z11/30 Read block 275 len 24576
93192024-02-28T22:16:47.987Z12/30 Write block 122 len 4096 data: 7
93202024-02-28T22:16:47.987Z13/30 Read block 250 len 20480
93212024-02-28T22:16:47.987Z14/30 Read block 530 len 36864
93222024-02-28T22:16:47.993Z15/30 Read block 256 len 24576
93232024-02-28T22:16:47.993Z16/30 Read block 549 len 28672
93242024-02-28T22:16:47.993Z17/30 Write block 185 len 4096 data: 8
93252024-02-28T22:16:47.993Z18/30 Flush
93262024-02-28T22:16:47.996Z19/30 Write block 136 len 40960 data: 8 8 8 8 7 6 6 8 9 11
93272024-02-28T22:16:47.996Z20/30 Read block 84 len 32768
93282024-02-28T22:16:47.996Z21/30 Write block 392 len 32768 data: 7 6 5 7 7 7 7 6
93292024-02-28T22:16:47.996Z22/30 Write block 11 len 28672 data: 10 12 11 8 8 9 8
93302024-02-28T22:16:48.000Z23/30 Write block 7 len 24576 data: 12 13 10 9 11 13
93312024-02-28T22:16:48.000Z24/30 Read block 182 len 24576
93322024-02-28T22:16:48.000Z25/30 Read block 248 len 24576
93332024-02-28T22:16:48.000Z26/30 Read block 435 len 40960
93342024-02-28T22:16:48.003Z27/30 Read block 377 len 36864
93352024-02-28T22:16:48.003Z28/30 Read block 450 len 8192
93362024-02-28T22:16:48.003Z29/30 Read block 181 len 8192
93372024-02-28T22:16:48.003Z30/30 Read block 63 len 8192
93382024-02-28T22:16:48.003Z----------------------------------------------------------------
93392024-02-28T22:16:48.003Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:28
93402024-02-28T22:16:48.003ZStates: Active Active Active
93412024-02-28T22:16:48.003ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93422024-02-28T22:16:48.007Z 3 Acked 1002 Write 1 Done Sent Done false
93432024-02-28T22:16:48.007Z 4 Acked 1003 Read 2 Done Sent Done false
93442024-02-28T22:16:48.007Z 5 Acked 1004 Read 8 Done Sent Done false
93452024-02-28T22:16:48.007Z 6 Acked 1005 Write 8 Done Sent Done false
93462024-02-28T22:16:48.007Z 7 Acked 1006 Write 10 Done Sent Done false
93472024-02-28T22:16:48.007Z 8 Acked 1007 Write 2 Done Sent Done false
93482024-02-28T22:16:48.007Z 9 Acked 1008 Read 10 Done Sent Done false
93492024-02-28T22:16:48.007Z 10 Acked 1009 Write 7 Done Sent Done false
93502024-02-28T22:16:48.007Z 11 Acked 1010 Read 6 Done Sent Done false
93512024-02-28T22:16:48.007Z 12 Acked 1011 Write 1 Done Sent Done false
93522024-02-28T22:16:48.007Z 13 Acked 1012 Read 5 Done Sent Done false
93532024-02-28T22:16:48.008Z 14 Acked 1013 Read 9 Done Sent Done false
93542024-02-28T22:16:48.008Z 15 Acked 1014 Read 6 Done Sent Done false
93552024-02-28T22:16:48.008Z 16 Acked 1015 Read 7 Done Sent Done false
93562024-02-28T22:16:48.008Z 17 Acked 1016 Write 1 Done Sent Done false
93572024-02-28T22:16:48.008Z 18 Acked 1017 Flush 0 Done Sent Done false
93582024-02-28T22:16:48.008Z 19 Acked 1018 Write 10 Done Sent Done false
93592024-02-28T22:16:48.008Z 20 Acked 1019 Read 8 Done Sent Done false
93602024-02-28T22:16:48.008Z 21 Acked 1020 Write 8 Done Sent Done false
93612024-02-28T22:16:48.008Z 22 Acked 1021 Write 7 Done Sent Done false
93622024-02-28T22:16:48.008Z 23 Acked 1022 Write 6 Done Sent Done false
93632024-02-28T22:16:48.008Z 24 Acked 1023 Read 6 Done Sent Done false
93642024-02-28T22:16:48.008Z 25 Acked 1024 Read 6 Done Sent Done false
93652024-02-28T22:16:48.008Z 26 Acked 1025 Read 10 Done Sent Done false
93662024-02-28T22:16:48.008Z 27 Acked 1026 Read 9 Done Sent Done false
93672024-02-28T22:16:48.008Z 28 Acked 1027 Read 2 Done Sent Done false
93682024-02-28T22:16:48.008Z 29 Acked 1028 Read 2 Done Sent Done false
93692024-02-28T22:16:48.008Z 30 Acked 1029 Read 2 Done Sent Sent false
93702024-02-28T22:16:48.008Z STATES DS:0 DS:1 DS:2 TOTAL
93712024-02-28T22:16:48.008Z New 0 0 0 0
93722024-02-28T22:16:48.008Z Sent 0 28 1 29
93732024-02-28T22:16:48.008Z Done 28 0 27 55
93742024-02-28T22:16:48.008Z Skipped 0 0 0 0
93752024-02-28T22:16:48.008Z Error 0 0 0 0
93762024-02-28T22:16:48.009ZLast Flush: 1017 1001 1017
93772024-02-28T22:16:48.009ZDownstairs last five completed: 1001 1000
93782024-02-28T22:16:48.009ZUpstairs last five completed: 30 29 28 27 26
93792024-02-28T22:16:48.009Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:46.895375044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1479}
93802024-02-28T22:16:48.009ZWrote out file "/tmp/test_repair_verify.data"
93812024-02-28T22:16:48.012Z
93822024-02-28T22:16:48.012ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93832024-02-28T22:16:48.027ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93842024-02-28T22:16:48.027Z 0 000-019 84 84 84 151 151 151 T F T <---
93852024-02-28T22:16:48.028Z 1 020-039 74 74 74 136 136 136 F F F
93862024-02-28T22:16:48.028Z 2 040-059 84 84 84 152 152 152 F F F
93872024-02-28T22:16:48.028Z 3 060-079 84 84 84 152 152 152 F F F
93882024-02-28T22:16:48.028Z 4 080-099 86 82 86 157 147 157 F F F <---
93892024-02-28T22:16:48.028Z 5 100-119 86 82 86 157 147 157 F F F <---
93902024-02-28T22:16:48.028Z 6 120-139 86 82 86 157 146 157 T F T <---
93912024-02-28T22:16:48.031Z 7 140-159 82 82 82 146 146 146 T F T <---
93922024-02-28T22:16:48.031Z 8 160-179 80 80 80 144 144 144 F F F
93932024-02-28T22:16:48.031Z 9 180-199 86 81 86 157 145 157 F F F <---
93942024-02-28T22:16:48.031Z 10 200-219 81 81 81 145 145 145 F F F
93952024-02-28T22:16:48.031Z 11 220-239 84 84 84 152 152 152 F F F
93962024-02-28T22:16:48.031Z 12 240-259 82 82 82 146 146 146 F F F
93972024-02-28T22:16:48.031Z 13 260-279 84 84 84 151 151 151 F F F
93982024-02-28T22:16:48.031Z 14 280-299 72 72 72 134 134 134 F F F
93992024-02-28T22:16:48.031Z 15 300-319 86 79 86 157 143 157 F F F <---
94002024-02-28T22:16:48.031Z 16 320-339 86 86 86 157 156 157 F F F <---
94012024-02-28T22:16:48.031Z 17 340-359 84 84 84 154 154 154 F F F
94022024-02-28T22:16:48.032Z 18 360-379 81 81 81 145 145 145 F F F
94032024-02-28T22:16:48.032Z 19 380-399 86 84 86 157 151 157 T F T <---
94042024-02-28T22:16:48.032Z 20 400-419 81 81 81 145 145 145 F F F
94052024-02-28T22:16:48.032Z 21 420-439 82 82 82 149 149 149 F F F
94062024-02-28T22:16:48.032Z 22 440-459 86 84 86 157 151 157 F F F <---
94072024-02-28T22:16:48.032Z 23 460-479 79 79 79 143 143 143 F F F
94082024-02-28T22:16:48.032Z 24 480-499 82 82 82 149 149 149 F F F
94092024-02-28T22:16:48.032Z 25 500-519 85 85 85 155 155 155 F F F
94102024-02-28T22:16:48.032Z 26 520-539 85 85 85 155 155 155 F F F
94112024-02-28T22:16:48.032Z 27 540-559 85 85 85 155 155 155 F F F
94122024-02-28T22:16:48.032Z 28 560-579 84 84 84 151 151 151 F F F
94132024-02-28T22:16:48.032Z 29 580-599 77 77 77 140 140 140 F F F
94142024-02-28T22:16:48.032ZMax gen: 86, Max flush: 157
94152024-02-28T22:16:48.032ZError: Difference in extent metadata found!
94162024-02-28T22:16:48.032ZA Difference in extent metadata is expected here
94172024-02-28T22:16:48.032ZOn loop 43
94182024-02-28T22:16:48.032Z
94192024-02-28T22:16:48.035ZVerifying data now
94202024-02-28T22:16:52.572ZNo change after verify
94212024-02-28T22:16:52.572ZLoop: 43 Downstairs dump after verify (and repair):
94222024-02-28T22:16:52.586ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94232024-02-28T22:16:52.586Z 0 000-019 87 87 87 158 158 158 F F F
94242024-02-28T22:16:52.586Z 1 020-039 74 74 74 136 136 136 F F F
94252024-02-28T22:16:52.586Z 2 040-059 84 84 84 152 152 152 F F F
94262024-02-28T22:16:52.586Z 3 060-079 84 84 84 152 152 152 F F F
94272024-02-28T22:16:52.586Z 4 080-099 86 86 86 157 157 157 F F F
94282024-02-28T22:16:52.587Z 5 100-119 86 86 86 157 157 157 F F F
94292024-02-28T22:16:52.587Z 6 120-139 87 87 87 158 158 158 F F F
94302024-02-28T22:16:52.587Z 7 140-159 87 87 87 158 158 158 F F F
94312024-02-28T22:16:52.587Z 8 160-179 80 80 80 144 144 144 F F F
94322024-02-28T22:16:52.590Z 9 180-199 86 86 86 157 157 157 F F F
94332024-02-28T22:16:52.590Z 10 200-219 81 81 81 145 145 145 F F F
94342024-02-28T22:16:52.590Z 11 220-239 84 84 84 152 152 152 F F F
94352024-02-28T22:16:52.590Z 12 240-259 82 82 82 146 146 146 F F F
94362024-02-28T22:16:52.590Z 13 260-279 84 84 84 151 151 151 F F F
94372024-02-28T22:16:52.590Z 14 280-299 72 72 72 134 134 134 F F F
94382024-02-28T22:16:52.590Z 15 300-319 86 86 86 157 157 157 F F F
94392024-02-28T22:16:52.591Z 16 320-339 86 86 86 157 157 157 F F F
94402024-02-28T22:16:52.591Z 17 340-359 84 84 84 154 154 154 F F F
94412024-02-28T22:16:52.591Z 18 360-379 81 81 81 145 145 145 F F F
94422024-02-28T22:16:52.591Z 19 380-399 87 87 87 158 158 158 F F F
94432024-02-28T22:16:52.591Z 20 400-419 81 81 81 145 145 145 F F F
94442024-02-28T22:16:52.591Z 21 420-439 82 82 82 149 149 149 F F F
94452024-02-28T22:16:52.591Z 22 440-459 86 86 86 157 157 157 F F F
94462024-02-28T22:16:52.591Z 23 460-479 79 79 79 143 143 143 F F F
94472024-02-28T22:16:52.591Z 24 480-499 82 82 82 149 149 149 F F F
94482024-02-28T22:16:52.591Z 25 500-519 85 85 85 155 155 155 F F F
94492024-02-28T22:16:52.591Z 26 520-539 85 85 85 155 155 155 F F F
94502024-02-28T22:16:52.591Z 27 540-559 85 85 85 155 155 155 F F F
94512024-02-28T22:16:52.591Z 28 560-579 84 84 84 151 151 151 F F F
94522024-02-28T22:16:52.591Z 29 580-599 77 77 77 140 140 140 F F F
94532024-02-28T22:16:52.591ZMax gen: 87, Max flush: 158
94542024-02-28T22:16:52.591Z
94552024-02-28T22:16:52.591ZBegin loop 44
94562024-02-28T22:16:52.591ZDownstairs to restart: 1
94572024-02-28T22:16:52.608Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.499751318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
94582024-02-28T22:16:52.608Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.499993611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
94592024-02-28T22:16:52.609Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500011201Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
94602024-02-28T22:16:52.615Z{"msg":"Crucible 63584ae9-83fc-4410-89bf-cc594a36f1f6 has session id: ae681ea9-50d4-476c-9bc2-ff0f61c88ab0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.50003403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94612024-02-28T22:16:52.615Z{"msg":"Upstairs opts: Upstairs UUID: 63584ae9-83fc-4410-89bf-cc594a36f1f6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0Crucible runtime is spawned
94622024-02-28T22:16:52.615Z,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500088838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94632024-02-28T22:16:52.616Z{"msg":"Crucible stats registered with UUID: 63584ae9-83fc-4410-89bf-cc594a36f1f6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500110988Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94642024-02-28T22:16:52.616Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500148357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
94652024-02-28T22:16:52.616Z{"msg":"63584ae9-83fc-4410-89bf-cc594a36f1f6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500231845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94662024-02-28T22:16:52.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500263334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94672024-02-28T22:16:52.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500304693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94682024-02-28T22:16:52.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500330452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94692024-02-28T22:16:52.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500366791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94702024-02-28T22:16:52.616Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.50039194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94712024-02-28T22:16:52.616Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500423769Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94722024-02-28T22:16:52.616Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500440019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94732024-02-28T22:16:52.616Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500482398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94742024-02-28T22:16:52.616Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500509807Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94752024-02-28T22:16:52.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500564245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94762024-02-28T22:16:52.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500661193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94772024-02-28T22:16:52.616Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.500715461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94782024-02-28T22:16:52.616Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501348904Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94792024-02-28T22:16:52.616Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501380183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94802024-02-28T22:16:52.616Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501413092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94812024-02-28T22:16:52.616Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501446411Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94822024-02-28T22:16:52.616Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501599877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94832024-02-28T22:16:52.616Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.501651515Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94842024-02-28T22:16:52.616Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502034405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94852024-02-28T22:16:52.616Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502110713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94862024-02-28T22:16:52.617Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502318707Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94872024-02-28T22:16:52.617Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502498272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94882024-02-28T22:16:52.617Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502534601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94892024-02-28T22:16:52.617Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502550581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94902024-02-28T22:16:52.617Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502606999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94912024-02-28T22:16:52.617Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502644318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94922024-02-28T22:16:52.617Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502660528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94932024-02-28T22:16:52.617Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502775524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94942024-02-28T22:16:52.617Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502810274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94952024-02-28T22:16:52.617Z{"msg":"[0]R flush_numbers[0..12]: [158, 136, 152, 152, 157, 157, 158, 158, 144, 157, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502874162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94962024-02-28T22:16:52.617Z{"msg":"[0]R generation[0..12]: [87, 74, 84, 84, 86, 86, 87, 87, 80, 86, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502911011Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94972024-02-28T22:16:52.617Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
94982024-02-28T22:16:52.617Z{"msg":"[1]R flush_numbers[0..12]: [158, 136, 152, 152, 157, 157, 158, 158, 144, 157, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502959939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"Wait for a query_work_queue command to finish before sending IO
94992024-02-28T22:16:52.617Z}
95002024-02-28T22:16:52.617Z{"msg":"[1]R generation[0..12]: [87, 74, 84, 84, 86, 86, 87, 87, 80, 86, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.502992018Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95012024-02-28T22:16:52.617Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503026798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95022024-02-28T22:16:52.617Z{"msg":"[2]R flush_numbers[0..12]: [158, 136, 152, 152, 157, 157, 158, 158, 144, 157, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503043417Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95032024-02-28T22:16:52.617Z{"msg":"[2]R generation[0..12]: [87, 74, 84, 84, 86, 86, 87, 87, 80, 86, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503058247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95042024-02-28T22:16:52.617Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503072526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95052024-02-28T22:16:52.617Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503085776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95062024-02-28T22:16:52.617Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503100846Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95072024-02-28T22:16:52.617Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503121135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95082024-02-28T22:16:52.617Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503135345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"":"downstairs","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95092024-02-28T22:16:52.617Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503157344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95102024-02-28T22:16:52.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503170004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"0","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95112024-02-28T22:16:52.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503194323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"1","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95122024-02-28T22:16:52.618Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503220812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"client":"2","":"downstairs","session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95132024-02-28T22:16:52.618Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503236992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95142024-02-28T22:16:52.618Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503250222Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95152024-02-28T22:16:52.618Z{"msg":"63584ae9-83fc-4410-89bf-cc594a36f1f6 is now active with session: ae681ea9-50d4-476c-9bc2-ff0f61c88ab0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503263771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95162024-02-28T22:16:52.618Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.503277181Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487,"session_id":"ae681ea9-50d4-476c-9bc2-ff0f61c88ab0"}
95172024-02-28T22:16:52.618Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:51.5032899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
95182024-02-28T22:16:52.621ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95192024-02-28T22:16:52.621ZLoading write count information from file "/tmp/test_repair_verify.data"
95202024-02-28T22:16:52.622ZRun Repair workload
95212024-02-28T22:16:52.622Z01/30 Write block 560 len 4096 data: 6
95222024-02-28T22:16:52.622Z02/30 Read block 452 len 8192
95232024-02-28T22:16:52.622Z03/30 Write block 174 len 16384 data: 11 9 8 7
95242024-02-28T22:16:52.622Z04/30 Write block 179 len 8192 data: 4 3
95252024-02-28T22:16:52.622Z05/30 Write block 94 len 4096 data: 5
95262024-02-28T22:16:52.622Z06/30 Read block 489 len 36864
95272024-02-28T22:16:52.622Z07/30 Read block 344 len 20480
95282024-02-28T22:16:52.622Z08/30 Write block 70 len 4096 data: 8
95292024-02-28T22:16:52.622Z09/30 Write block 129 len 28672 data: 9 9 10 9 10 9 9
95302024-02-28T22:16:52.622Z10/30 Read block 424 len 28672
95312024-02-28T22:16:52.622Z11/30 Read block 9 len 24576
95322024-02-28T22:16:52.622Z12/30 Write block 390 len 8192 data: 8 8
95332024-02-28T22:16:52.625Z13/30 Read block 221 len 8192
95342024-02-28T22:16:52.625Z14/30 Flush
95352024-02-28T22:16:52.625Z15/30 Flush
95362024-02-28T22:16:52.631Z16/30 Write block 536 len 8192 data: 8 7
95372024-02-28T22:16:52.631Z17/30 Flush
95382024-02-28T22:16:52.631Z18/30 Write block 89 len 16384 data: 7 8 7 8
95392024-02-28T22:16:52.631Z19/30 Read block 591 len 16384
95402024-02-28T22:16:52.631Z20/30 Write block 283 len 16384 data: 4 4 3 5
95412024-02-28T22:16:52.631Z21/30 Read block 491 len 28672
95422024-02-28T22:16:52.631Z22/30 Read block 489 len 36864
95432024-02-28T22:16:52.635Z23/30 Write block 122 len 28672 data: 8 7 8 8 7 11 12
95442024-02-28T22:16:52.635Z24/30 Write block 453 len 20480 data: 8 6 6 7 6
95452024-02-28T22:16:52.635Z25/30 Read block 127 len 16384
95462024-02-28T22:16:52.635Z26/30 Read block 258 len 4096
95472024-02-28T22:16:52.638Z27/30 Write block 560 len 16384 data: 7 8 7 7
95482024-02-28T22:16:52.638Z28/30 Read block 315 len 12288
95492024-02-28T22:16:52.638Z29/30 Read block 230 len 32768
95502024-02-28T22:16:52.638Z30/30 Read block 378 len 36864
95512024-02-28T22:16:52.641Z----------------------------------------------------------------
95522024-02-28T22:16:52.642Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95532024-02-28T22:16:52.642ZStates: Active Active Active
95542024-02-28T22:16:52.642ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95552024-02-28T22:16:52.642Z 1 Acked 1000 Write 1 Done Done Done false
95562024-02-28T22:16:52.642Z 2 Acked 1001 Read 2 Done Done Done false
95572024-02-28T22:16:52.642Z 3 Acked 1002 Write 4 Done Done Done false
95582024-02-28T22:16:52.642Z 4 Acked 1003 Write 2 Done Done Done false
95592024-02-28T22:16:52.642Z 5 Acked 1004 Write 1 Done Sent Done false
95602024-02-28T22:16:52.642Z 6 Acked 1005 Read 9 Done Sent Done false
95612024-02-28T22:16:52.642Z 7 Acked 1006 Read 5 Done Sent Done false
95622024-02-28T22:16:52.642Z 8 Acked 1007 Write 1 Done Sent Done false
95632024-02-28T22:16:52.642Z 9 Acked 1008 Write 7 Done Sent Done false
95642024-02-28T22:16:52.642Z 10 Acked 1009 Read 7 Done Sent Done false
95652024-02-28T22:16:52.642Z 11 Acked 1010 Read 6 Done Sent Done false
95662024-02-28T22:16:52.642Z 12 Acked 1011 Write 2 Done Sent Done false
95672024-02-28T22:16:52.642Z 13 Acked 1012 Read 2 Done Sent Done false
95682024-02-28T22:16:52.642Z 14 Acked 1013 Flush 0 Done Sent Done false
95692024-02-28T22:16:52.642Z 15 Acked 1014 Flush 0 Done Sent Done false
95702024-02-28T22:16:52.642Z 16 Acked 1015 Write 2 Done Sent Done false
95712024-02-28T22:16:52.642Z 17 Acked 1016 Flush 0 Done Sent Done false
95722024-02-28T22:16:52.642Z 18 Acked 1017 Write 4 Done Sent Done false
95732024-02-28T22:16:52.642Z 19 Acked 1018 Read 4 Done Sent Done false
95742024-02-28T22:16:52.643Z 20 Acked 1019 Write 4 Done Sent Done false
95752024-02-28T22:16:52.643Z 21 Acked 1020 Read 7 Done Sent Done false
95762024-02-28T22:16:52.643Z 22 Acked 1021 Read 9 Done Sent Done false
95772024-02-28T22:16:52.643Z 23 Acked 1022 Write 7 Done Sent Done false
95782024-02-28T22:16:52.643Z 24 Acked 1023 Write 5 Done Sent Done false
95792024-02-28T22:16:52.643Z 25 Acked 1024 Read 4 Done Sent Done false
95802024-02-28T22:16:52.643Z 26 Acked 1025 Read 1 Done Sent Done false
95812024-02-28T22:16:52.643Z 27 Acked 1026 Write 4 Done Sent Done false
95822024-02-28T22:16:52.643Z 28 Acked 1027 Read 3 Done Sent Done false
95832024-02-28T22:16:52.643Z 29 Acked 1028 Read 8 Done Sent Done false
95842024-02-28T22:16:52.643Z 30 Acked 1029 Read 9 Done Sent Sent false
95852024-02-28T22:16:52.643Z STATES DS:0 DS:1 DS:2 TOTAL
95862024-02-28T22:16:52.643Z New 0 0 0 0
95872024-02-28T22:16:52.643Z Sent 0 26 1 27
95882024-02-28T22:16:52.643Z Done 30 4 29 63
95892024-02-28T22:16:52.643Z Skipped 0 0 0 0
95902024-02-28T22:16:52.643Z Error 0 0 0 0
95912024-02-28T22:16:52.643ZLast Flush: 1016 0 1016
95922024-02-28T22:16:52.643ZDownstairs last five completed:
95932024-02-28T22:16:52.643ZUpstairs last five completed: 30 29 28 27 26
95942024-02-28T22:16:52.643Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:51.530128891Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1487}
95952024-02-28T22:16:52.643ZWrote out file "/tmp/test_repair_verify.data"
95962024-02-28T22:16:52.647Z
95972024-02-28T22:16:52.647ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
95982024-02-28T22:16:52.841ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95992024-02-28T22:16:52.841Z 0 000-019 87 87 87 158 158 158 F F F
96002024-02-28T22:16:52.841Z 1 020-039 74 74 74 136 136 136 F F F
96012024-02-28T22:16:52.841Z 2 040-059 84 84 84 152 152 152 F F F
96022024-02-28T22:16:52.841Z 3 060-079 88 84 88 159 152 159 F F F <---
96032024-02-28T22:16:52.841Z 4 080-099 88 86 88 159 157 159 T F T <---
96042024-02-28T22:16:52.841Z 5 100-119 86 86 86 157 157 157 F F F
96052024-02-28T22:16:52.841Z 6 120-139 88 87 88 159 158 159 T F T <---
96062024-02-28T22:16:52.841Z 7 140-159 87 87 87 158 158 158 F F F
96072024-02-28T22:16:52.841Z 8 160-179 88 80 88 159 144 159 F T F <---
96082024-02-28T22:16:52.841Z 9 180-199 88 86 88 159 157 159 F T F <---
96092024-02-28T22:16:52.841Z 10 200-219 81 81 81 145 145 145 F F F
96102024-02-28T22:16:52.845Z 11 220-239 84 84 84 152 152 152 F F F
96112024-02-28T22:16:52.845Z 12 240-259 82 82 82 146 146 146 F F F
96122024-02-28T22:16:52.846Z 13 260-279 84 84 84 151 151 151 F F F
96132024-02-28T22:16:52.846Z 14 280-299 72 72 72 134 134 134 T F T <---
96142024-02-28T22:16:52.846Z 15 300-319 86 86 86 157 157 157 F F F
96152024-02-28T22:16:52.846Z 16 320-339 86 86 86 157 157 157 F F F
96162024-02-28T22:16:52.846Z 17 340-359 84 84 84 154 154 154 F F F
96172024-02-28T22:16:52.846Z 18 360-379 81 81 81 145 145 145 F F F
96182024-02-28T22:16:52.846Z 19 380-399 88 87 88 159 158 159 F F F <---
96192024-02-28T22:16:52.846Z 20 400-419 81 81 81 145 145 145 F F F
96202024-02-28T22:16:52.846Z 21 420-439 82 82 82 149 149 149 F F F
96212024-02-28T22:16:52.846Z 22 440-459 86 86 86 157 157 157 T F T <---
96222024-02-28T22:16:52.846Z 23 460-479 79 79 79 143 143 143 F F F
96232024-02-28T22:16:52.846Z 24 480-499 82 82 82 149 149 149 F F F
96242024-02-28T22:16:52.846Z 25 500-519 85 85 85 155 155 155 F F F
96252024-02-28T22:16:52.847Z 26 520-539 88 85 88 161 155 161 F F F <---
96262024-02-28T22:16:52.847Z 27 540-559 85 85 85 155 155 155 F F F
96272024-02-28T22:16:52.847Z 28 560-579 88 84 88 159 151 159 T T T <---
96282024-02-28T22:16:52.847Z 29 580-599 77 77 77 140 140 140 F F F
96292024-02-28T22:16:52.847ZMax gen: 88, Max flush: 161
96302024-02-28T22:16:52.847ZError: Difference in extent metadata found!
96312024-02-28T22:16:52.847ZA Difference in extent metadata is expected here
96322024-02-28T22:16:52.847ZOn loop 44
96332024-02-28T22:16:52.847Z
96342024-02-28T22:16:52.847ZVerifying data now
96352024-02-28T22:16:54.207ZNo change after verify
96362024-02-28T22:16:54.207ZLoop: 44 Downstairs dump after verify (and repair):
96372024-02-28T22:16:54.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96382024-02-28T22:16:54.282Z 0 000-019 87 87 87 158 158 158 F F F
96392024-02-28T22:16:54.282Z 1 020-039 74 74 74 136 136 136 F F F
96402024-02-28T22:16:54.282Z 2 040-059 84 84 84 152 152 152 F F F
96412024-02-28T22:16:54.282Z 3 060-079 88 88 88 159 159 159 F F F
96422024-02-28T22:16:54.282Z 4 080-099 89 89 89 162 162 162 F F F
96432024-02-28T22:16:54.283Z 5 100-119 86 86 86 157 157 157 F F F
96442024-02-28T22:16:54.283Z 6 120-139 89 89 89 162 162 162 F F F
96452024-02-28T22:16:54.283Z 7 140-159 87 87 87 158 158 158 F F F
96462024-02-28T22:16:54.283Z 8 160-179 88 88 88 159 159 159 F F F
96472024-02-28T22:16:54.283Z 9 180-199 88 88 88 159 159 159 F F F
96482024-02-28T22:16:54.283Z 10 200-219 81 81 81 145 145 145 F F F
96492024-02-28T22:16:54.283Z 11 220-239 84 84 84 152 152 152 F F F
96502024-02-28T22:16:54.283Z 12 240-259 82 82 82 146 146 146 F F F
96512024-02-28T22:16:54.283Z 13 260-279 84 84 84 151 151 151 F F F
96522024-02-28T22:16:54.283Z 14 280-299 89 89 89 162 162 162 F F F
96532024-02-28T22:16:54.283Z 15 300-319 86 86 86 157 157 157 F F F
96542024-02-28T22:16:54.284Z 16 320-339 86 86 86 157 157 157 F F F
96552024-02-28T22:16:54.284Z 17 340-359 84 84 84 154 154 154 F F F
96562024-02-28T22:16:54.284Z 18 360-379 81 81 81 145 145 145 F F F
96572024-02-28T22:16:54.284Z 19 380-399 88 88 88 159 159 159 F F F
96582024-02-28T22:16:54.284Z 20 400-419 81 81 81 145 145 145 F F F
96592024-02-28T22:16:54.284Z 21 420-439 82 82 82 149 149 149 F F F
96602024-02-28T22:16:54.284Z 22 440-459 89 89 89 162 162 162 F F F
96612024-02-28T22:16:54.284Z 23 460-479 79 79 79 143 143 143 F F F
96622024-02-28T22:16:54.284Z 24 480-499 82 82 82 149 149 149 F F F
96632024-02-28T22:16:54.284Z 25 500-519 85 85 85 155 155 155 F F F
96642024-02-28T22:16:54.284Z 26 520-539 88 88 88 161 161 161 F F F
96652024-02-28T22:16:54.285Z 27 540-559 85 85 85 155 155 155 F F F
96662024-02-28T22:16:54.285Z 28 560-579 89 89 89 162 162 162 F F F
96672024-02-28T22:16:54.285Z 29 580-599 77 77 77 140 140 140 F F F
96682024-02-28T22:16:54.285ZMax gen: 89, Max flush: 162
96692024-02-28T22:16:54.285Z
96702024-02-28T22:16:54.285ZBegin loop 45
96712024-02-28T22:16:54.285ZDownstairs to restart: 1
96722024-02-28T22:16:54.285Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.130819144Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
96732024-02-28T22:16:54.285Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131119535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
96742024-02-28T22:16:54.285Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131166574Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
96752024-02-28T22:16:54.285Z{"msg":"Crucible d0d46973-6bd9-44fc-8afe-56c344ce0df4 has session id: e3e0877e-9578-45cd-8d29-68069dc4f001","v":0,"name":"crucible","level":30Crucible runtime is spawned
96762024-02-28T22:16:54.286Z,"time":"2024-02-28T22:16:53.131196363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96772024-02-28T22:16:54.286Z{"msg":"Upstairs opts: Upstairs UUID: d0d46973-6bd9-44fc-8afe-56c344ce0df4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131246632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96782024-02-28T22:16:54.286Z{"msg":"Crucible stats registered with UUID: d0d46973-6bd9-44fc-8afe-56c344ce0df4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131286531Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96792024-02-28T22:16:54.286Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.13131811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
96802024-02-28T22:16:54.286Z{"msg":"d0d46973-6bd9-44fc-8afe-56c344ce0df4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131361979Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96812024-02-28T22:16:54.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131386428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96822024-02-28T22:16:54.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131427437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96832024-02-28T22:16:54.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131462176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96842024-02-28T22:16:54.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131495035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96852024-02-28T22:16:54.286Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131538114Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96862024-02-28T22:16:54.286Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131571183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96872024-02-28T22:16:54.286Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131620602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96882024-02-28T22:16:54.287Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131660241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96892024-02-28T22:16:54.287Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131704439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96902024-02-28T22:16:54.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131725229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96912024-02-28T22:16:54.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131917363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96922024-02-28T22:16:54.287Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.131981342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96932024-02-28T22:16:54.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.132346511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96942024-02-28T22:16:54.287Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.13241721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96952024-02-28T22:16:54.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.132511767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96962024-02-28T22:16:54.287Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.132541396Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96972024-02-28T22:16:54.287Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.132627824Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96982024-02-28T22:16:54.288Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.132701112Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
96992024-02-28T22:16:54.288Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133007113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97002024-02-28T22:16:54.288Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133156649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97012024-02-28T22:16:54.288Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133394143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97022024-02-28T22:16:54.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133548089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97032024-02-28T22:16:54.288Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133589547Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97042024-02-28T22:16:54.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133638346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97052024-02-28T22:16:54.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133684225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97062024-02-28T22:16:54.288Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133719764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97072024-02-28T22:16:54.288Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133751763Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97082024-02-28T22:16:54.288Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133932388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97092024-02-28T22:16:54.288Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133972387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97102024-02-28T22:16:54.288Z{"msg":"[0]R flush_numbers[0..12]: [158, 136, 152, 159, 162, 157, 162, 158, 159, 159, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.133989366Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97112024-02-28T22:16:54.289Z{"msg":"[0]R generation[0..12]: [87, 74, 84, 88, 89, 86, 89, 87, 88, 88, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134024765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97122024-02-28T22:16:54.289Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134039905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97132024-02-28T22:16:54.289Z{"msg":"[1]R flush_numbers[0..12]: [158, 136, 152, 159, 162, 157, 162, 158, 159, 159, 145, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134061934Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97142024-02-28T22:16:54.289Z{"msg":"[1]R generation[0..12]: [87, 74, 84, 88, 89, 86, 89, 87, 88, 88, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134076694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97152024-02-28T22:16:54.289Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134091134Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97162024-02-28T22:16:54.289Z{"msg":"[2]R flush_numbers[0..12]: [158, 136, 152, 159, 162, 157, 162, 158, 159, 159, 145, 152]","v":0,"Wait for a query_work_queue command to finish before sending IO
97172024-02-28T22:16:54.289Zname":"crucible","level":30,"time":"2024-02-28T22:16:53.134109523Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97182024-02-28T22:16:54.289Z{"msg":"[2]R generation[0..12]: [87, 74, 84, 88, 89, 86, 89, 87, 88, 88, 81, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134124913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97192024-02-28T22:16:54.289Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134145982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97202024-02-28T22:16:54.289Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134159742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97212024-02-28T22:16:54.289Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134180821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97222024-02-28T22:16:54.289Z{"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134194351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97232024-02-28T22:16:54.290Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.13421416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"":"downstairs","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97242024-02-28T22:16:54.290Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.13422818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97252024-02-28T22:16:54.290Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134240419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"0","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97262024-02-28T22:16:54.290Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134261379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"1","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97272024-02-28T22:16:54.290Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134275608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"client":"2","":"downstairs","session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97282024-02-28T22:16:54.290Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134296228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97292024-02-28T22:16:54.290Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134308877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97302024-02-28T22:16:54.290Z{"msg":"d0d46973-6bd9-44fc-8afe-56c344ce0df4 is now active with session: e3e0877e-9578-45cd-8d29-68069dc4f001","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134329087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97312024-02-28T22:16:54.290Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134341407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495,"session_id":"e3e0877e-9578-45cd-8d29-68069dc4f001"}
97322024-02-28T22:16:54.290Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:53.134361226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
97332024-02-28T22:16:54.290ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97342024-02-28T22:16:54.291ZLoading write count information from file "/tmp/test_repair_verify.data"
97352024-02-28T22:16:54.291ZRun Repair workload
97362024-02-28T22:16:54.291Z01/30 Write block 247 len 40960 data: 8 10 7 5 7 7 9 7 6 9
97372024-02-28T22:16:54.302Z02/30 Read block 141 len 16384
97382024-02-28T22:16:54.302Z03/30 Read block 471 len 40960
97392024-02-28T22:16:54.302Z04/30 Write block 420 len 24576 data: 11 11 10 9 8 8
97402024-02-28T22:16:54.302Z05/30 Read block 304 len 16384
97412024-02-28T22:16:54.302Z06/30 Read block 243 len 40960
97422024-02-28T22:16:54.302Z07/30 Write block 296 len 32768 data: 8 7 5 4 3 3 4 7
97432024-02-28T22:16:54.302Z08/30 Write block 243 len 32768 data: 4 6 6 4 9 11 8 6
97442024-02-28T22:16:54.302Z09/30 Write block 373 len 32768 data: 9 8 7 7 8 9 9 9
97452024-02-28T22:16:54.302Z10/30 Read block 383 len 36864
97462024-02-28T22:16:54.303Z11/30 Write block 216 len 16384 data: 4 4 4 4
97472024-02-28T22:16:54.303Z12/30 Read block 504 len 36864
97482024-02-28T22:16:54.303Z13/30 Read block 594 len 12288
97492024-02-28T22:16:54.303Z14/30 Write block 136 len 32768 data: 9 9 9 9 8 7 7 9
97502024-02-28T22:16:54.303Z15/30 Read block 304 len 4096
97512024-02-28T22:16:54.303Z16/30 Read block 149 len 24576
97522024-02-28T22:16:54.303Z17/30 Write block 6 len 12288 data: 10 13 14
97532024-02-28T22:16:54.303Z18/30 Read block 17 len 20480
97542024-02-28T22:16:54.303Z19/30 Read block 474 len 12288
97552024-02-28T22:16:54.303Z20/30 Write block 328 len 12288 data: 13 12 13
97562024-02-28T22:16:54.303Z21/30 Write block 540 len 28672 data: 7 8 9 8 8 9 10
97572024-02-28T22:16:54.303Z22/30 Read block 3 len 32768
97582024-02-28T22:16:54.303Z23/30 Read block 377 len 32768
97592024-02-28T22:16:54.304Z24/30 Flush
97602024-02-28T22:16:54.304Z25/30 Write block 180 len 40960 data: 4 5 6 6 9 9 7 8 7 7
97612024-02-28T22:16:54.304Z26/30 Read block 78 len 16384
97622024-02-28T22:16:54.304Z27/30 Read block 82 len 24576
97632024-02-28T22:16:54.304Z28/30 Read block 474 len 8192
97642024-02-28T22:16:54.304Z29/30 Write block 237 len 36864 data: 6 8 6 6 6 5 5 7 7
97652024-02-28T22:16:54.304Z30/30 Write block 301 len 36864 data: 4 5 8 8 7 7 9 8 6
97662024-02-28T22:16:54.304Z----------------------------------------------------------------
97672024-02-28T22:16:54.304Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97682024-02-28T22:16:54.304ZStates: Active Active Active
97692024-02-28T22:16:54.304ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97702024-02-28T22:16:54.304Z 1 Acked 1000 Write 10 Done Done Done false
97712024-02-28T22:16:54.304Z 2 Acked 1001 Read 4 Done Done Done false
97722024-02-28T22:16:54.305Z 3 Acked 1002 Read 10 Done Done Done false
97732024-02-28T22:16:54.305Z 4 Acked 1003 Write 6 Done Sent Done false
97742024-02-28T22:16:54.305Z 5 Acked 1004 Read 4 Done Sent Done false
97752024-02-28T22:16:54.305Z 6 Acked 1005 Read 10 Done Sent Done false
97762024-02-28T22:16:54.305Z 7 Acked 1006 Write 8 Done Sent Done false
97772024-02-28T22:16:54.305Z 8 Acked 1007 Write 8 Done Sent Done false
97782024-02-28T22:16:54.305Z 9 Acked 1008 Write 8 Done Sent Done false
97792024-02-28T22:16:54.305Z 10 Acked 1009 Read 9 Done Sent Done false
97802024-02-28T22:16:54.305Z 11 Acked 1010 Write 4 Done Sent Done false
97812024-02-28T22:16:54.305Z 12 Acked 1011 Read 9 Done Sent Done false
97822024-02-28T22:16:54.305Z 13 Acked 1012 Read 3 Done Sent Done false
97832024-02-28T22:16:54.305Z 14 Acked 1013 Write 8 Done Sent Done false
97842024-02-28T22:16:54.305Z 15 Acked 1014 Read 1 Done Sent Done false
97852024-02-28T22:16:54.306Z 16 Acked 1015 Read 6 Done Sent Done false
97862024-02-28T22:16:54.306Z 17 Acked 1016 Write 3 Done Sent Done false
97872024-02-28T22:16:54.306Z 18 Acked 1017 Read 5 Done Sent Done false
97882024-02-28T22:16:54.306Z 19 Acked 1018 Read 3 Done Sent Done false
97892024-02-28T22:16:54.306Z 20 Acked 1019 Write 3 Done Sent Done false
97902024-02-28T22:16:54.306Z 21 Acked 1020 Write 7 Done Sent Done false
97912024-02-28T22:16:54.306Z 22 Acked 1021 Read 8 Done Sent Done false
97922024-02-28T22:16:54.306Z 23 Acked 1022 Read 8 Done Sent Done false
97932024-02-28T22:16:54.306Z 24 Acked 1023 Flush 0 Done Sent Done false
97942024-02-28T22:16:54.306Z 25 Acked 1024 Write 10 Done Sent Done false
97952024-02-28T22:16:54.306Z 26 Acked 1025 Read 4 Done Sent Done false
97962024-02-28T22:16:54.306Z 27 Acked 1026 Read 6 Done Sent Done false
97972024-02-28T22:16:54.306Z 28 Acked 1027 Read 2 Done Sent Done false
97982024-02-28T22:16:54.306Z 29 Acked 1028 Write 9 Sent Sent Sent false
97992024-02-28T22:16:54.307Z 30 Acked 1029 Write 9 Sent Sent Sent false
98002024-02-28T22:16:54.307Z STATES DS:0 DS:1 DS:2 TOTAL
98012024-02-28T22:16:54.307Z New 0 0 0 0
98022024-02-28T22:16:54.307Z Sent 2 27 2 31
98032024-02-28T22:16:54.307Z Done 28 3 28 59
98042024-02-28T22:16:54.307Z Skipped 0 0 0 0
98052024-02-28T22:16:54.307Z Error 0 0 0 0
98062024-02-28T22:16:54.307ZLast Flush: 1023 0 1023
98072024-02-28T22:16:54.307ZDownstairs last five completed:
98082024-02-28T22:16:54.307ZUpstairs last five completed: 30 29 28 27 26
98092024-02-28T22:16:54.307Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:53.169118318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1495}
98102024-02-28T22:16:54.307ZWrote out file "/tmp/test_repair_verify.data"
98112024-02-28T22:16:54.307Z
98122024-02-28T22:16:54.308ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98132024-02-28T22:16:54.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98142024-02-28T22:16:54.328Z 0 000-019 90 87 90 163 158 163 F F F <---
98152024-02-28T22:16:54.328Z 1 020-039 74 74 74 136 136 136 F F F
98162024-02-28T22:16:54.328Z 2 040-059 84 84 84 152 152 152 F F F
98172024-02-28T22:16:54.328Z 3 060-079 88 88 88 159 159 159 F F F
98182024-02-28T22:16:54.328Z 4 080-099 89 89 89 162 162 162 F F F
98192024-02-28T22:16:54.329Z 5 100-119 86 86 86 157 157 157 F F F
98202024-02-28T22:16:54.329Z 6 120-139 90 89 90 163 162 163 F F F <---
98212024-02-28T22:16:54.329Z 7 140-159 90 87 90 163 158 163 F F F <---
98222024-02-28T22:16:54.329Z 8 160-179 88 88 88 159 159 159 F F F
98232024-02-28T22:16:54.329Z 9 180-199 88 88 88 159 159 159 T F T <---
98242024-02-28T22:16:54.329Z 10 200-219 90 81 90 163 145 163 F F F <---
98252024-02-28T22:16:54.329Z 11 220-239 84 84 84 152 152 152 T F T <---
98262024-02-28T22:16:54.329Z 12 240-259 90 82 90 163 146 163 T T T <---
98272024-02-28T22:16:54.329Z 13 260-279 84 84 84 151 151 151 F F F
98282024-02-28T22:16:54.330Z 14 280-299 90 89 90 163 162 163 F F F <---
98292024-02-28T22:16:54.330Z 15 300-319 90 86 90 163 157 163 T F T <---
98302024-02-28T22:16:54.330Z 16 320-339 90 86 90 163 157 163 F F F <---
98312024-02-28T22:16:54.330Z 17 340-359 84 84 84 154 154 154 F F F
98322024-02-28T22:16:54.330Z 18 360-379 90 81 90 163 145 163 F F F <---
98332024-02-28T22:16:54.330Z 19 380-399 90 88 90 163 159 163 F F F <---
98342024-02-28T22:16:54.330Z 20 400-419 81 81 81 145 145 145 F F F
98352024-02-28T22:16:54.330Z 21 420-439 90 82 90 163 149 163 F F F <---
98362024-02-28T22:16:54.330Z 22 440-459 89 89 89 162 162 162 F F F
98372024-02-28T22:16:54.330Z 23 460-479 79 79 79 143 143 143 F F F
98382024-02-28T22:16:54.330Z 24 480-499 82 82 82 149 149 149 F F F
98392024-02-28T22:16:54.330Z 25 500-519 85 85 85 155 155 155 F F F
98402024-02-28T22:16:54.330Z 26 520-539 88 88 88 161 161 161 F F F
98412024-02-28T22:16:54.331Z 27 540-559 90 85 90 163 155 163 F F F <---
98422024-02-28T22:16:54.331Z 28 560-579 89 89 89 162 162 162 F F F
98432024-02-28T22:16:54.331Z 29 580-599 77 77 77 140 140 140 F F F
98442024-02-28T22:16:54.331ZMax gen: 90, Max flush: 163
98452024-02-28T22:16:54.331ZError: Difference in extent metadata found!
98462024-02-28T22:16:54.331ZA Difference in extent metadata is expected here
98472024-02-28T22:16:54.331ZOn loop 45
98482024-02-28T22:16:54.331Z
98492024-02-28T22:16:54.331ZVerifying data now
98502024-02-28T22:17:08.492ZNo change after verify
98512024-02-28T22:17:08.492ZLoop: 45 Downstairs dump after verify (and repair):
98522024-02-28T22:17:08.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98532024-02-28T22:17:08.644Z 0 000-019 90 90 90 163 163 163 F F F
98542024-02-28T22:17:08.644Z 1 020-039 74 74 74 136 136 136 F F F
98552024-02-28T22:17:08.644Z 2 040-059 84 84 84 152 152 152 F F F
98562024-02-28T22:17:08.644Z 3 060-079 88 88 88 159 159 159 F F F
98572024-02-28T22:17:08.644Z 4 080-099 89 89 89 162 162 162 F F F
98582024-02-28T22:17:08.645Z 5 100-119 86 86 86 157 157 157 F F F
98592024-02-28T22:17:08.645Z 6 120-139 90 90 90 163 163 163 F F F
98602024-02-28T22:17:08.645Z 7 140-159 90 90 90 163 163 163 F F F
98612024-02-28T22:17:08.645Z 8 160-179 88 88 88 159 159 159 F F F
98622024-02-28T22:17:08.645Z 9 180-199 91 91 91 164 164 164 F F F
98632024-02-28T22:17:08.645Z 10 200-219 90 90 90 163 163 163 F F F
98642024-02-28T22:17:08.645Z 11 220-239 91 91 91 164 164 164 F F F
98652024-02-28T22:17:08.645Z 12 240-259 91 91 91 164 164 164 F F F
98662024-02-28T22:17:08.645Z 13 260-279 84 84 84 151 151 151 F F F
98672024-02-28T22:17:08.645Z 14 280-299 90 90 90 163 163 163 F F F
98682024-02-28T22:17:08.645Z 15 300-319 91 91 91 164 164 164 F F F
98692024-02-28T22:17:08.645Z 16 320-339 90 90 90 163 163 163 F F F
98702024-02-28T22:17:08.645Z 17 340-359 84 84 84 154 154 154 F F F
98712024-02-28T22:17:08.645Z 18 360-379 90 90 90 163 163 163 F F F
98722024-02-28T22:17:08.645Z 19 380-399 90 90 90 163 163 163 F F F
98732024-02-28T22:17:08.645Z 20 400-419 81 81 81 145 145 145 F F F
98742024-02-28T22:17:08.646Z 21 420-439 90 90 90 163 163 163 F F F
98752024-02-28T22:17:08.646Z 22 440-459 89 89 89 162 162 162 F F F
98762024-02-28T22:17:08.646Z 23 460-479 79 79 79 143 143 143 F F F
98772024-02-28T22:17:08.646Z 24 480-499 82 82 82 149 149 149 F F F
98782024-02-28T22:17:08.646Z 25 500-519 85 85 85 155 155 155 F F F
98792024-02-28T22:17:08.646Z 26 520-539 88 88 88 161 161 161 F F F
98802024-02-28T22:17:08.646Z 27 540-559 90 90 90 163 163 163 F F F
98812024-02-28T22:17:08.646Z 28 560-579 89 89 89 162 162 162 F F F
98822024-02-28T22:17:08.646Z 29 580-599 77 77 77 140 140 140 F F F
98832024-02-28T22:17:08.646ZMax gen: 91, Max flush: 164
98842024-02-28T22:17:08.646Z
98852024-02-28T22:17:08.646ZBegin loop 46
98862024-02-28T22:17:08.646ZDownstairs to restart: 2
98872024-02-28T22:17:08.646Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.939704113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
98882024-02-28T22:17:08.647Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
98892024-02-28T22:17:08.647Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940029654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
98902024-02-28T22:17:08.647Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940063403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
98912024-02-28T22:17:08.647Z{"msg":"Crucible 73df0f20-3ee6-45e3-bc1c-79e39fc0fc66 has session id: f23a7300-5904-4cc1-a2de-7f69c8df2b6c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940080193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98922024-02-28T22:17:08.647Z{"msg":"Upstairs opts: Upstairs UUID: 73df0f20-3ee6-45e3-bc1c-79e39fc0fc66, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940110992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98932024-02-28T22:17:08.647Z{"msg":"Crucible stats registered with UUID: 73df0f20-3ee6-45e3-bc1c-79e39fc0fc66","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940126712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98942024-02-28T22:17:08.647Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940139581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
98952024-02-28T22:17:08.647Z{"msg":"73df0f20-3ee6-45e3-bc1c-79e39fc0fc66 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940261208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98962024-02-28T22:17:08.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940291097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98972024-02-28T22:17:08.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940316516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98982024-02-28T22:17:08.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940360645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
98992024-02-28T22:17:08.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940406334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99002024-02-28T22:17:08.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940431723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99012024-02-28T22:17:08.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940464883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99022024-02-28T22:17:08.647Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940490842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99032024-02-28T22:17:08.647Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.94054188Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99042024-02-28T22:17:08.647Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940581299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99052024-02-28T22:17:08.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940618668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99062024-02-28T22:17:08.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940720315Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99072024-02-28T22:17:08.647Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.940780074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99082024-02-28T22:17:08.647Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941390677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99092024-02-28T22:17:08.648Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941427566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99102024-02-28T22:17:08.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941451445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99112024-02-28T22:17:08.648Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941473825Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99122024-02-28T22:17:08.648Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941487914Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99132024-02-28T22:17:08.648Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.941501434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99142024-02-28T22:17:08.648Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.94201599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99152024-02-28T22:17:08.648Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942054209Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99162024-02-28T22:17:08.648Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942175835Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99172024-02-28T22:17:08.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942581564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99182024-02-28T22:17:08.648Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942637903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99192024-02-28T22:17:08.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942666162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99202024-02-28T22:17:08.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942696871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99212024-02-28T22:17:08.648Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.94272211Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99222024-02-28T22:17:08.648Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942750649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99232024-02-28T22:17:08.648Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942773659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99242024-02-28T22:17:08.648Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942801268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99252024-02-28T22:17:08.648Z{"msg":"[0]R flush_numbers[0..12]: [163, 136, 152, 159, 162, 157, 163, 163, 159, 164, 163, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942826667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99262024-02-28T22:17:08.648Z{"msg":"[0]R generation[0..12]: [90, 74, 84, 88, 89, 86, 90, 90, 88, 91, 90, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942854677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99272024-02-28T22:17:08.648Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942879896Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99282024-02-28T22:17:08.648ZWait for a query_work_queue command to finish before sending IO
99292024-02-28T22:17:08.648Z{"msg":"[1]R flush_numbers[0..12]: [163, 136, 152, 159, 162, 157, 163, 163, 159, 164, 163, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942915125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99302024-02-28T22:17:08.648Z{"msg":"[1]R generation[0..12]: [90, 74, 84, 88, 89, 86, 90, 90, 88, 91, 90, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942947374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99312024-02-28T22:17:08.648Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942973733Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99322024-02-28T22:17:08.649Z{"msg":"[2]R flush_numbers[0..12]: [163, 136, 152, 159, 162, 157, 163, 163, 159, 164, 163, 164]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.942999733Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99332024-02-28T22:17:08.649Z{"msg":"[2]R generation[0..12]: [90, 74, 84, 88, 89, 86, 90, 90, 88, 91, 90, 91]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943026742Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99342024-02-28T22:17:08.649Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943042192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99352024-02-28T22:17:08.649Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943062501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99362024-02-28T22:17:08.649Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943076191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99372024-02-28T22:17:08.649Z{"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.9430894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99382024-02-28T22:17:08.649Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.94310286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"":"downstairs","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99392024-02-28T22:17:08.649Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943117379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99402024-02-28T22:17:08.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943129609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"0","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99412024-02-28T22:17:08.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943151058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"1","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99422024-02-28T22:17:08.649Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943165318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"client":"2","":"downstairs","session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99432024-02-28T22:17:08.649Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943178678Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99442024-02-28T22:17:08.649Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943198527Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99452024-02-28T22:17:08.649Z{"msg":"73df0f20-3ee6-45e3-bc1c-79e39fc0fc66 is now active with session: f23a7300-5904-4cc1-a2de-7f69c8df2b6c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943212037Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99462024-02-28T22:17:08.649Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943225016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503,"session_id":"f23a7300-5904-4cc1-a2de-7f69c8df2b6c"}
99472024-02-28T22:17:08.649Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:16:57.943244056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
99482024-02-28T22:17:08.649ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99492024-02-28T22:17:08.649ZLoading write count information from file "/tmp/test_repair_verify.data"
99502024-02-28T22:17:08.649ZRun Repair workload
99512024-02-28T22:17:08.649Z01/30 Write block 212 len 40960 data: 8 7 6 5 5 5 5 5 4 5
99522024-02-28T22:17:08.769Z02/30 Write block 81 len 4096 data: 7
99532024-02-28T22:17:08.769Z03/30 Flush
99542024-02-28T22:17:08.769Z04/30 Write block 450 len 40960 data: 9 9 8 9 7 7 8 7 8 7
99552024-02-28T22:17:08.769Z05/30 Write block 586 len 16384 data: 6 5 5 5
99562024-02-28T22:17:08.769Z06/30 Read block 192 len 4096
99572024-02-28T22:17:08.769Z07/30 Write block 41 len 36864 data: 10 5 5 3 5 4 6 6 7
99582024-02-28T22:17:08.770Z08/30 Flush
99592024-02-28T22:17:08.770Z09/30 Write block 444 len 12288 data: 8 8 9
99602024-02-28T22:17:08.770Z10/30 Write block 122 len 20480 data: 9 8 9 9 8
99612024-02-28T22:17:08.770Z11/30 Write block 531 len 40960 data: 7 6 7 9 6 9 8 7 7 8
99622024-02-28T22:17:08.770Z12/30 Read block 132 len 20480
99632024-02-28T22:17:08.770Z13/30 Flush
99642024-02-28T22:17:08.770Z14/30 Write block 372 len 20480 data: 12 10 9 8 8
99652024-02-28T22:17:08.770Z15/30 Read block 57 len 28672
99662024-02-28T22:17:08.770Z16/30 Write block 240 len 16384 data: 7 7 6 6
99672024-02-28T22:17:08.770Z17/30 Read block 165 len 12288
99682024-02-28T22:17:08.770Z18/30 Flush
99692024-02-28T22:17:08.770Z19/30 Write block 113 len 16384 data: 7 9 9 8
99702024-02-28T22:17:08.770Z20/30 Write block 183 len 4096 data: 7
99712024-02-28T22:17:08.770Z21/30 Read block 576 len 36864
99722024-02-28T22:17:08.770Z22/30 Read block 397 len 12288
99732024-02-28T22:17:08.770Z23/30 Write block 234 len 12288 data: 11 7 6
99742024-02-28T22:17:08.770Z24/30 Read block 453 len 32768
99752024-02-28T22:17:08.770Z25/30 Flush
99762024-02-28T22:17:08.770Z26/30 Write block 92 len 36864 data: 9 6 6 5 5 5 5 5 6
99772024-02-28T22:17:08.770Z27/30 Read block 436 len 4096
99782024-02-28T22:17:08.770Z28/30 Write block 81 len 12288 data: 8 10 8
99792024-02-28T22:17:08.771Z29/30 Write block 309 len 40960 data: 7 4 9 9 7 7 5 6 4 5
99802024-02-28T22:17:08.771Z30/30 Read block 51 len 32768
99812024-02-28T22:17:08.771Z----------------------------------------------------------------
99822024-02-28T22:17:08.771Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:27
99832024-02-28T22:17:08.771ZStates: Active Active Active
99842024-02-28T22:17:08.771ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99852024-02-28T22:17:08.771Z 4 Acked 1003 Write 10 Done Done Done false
99862024-02-28T22:17:08.771Z 5 Acked 1004 Write 4 Done Done Done false
99872024-02-28T22:17:08.771Z 6 Acked 1005 Read 1 Done Done Done false
99882024-02-28T22:17:08.771Z 7 Acked 1006 Write 9 Done Done Sent false
99892024-02-28T22:17:08.771Z 8 Acked 1007 Flush 0 Done Done Sent false
99902024-02-28T22:17:08.771Z 9 Acked 1008 Write 3 Done Done Sent false
99912024-02-28T22:17:08.771Z 10 Acked 1009 Write 5 Done Done Sent false
99922024-02-28T22:17:08.771Z 11 Acked 1010 Write 10 Done Done Sent false
99932024-02-28T22:17:08.771Z 12 Acked 1011 Read 5 Done Done Sent false
99942024-02-28T22:17:08.771Z 13 Acked 1012 Flush 0 Done Done Sent false
99952024-02-28T22:17:08.771Z 14 Acked 1013 Write 5 Done Done Sent false
99962024-02-28T22:17:08.771Z 15 Acked 1014 Read 7 Done Done Sent false
99972024-02-28T22:17:08.771Z 16 Acked 1015 Write 4 Done Done Sent false
99982024-02-28T22:17:08.771Z 17 Acked 1016 Read 3 Done Done Sent false
99992024-02-28T22:17:08.771Z 18 Acked 1017 Flush 0 Done Done Sent false
100002024-02-28T22:17:08.771Z 19 Acked 1018 Write 4 Done Done Sent false
100012024-02-28T22:17:08.772Z 20 Acked 1019 Write 1 Done Done Sent false
100022024-02-28T22:17:08.772Z 21 Acked 1020 Read 9 Done Done Sent false
100032024-02-28T22:17:08.772Z 22 Acked 1021 Read 3 Done Done Sent false
100042024-02-28T22:17:08.772Z 23 Acked 1022 Write 3 Done Done Sent false
100052024-02-28T22:17:08.772Z 24 Acked 1023 Read 8 Done Done Sent false
100062024-02-28T22:17:08.772Z 25 Acked 1024 Flush 0 Done Done Sent false
100072024-02-28T22:17:08.772Z 26 Acked 1025 Write 9 Done Done Sent false
100082024-02-28T22:17:08.772Z 27 Acked 1026 Read 1 Done Done Sent false
100092024-02-28T22:17:08.772Z 28 Acked 1027 Write 3 Done Done Sent false
100102024-02-28T22:17:08.772Z 29 Acked 1028 Write 10 Done Done Sent false
100112024-02-28T22:17:08.772Z 30 Acked 1029 Read 8 Sent Done Sent false
100122024-02-28T22:17:08.772Z STATES DS:0 DS:1 DS:2 TOTAL
100132024-02-28T22:17:08.772Z New 0 0 0 0
100142024-02-28T22:17:08.772Z Sent 1 0 24 25
100152024-02-28T22:17:08.772Z Done 26 27 3 56
100162024-02-28T22:17:08.772Z Skipped 0 0 0 0
100172024-02-28T22:17:08.772Z Error 0 0 0 0
100182024-02-28T22:17:08.772ZLast Flush: 1024 1024 1002
100192024-02-28T22:17:08.772ZDownstairs last five completed: 1002 1001 1000
100202024-02-28T22:17:08.772ZUpstairs last five completed: 30 29 28 27 26
100212024-02-28T22:17:08.772Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:16:57.98085553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1503}
100222024-02-28T22:17:08.772ZWrote out file "/tmp/test_repair_verify.data"
100232024-02-28T22:17:08.772Z
100242024-02-28T22:17:08.773ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100252024-02-28T22:17:08.773ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100262024-02-28T22:17:08.773Z 0 000-019 90 90 90 163 163 163 F F F
100272024-02-28T22:17:08.773Z 1 020-039 74 74 74 136 136 136 F F F
100282024-02-28T22:17:08.773Z 2 040-059 92 92 84 166 166 152 F F F <---
100292024-02-28T22:17:08.773Z 3 060-079 88 88 88 159 159 159 F F F
100302024-02-28T22:17:08.773Z 4 080-099 92 92 92 165 165 165 T T F <---
100312024-02-28T22:17:08.773Z 5 100-119 92 92 86 169 169 157 T T F <---
100322024-02-28T22:17:08.773Z 6 120-139 92 92 90 167 167 163 F F F <---
100332024-02-28T22:17:08.773Z 7 140-159 90 90 90 163 163 163 F F F
100342024-02-28T22:17:08.773Z 8 160-179 88 88 88 159 159 159 F F F
100352024-02-28T22:17:08.773Z 9 180-199 92 92 91 169 169 164 F F F <---
100362024-02-28T22:17:08.773Z 10 200-219 92 92 92 165 165 165 F F F
100372024-02-28T22:17:08.773Z 11 220-239 92 92 92 169 169 165 F F F <---
100382024-02-28T22:17:08.773Z 12 240-259 92 92 91 168 168 164 F F F <---
100392024-02-28T22:17:08.773Z 13 260-279 84 84 84 151 151 151 F F F
100402024-02-28T22:17:08.773Z 14 280-299 90 90 90 163 163 163 F F F
100412024-02-28T22:17:08.773Z 15 300-319 91 91 91 164 164 164 T T F <---
100422024-02-28T22:17:08.773Z 16 320-339 90 90 90 163 163 163 F F F
100432024-02-28T22:17:08.773Z 17 340-359 84 84 84 154 154 154 F F F
100442024-02-28T22:17:08.773Z 18 360-379 92 92 90 168 168 163 F F F <---
100452024-02-28T22:17:08.773Z 19 380-399 90 90 90 163 163 163 F F F
100462024-02-28T22:17:08.773Z 20 400-419 81 81 81 145 145 145 F F F
100472024-02-28T22:17:08.773Z 21 420-439 90 90 90 163 163 163 F F F
100482024-02-28T22:17:08.774Z 22 440-459 92 92 89 167 167 162 F F T <---
100492024-02-28T22:17:08.774Z 23 460-479 79 79 79 143 143 143 F F F
100502024-02-28T22:17:08.774Z 24 480-499 82 82 82 149 149 149 F F F
100512024-02-28T22:17:08.774Z 25 500-519 85 85 85 155 155 155 F F F
100522024-02-28T22:17:08.805Z 26 520-539 92 92 88 167 167 161 F F F <---
100532024-02-28T22:17:08.805Z 27 540-559 92 92 90 167 167 163 F F F <---
100542024-02-28T22:17:08.805Z 28 560-579 89 89 89 162 162 162 F F F
100552024-02-28T22:17:08.805Z 29 580-599 92 92 77 166 166 140 F F T <---
100562024-02-28T22:17:08.805ZMax gen: 92, Max flush: 169
100572024-02-28T22:17:08.805ZError: Difference in extent metadata found!
100582024-02-28T22:17:08.805ZA Difference in extent metadata is expected here
100592024-02-28T22:17:08.805ZOn loop 46
100602024-02-28T22:17:08.805Z
100612024-02-28T22:17:08.805ZVerifying data now
100622024-02-28T22:17:08.805ZNo change after verify
100632024-02-28T22:17:08.805ZLoop: 46 Downstairs dump after verify (and repair):
100642024-02-28T22:17:08.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100652024-02-28T22:17:08.805Z 0 000-019 90 90 90 163 163 163 F F F
100662024-02-28T22:17:08.805Z 1 020-039 74 74 74 136 136 136 F F F
100672024-02-28T22:17:08.805Z 2 040-059 92 92 92 166 166 166 F F F
100682024-02-28T22:17:08.805Z 3 060-079 88 88 88 159 159 159 F F F
100692024-02-28T22:17:08.806Z 4 080-099 93 93 93 170 170 170 F F F
100702024-02-28T22:17:08.806Z 5 100-119 93 93 93 170 170 170 F F F
100712024-02-28T22:17:08.806Z 6 120-139 92 92 92 167 167 167 F F F
100722024-02-28T22:17:08.806Z 7 140-159 90 90 90 163 163 163 F F F
100732024-02-28T22:17:08.806Z 8 160-179 88 88 88 159 159 159 F F F
100742024-02-28T22:17:08.806Z 9 180-199 92 92 92 169 169 169 F F F
100752024-02-28T22:17:08.806Z 10 200-219 92 92 92 165 165 165 F F F
100762024-02-28T22:17:08.806Z 11 220-239 92 92 92 169 169 169 F F F
100772024-02-28T22:17:08.806Z 12 240-259 92 92 92 168 168 168 F F F
100782024-02-28T22:17:08.806Z 13 260-279 84 84 84 151 151 151 F F F
100792024-02-28T22:17:08.806Z 14 280-299 90 90 90 163 163 163 F F F
100802024-02-28T22:17:08.806Z 15 300-319 93 93 93 170 170 170 F F F
100812024-02-28T22:17:08.806Z 16 320-339 90 90 90 163 163 163 F F F
100822024-02-28T22:17:08.806Z 17 340-359 84 84 84 154 154 154 F F F
100832024-02-28T22:17:08.806Z 18 360-379 92 92 92 168 168 168 F F F
100842024-02-28T22:17:08.806Z 19 380-399 90 90 90 163 163 163 F F F
100852024-02-28T22:17:08.806Z 20 400-419 81 81 81 145 145 145 F F F
100862024-02-28T22:17:08.806Z 21 420-439 90 90 90 163 163 163 F F F
100872024-02-28T22:17:08.806Z 22 440-459 92 92 92 167 167 167 F F F
100882024-02-28T22:17:08.806Z 23 460-479 79 79 79 143 143 143 F F F
100892024-02-28T22:17:08.806Z 24 480-499 82 82 82 149 149 149 F F F
100902024-02-28T22:17:08.806Z 25 500-519 85 85 85 155 155 155 F F F
100912024-02-28T22:17:08.806Z 26 520-539 92 92 92 167 167 167 F F F
100922024-02-28T22:17:08.807Z 27 540-559 92 92 92 167 167 167 F F F
100932024-02-28T22:17:08.807Z 28 560-579 89 89 89 162 162 162 F F F
100942024-02-28T22:17:08.807Z 29 580-599 92 92 92 166 166 166 F F F
100952024-02-28T22:17:08.807ZMax gen: 93, Max flush: 170
100962024-02-28T22:17:08.807Z
100972024-02-28T22:17:08.807ZBegin loop 47
100982024-02-28T22:17:08.807ZDownstairs to restart: 1
100992024-02-28T22:17:08.807Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.699726659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
101002024-02-28T22:17:08.807Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
101012024-02-28T22:17:08.807Z,"time":"2024-02-28T22:17:02.700041131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
101022024-02-28T22:17:08.807Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70007762Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
101032024-02-28T22:17:08.807Z{"msg":"Crucible 155cca4e-7f42-486a-901c-a4a6fb8d6720 has session id: 0f8a92a9-2846-40ae-9d66-488970f22d95","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700113479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101042024-02-28T22:17:08.807Z{"msg":"Upstairs opts: Upstairs UUID: 155cca4e-7f42-486a-901c-a4a6fb8d6720, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700149778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101052024-02-28T22:17:08.807Z{"msg":"Crucible stats registered with UUID: 155cca4e-7f42-486a-901c-a4a6fb8d6720","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700177897Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101062024-02-28T22:17:08.807Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700202426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
101072024-02-28T22:17:08.807Z{"msg":"155cca4e-7f42-486a-901c-a4a6fb8d6720 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700263864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101082024-02-28T22:17:08.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700295604Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101092024-02-28T22:17:08.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700313583Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101102024-02-28T22:17:08.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700341962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101112024-02-28T22:17:08.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700385621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101122024-02-28T22:17:08.807Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700512248Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101132024-02-28T22:17:08.807Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700570576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101142024-02-28T22:17:08.808Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700606045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101152024-02-28T22:17:08.808Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700668743Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101162024-02-28T22:17:08.808Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700721212Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101172024-02-28T22:17:08.808Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700751811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101182024-02-28T22:17:08.808Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70078652Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101192024-02-28T22:17:08.808Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.700813489Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101202024-02-28T22:17:08.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701214818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101212024-02-28T22:17:08.808Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701249717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101222024-02-28T22:17:08.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701376784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101232024-02-28T22:17:08.808Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701417633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101242024-02-28T22:17:08.808Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701434362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101252024-02-28T22:17:08.808Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701456692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101262024-02-28T22:17:08.808Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.701812842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101272024-02-28T22:17:08.808Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702118673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101282024-02-28T22:17:08.808Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702154003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101292024-02-28T22:17:08.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702381506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101302024-02-28T22:17:08.808Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702436655Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101312024-02-28T22:17:08.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702463764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101322024-02-28T22:17:08.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702720677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101332024-02-28T22:17:08.808Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702778845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101342024-02-28T22:17:08.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702812284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101352024-02-28T22:17:08.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702843864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101362024-02-28T22:17:08.808Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702875623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101372024-02-28T22:17:08.809Z{"msg":"[0]R flush_numbers[0..12]: [163, 136, 166, 159, 170, 170, 167, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.702908012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511Wait for a query_work_queue command to finish before sending IO
101382024-02-28T22:17:08.809Z,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101392024-02-28T22:17:08.809Z{"msg":"[0]R generation[0..12]: [90, 74, 92, 88, 93, 93, 92, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70294983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101402024-02-28T22:17:08.809Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70298149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101412024-02-28T22:17:08.809Z{"msg":"[1]R flush_numbers[0..12]: [163, 136, 166, 159, 170, 170, 167, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703024139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101422024-02-28T22:17:08.809Z{"msg":"[1]R generation[0..12]: [90, 74, 92, 88, 93, 93, 92, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703058398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101432024-02-28T22:17:08.809Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703076727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101442024-02-28T22:17:08.809Z{"msg":"[2]R flush_numbers[0..12]: [163, 136, 166, 159, 170, 170, 167, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703099006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101452024-02-28T22:17:08.809Z{"msg":"[2]R generation[0..12]: [90, 74, 92, 88, 93, 93, 92, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703120296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101462024-02-28T22:17:08.809Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703142175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101472024-02-28T22:17:08.809Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703156795Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101482024-02-28T22:17:08.809Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703171595Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101492024-02-28T22:17:08.809Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703185444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101502024-02-28T22:17:08.809Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703211763Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"":"downstairs","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101512024-02-28T22:17:08.809Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703225743Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101522024-02-28T22:17:08.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703238353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"0","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101532024-02-28T22:17:08.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703264092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"1","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101542024-02-28T22:17:08.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703277721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"client":"2","":"downstairs","session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101552024-02-28T22:17:08.827Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703297631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101562024-02-28T22:17:08.827Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70332031Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101572024-02-28T22:17:08.828Z{"msg":"155cca4e-7f42-486a-901c-a4a6fb8d6720 is now active with session: 0f8a92a9-2846-40ae-9d66-488970f22d95","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.70334741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101582024-02-28T22:17:08.828Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703371439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511,"session_id":"0f8a92a9-2846-40ae-9d66-488970f22d95"}
101592024-02-28T22:17:08.828Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:02.703401758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
101602024-02-28T22:17:08.828ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101612024-02-28T22:17:08.828ZLoading write count information from file "/tmp/test_repair_verify.data"
101622024-02-28T22:17:08.828ZRun Repair workload
101632024-02-28T22:17:08.828ZError: Difference in extent metadata found!
101642024-02-28T22:17:08.828Z01/30 Write block 415 len 12288 data: 11 10 7
101652024-02-28T22:17:08.828Z02/30 Read block 165 len 28672
101662024-02-28T22:17:08.828Z03/30 Write block 532 len 16384 data: 7 8 10 7
101672024-02-28T22:17:08.828Z04/30 Read block 253 len 16384
101682024-02-28T22:17:08.828Z05/30 Read block 348 len 32768
101692024-02-28T22:17:08.828Z06/30 Write block 309 len 24576 data: 8 5 10 10 8 8
101702024-02-28T22:17:08.828Z07/30 Read block 101 len 16384
101712024-02-28T22:17:08.828Z08/30 Write block 414 len 8192 data: 11 12
101722024-02-28T22:17:08.828Z09/30 Write block 62 len 36864 data: 12 10 10 10 9 11 10 7 9
101732024-02-28T22:17:08.828Z10/30 Write block 306 len 16384 data: 8 10 9 9
101742024-02-28T22:17:08.828Z11/30 Write block 369 len 16384 data: 8 9 10 13
101752024-02-28T22:17:08.828Z12/30 Write block 561 len 24576 data: 9 8 8 8 9 10
101762024-02-28T22:17:08.828Z13/30 Read block 592 len 16384
101772024-02-28T22:17:08.829Z14/30 Write block 532 len 40960 data: 8 9 11 8 10 9 8 8 9 9
101782024-02-28T22:17:08.829Z15/30 Write block 512 len 20480 data: 9 8 7 8 8
101792024-02-28T22:17:08.829Z16/30 Write block 321 len 16384 data: 5 6 6 8
101802024-02-28T22:17:08.829Z17/30 Write block 77 len 12288 data: 6 6 5
101812024-02-28T22:17:08.829Z18/30 Flush
101822024-02-28T22:17:08.829Z19/30 Flush
101832024-02-28T22:17:08.829Z20/30 Write block 134 len 20480 data: 10 10 10 10 10
101842024-02-28T22:17:08.829Z21/30 Write block 447 len 20480 data: 9 8 9 10 10
101852024-02-28T22:17:08.829Z22/30 Write block 576 len 16384 data: 7 8 9 10
101862024-02-28T22:17:08.829Z23/30 Read block 372 len 24576
101872024-02-28T22:17:08.829Z24/30 Read block 306 len 36864
101882024-02-28T22:17:08.829Z25/30 Flush
101892024-02-28T22:17:08.829Z26/30 Write block 125 len 40960 data: 10 9 12 13 10 10 11 10 11 11
101902024-02-28T22:17:08.829Z27/30 Read block 282 len 32768
101912024-02-28T22:17:08.829Z28/30 Read block 152 len 20480
101922024-02-28T22:17:08.829Z29/30 Write block 382 len 24576 data: 13 13 13 11 11 8
101932024-02-28T22:17:08.829Z30/30 Write block 474 len 20480 data: 10 9 8 6 4
101942024-02-28T22:17:08.830Z----------------------------------------------------------------
101952024-02-28T22:17:08.830Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
101962024-02-28T22:17:08.830ZStates: Active Active Active
101972024-02-28T22:17:08.830ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101982024-02-28T22:17:08.830Z 1 Acked 1000 Write 3 Done Done Done false
101992024-02-28T22:17:08.830Z 2 Acked 1001 Read 7 Done Sent Done false
102002024-02-28T22:17:08.830Z 3 Acked 1002 Write 4 Done Sent Done false
102012024-02-28T22:17:08.830Z 4 Acked 1003 Read 4 Done Sent Done false
102022024-02-28T22:17:08.830Z 5 Acked 1004 Read 8 Done Sent Done false
102032024-02-28T22:17:08.830Z 6 Acked 1005 Write 6 Done Sent Done false
102042024-02-28T22:17:08.830Z 7 Acked 1006 Read 4 Done Sent Done false
102052024-02-28T22:17:08.830Z 8 Acked 1007 Write 2 Done Sent Done false
102062024-02-28T22:17:08.830Z 9 Acked 1008 Write 9 Done Sent Done false
102072024-02-28T22:17:08.830Z 10 Acked 1009 Write 4 Done Sent Done false
102082024-02-28T22:17:08.830Z 11 Acked 1010 Write 4 Done Sent Done false
102092024-02-28T22:17:08.830Z 12 Acked 1011 Write 6 Done Sent Done false
102102024-02-28T22:17:08.830Z 13 Acked 1012 Read 4 Done Sent Done false
102112024-02-28T22:17:08.830Z 14 Acked 1013 Write 10 Done Sent Done false
102122024-02-28T22:17:08.830Z 15 Acked 1014 Write 5 Done Sent Done false
102132024-02-28T22:17:08.830Z 16 Acked 1015 Write 4 Done Sent Done false
102142024-02-28T22:17:08.830Z 17 Acked 1016 Write 3 Done Sent Done false
102152024-02-28T22:17:08.830Z 18 Acked 1017 Flush 0 Done Sent Done false
102162024-02-28T22:17:08.830Z 19 Acked 1018 Flush 0 Done Sent Done false
102172024-02-28T22:17:08.831Z 20 Acked 1019 Write 5 Done Sent Done false
102182024-02-28T22:17:08.831Z 21 Acked 1020 Write 5 Done Sent Done false
102192024-02-28T22:17:08.831Z 22 Acked 1021 Write 4 Done Sent Done false
102202024-02-28T22:17:08.831Z 23 Acked 1022 Read 6 Done Sent Done false
102212024-02-28T22:17:08.831Z 24 Acked 1023 Read 9 Done Sent Done false
102222024-02-28T22:17:08.831Z 25 Acked 1024 Flush 0 Done Sent Done false
102232024-02-28T22:17:08.831Z 26 Acked 1025 Write 10 Done Sent Done false
102242024-02-28T22:17:08.831Z 27 Acked 1026 Read 8 Done Sent Done false
102252024-02-28T22:17:08.831Z 28 Acked 1027 Read 5 Done Sent Done false
102262024-02-28T22:17:08.831Z 29 Acked 1028 Write 6 Done Sent Done false
102272024-02-28T22:17:08.831Z 30 Acked 1029 Write 5 Sent Sent Sent false
102282024-02-28T22:17:08.831Z STATES DS:0 DS:1 DS:2 TOTAL
102292024-02-28T22:17:08.831Z New 0 0 0 0
102302024-02-28T22:17:08.831Z Sent 1 29 1 31
102312024-02-28T22:17:08.831Z Done 29 1 29 59
102322024-02-28T22:17:08.831Z Skipped 0 0 0 0
102332024-02-28T22:17:08.831Z Error 0 0 0 0
102342024-02-28T22:17:08.831ZLast Flush: 1024 0 1024
102352024-02-28T22:17:08.831ZDownstairs last five completed:
102362024-02-28T22:17:08.831ZUpstairs last five completed: 30 29 28 27 26
102372024-02-28T22:17:08.831Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:02.734807433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1511}
102382024-02-28T22:17:08.832ZWrote out file "/tmp/test_repair_verify.data"
102392024-02-28T22:17:08.832Z
102402024-02-28T22:17:08.832ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102412024-02-28T22:17:08.832ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102422024-02-28T22:17:08.832Z 0 000-019 90 90 90 163 163 163 F F F
102432024-02-28T22:17:08.832Z 1 020-039 74 74 74 136 136 136 F F F
102442024-02-28T22:17:08.832Z 2 040-059 92 92 92 166 166 166 F F F
102452024-02-28T22:17:08.832Z 3 060-079 94 88 94 171 159 171 F F F <---
102462024-02-28T22:17:08.832Z 4 080-099 93 93 93 170 170 170 F F F
102472024-02-28T22:17:08.832Z 5 100-119 93 93 93 170 170 170 F F F
102482024-02-28T22:17:08.832Z 6 120-139 94 92 94 173 167 173 T F T <---
102492024-02-28T22:17:08.832Z 7 140-159 90 90 90 163 163 163 F F F
102502024-02-28T22:17:08.832Z 8 160-179 88 88 88 159 159 159 F F F
102512024-02-28T22:17:08.832Z 9 180-199 92 92 92 169 169 169 F F F
102522024-02-28T22:17:08.841Z 10 200-219 92 92 92 165 165 165 F F F
102532024-02-28T22:17:08.841Z 11 220-239 92 92 92 169 169 169 F F F
102542024-02-28T22:17:08.841Z 12 240-259 92 92 92 168 168 168 F F F
102552024-02-28T22:17:08.841Z 13 260-279 84 84 84 151 151 151 F F F
102562024-02-28T22:17:08.841Z 14 280-299 90 90 90 163 163 163 F F F
102572024-02-28T22:17:08.841Z 15 300-319 94 93 94 171 170 171 F F F <---
102582024-02-28T22:17:08.842Z 16 320-339 94 90 94 171 163 171 F F F <---
102592024-02-28T22:17:08.842Z 17 340-359 84 84 84 154 154 154 F F F
102602024-02-28T22:17:08.842Z 18 360-379 94 92 94 171 168 171 F F F <---
102612024-02-28T22:17:08.842Z 19 380-399 90 90 90 163 163 163 T F T <---
102622024-02-28T22:17:08.842Z 20 400-419 94 81 94 171 145 171 F T F <---
102632024-02-28T22:17:08.842Z 21 420-439 90 90 90 163 163 163 F F F
102642024-02-28T22:17:08.842Z 22 440-459 94 92 94 173 167 173 F F F <---
102652024-02-28T22:17:08.842Z 23 460-479 79 79 79 143 143 143 T F T <---
102662024-02-28T22:17:08.842Z 24 480-499 82 82 82 149 149 149 F F F
102672024-02-28T22:17:08.842Z 25 500-519 94 85 94 171 155 171 F F F <---
102682024-02-28T22:17:08.842Z 26 520-539 94 92 94 171 167 171 F F F <---
102692024-02-28T22:17:08.842Z 27 540-559 94 92 94 171 167 171 F F F <---
102702024-02-28T22:17:08.842Z 28 560-579 94 89 94 173 162 173 F F F <---
102712024-02-28T22:17:08.842Z 29 580-599 92 92 92 166 166 166 F F F
102722024-02-28T22:17:08.842ZMax gen: 94, Max flush: 173
102732024-02-28T22:17:08.842ZA Difference in extent metadata is expected here
102742024-02-28T22:17:08.842ZOn loop 47
102752024-02-28T22:17:08.842Z
102762024-02-28T22:17:08.842ZVerifying data now
102772024-02-28T22:17:08.843ZNo change after verify
102782024-02-28T22:17:08.843ZLoop: 47 Downstairs dump after verify (and repair):
102792024-02-28T22:17:08.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102802024-02-28T22:17:08.843Z 0 000-019 90 90 90 163 163 163 F F F
102812024-02-28T22:17:08.843Z 1 020-039 74 74 74 136 136 136 F F F
102822024-02-28T22:17:08.843Z 2 040-059 92 92 92 166 166 166 F F F
102832024-02-28T22:17:08.843Z 3 060-079 94 94 94 171 171 171 F F F
102842024-02-28T22:17:08.843Z 4 080-099 93 93 93 170 170 170 F F F
102852024-02-28T22:17:08.843Z 5 100-119 93 93 93 170 170 170 F F F
102862024-02-28T22:17:08.843Z 6 120-139 95 95 95 174 174 174 F F F
102872024-02-28T22:17:08.843Z 7 140-159 90 90 90 163 163 163 F F F
102882024-02-28T22:17:08.843Z 8 160-179 88 88 88 159 159 159 F F F
102892024-02-28T22:17:08.843Z 9 180-199 92 92 92 169 169 169 F F F
102902024-02-28T22:17:08.843Z 10 200-219 92 92 92 165 165 165 F F F
102912024-02-28T22:17:08.843Z 11 220-239 92 92 92 169 169 169 F F F
102922024-02-28T22:17:08.843Z 12 240-259 92 92 92 168 168 168 F F F
102932024-02-28T22:17:08.843Z 13 260-279 84 84 84 151 151 151 F F F
102942024-02-28T22:17:08.843Z 14 280-299 90 90 90 163 163 163 F F F
102952024-02-28T22:17:08.843Z 15 300-319 94 94 94 171 171 171 F F F
102962024-02-28T22:17:08.843Z 16 320-339 94 94 94 171 171 171 F F F
102972024-02-28T22:17:08.843Z 17 340-359 84 84 84 154 154 154 F F F
102982024-02-28T22:17:08.844Z 18 360-379 94 94 94 171 171 171 F F F
102992024-02-28T22:17:08.844Z 19 380-399 95 95 95 174 174 174 F F F
103002024-02-28T22:17:08.844Z 20 400-419 94 94 94 171 171 171 F F F
103012024-02-28T22:17:08.844Z 21 420-439 90 90 90 163 163 163 F F F
103022024-02-28T22:17:08.844Z 22 440-459 94 94 94 173 173 173 F F F
103032024-02-28T22:17:08.844Z 23 460-479 95 95 95 174 174 174 F F F
103042024-02-28T22:17:08.844Z 24 480-499 82 82 82 149 149 149 F F F
103052024-02-28T22:17:08.844Z 25 500-519 94 94 94 171 171 171 F F F
103062024-02-28T22:17:08.844Z 26 520-539 94 94 94 171 171 171 F F F
103072024-02-28T22:17:08.844Z 27 540-559 94 94 94 171 171 171 F F F
103082024-02-28T22:17:08.844Z 28 560-579 94 94 94 173 173 173 F F F
103092024-02-28T22:17:08.844Z 29 580-599 92 92 92 166 166 166 F F F
103102024-02-28T22:17:08.844ZMax gen: 95, Max flush: 174
103112024-02-28T22:17:08.844Z
103122024-02-28T22:17:08.844ZBegin loop 48
103132024-02-28T22:17:08.844ZDownstairs to restart: 0
103142024-02-28T22:17:08.844Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.449745141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
103152024-02-28T22:17:08.844Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450008044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
103162024-02-28T22:17:08.844Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450040543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
103172024-02-28T22:17:08.845Z{"msg":"Crucible runtime is spawned
103182024-02-28T22:17:08.845ZCrucible debe782a-1bdf-4cc1-84d6-1fbbd7436075 has session id: 711ed1f2-5efd-4358-837d-c14db5118d6f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450076212Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103192024-02-28T22:17:08.845Z{"msg":"Upstairs opts: Upstairs UUID: debe782a-1bdf-4cc1-84d6-1fbbd7436075, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450117161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103202024-02-28T22:17:08.845Z{"msg":"Crucible stats registered with UUID: debe782a-1bdf-4cc1-84d6-1fbbd7436075","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450137581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103212024-02-28T22:17:08.845Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.45015371Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
103222024-02-28T22:17:08.845Z{"msg":"debe782a-1bdf-4cc1-84d6-1fbbd7436075 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.45017704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103232024-02-28T22:17:08.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450193759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103242024-02-28T22:17:08.845Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450211899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103252024-02-28T22:17:08.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450240818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103262024-02-28T22:17:08.845Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450273047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103272024-02-28T22:17:08.845Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450302996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103282024-02-28T22:17:08.845Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450334475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103292024-02-28T22:17:08.845Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450368004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103302024-02-28T22:17:08.845Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450404173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103312024-02-28T22:17:08.845Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450438362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103322024-02-28T22:17:08.845Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450553889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103332024-02-28T22:17:08.845Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450600478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103342024-02-28T22:17:08.845Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.450633587Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103352024-02-28T22:17:08.845Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.451393596Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103362024-02-28T22:17:08.845Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.451421065Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103372024-02-28T22:17:08.845Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.451435975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103382024-02-28T22:17:08.845Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.451450085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103392024-02-28T22:17:08.845Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.45162234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103402024-02-28T22:17:08.846Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.451662079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103412024-02-28T22:17:08.846Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.45195772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103422024-02-28T22:17:08.846Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452108046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103432024-02-28T22:17:08.846Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452296341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103442024-02-28T22:17:08.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452483386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103452024-02-28T22:17:08.846Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452519615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103462024-02-28T22:17:08.846Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452545764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103472024-02-28T22:17:08.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452561024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103482024-02-28T22:17:08.846Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452576794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103492024-02-28T22:17:08.846Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452590663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103502024-02-28T22:17:08.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452760688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103512024-02-28T22:17:08.846Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452790828Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103522024-02-28T22:17:08.855Z{"msg":"[0]R flush_numbers[0..12]: [163, 136, 166, 171, 170, 170, 174, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452810997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103532024-02-28T22:17:08.855Z{"msg":"[0]R generation[0..12]: [90, 74, 92, 94, 93, 93, 95, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452825677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103542024-02-28T22:17:08.855Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452840276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103552024-02-28T22:17:08.855Z{"msg":"[1]R flush_numbers[0..12]: [163, 136, 166, 171, 170, 170, 174, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452854446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103562024-02-28T22:17:08.855Z{"msg":"[1]R generation[0..12]: [90, 74, 92, 94, 93, 93, 95, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452868266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103572024-02-28T22:17:08.855Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452888125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103582024-02-28T22:17:08.855Z{"msg":"[2]R flush_numbers[0..12]: [163, 136, 166, 171, 170, 170, 174, 163, 159, 169, 165, 169]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452902634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103592024-02-28T22:17:08.855Z{"msg":"[2]R generation[0..12]: [90, 74, 92, 94, 93, 93, 95, 90, 88, 92, 92, 92]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
103602024-02-28T22:17:08.855Z,"time":"2024-02-28T22:17:07.452916694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103612024-02-28T22:17:08.855Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452933994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103622024-02-28T22:17:08.855Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452947813Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103632024-02-28T22:17:08.855Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452961463Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103642024-02-28T22:17:08.855Z{"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452974903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103652024-02-28T22:17:08.855Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.452988092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"":"downstairs","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103662024-02-28T22:17:08.855Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.453001612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103672024-02-28T22:17:08.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.453014311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"0","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103682024-02-28T22:17:08.855Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103692024-02-28T22:17:08.855ZTransition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.51878036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"1","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103702024-02-28T22:17:08.855Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518833558Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"client":"2","":"downstairs","session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103712024-02-28T22:17:08.855Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518848778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103722024-02-28T22:17:08.856Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518861688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103732024-02-28T22:17:08.856Z{"msg":"debe782a-1bdf-4cc1-84d6-1fbbd7436075 is now active with session: 711ed1f2-5efd-4358-837d-c14db5118d6f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518875307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103742024-02-28T22:17:08.856Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518888457Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519,"session_id":"711ed1f2-5efd-4358-837d-c14db5118d6f"}
103752024-02-28T22:17:08.856Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:07.518901906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
103762024-02-28T22:17:08.856ZLoading write count information from file "/tmp/test_repair_verify.data"
103772024-02-28T22:17:08.856ZRun Repair workload
103782024-02-28T22:17:08.856Z01/30 Write block 98 len 12288 data: 6 6 7
103792024-02-28T22:17:08.856Z02/30 Read block 349 len 28672
103802024-02-28T22:17:08.856Z03/30 Read block 125 len 24576
103812024-02-28T22:17:08.856Z04/30 Read block 408 len 4096
103822024-02-28T22:17:08.856Z05/30 Write block 329 len 12288 data: 13 14 11
103832024-02-28T22:17:08.856Z06/30 Read block 196 len 32768
103842024-02-28T22:17:08.856Z07/30 Read block 341 len 12288
103852024-02-28T22:17:08.856Z08/30 Write block 142 len 16384 data: 8 10 10 12
103862024-02-28T22:17:08.856Z09/30 Read block 36 len 20480
103872024-02-28T22:17:08.856Z10/30 Read block 501 len 40960
103882024-02-28T22:17:08.857Z11/30 Read block 321 len 36864
103892024-02-28T22:17:08.857Z12/30 Read block 257 len 20480
103902024-02-28T22:17:08.857Z13/30 Write block 42 len 8192 data: 6 6
103912024-02-28T22:17:08.857Z14/30 Read block 204 len 40960
103922024-02-28T22:17:08.857Z15/30 Write block 556 len 24576 data: 6 6 6 6 8 10
103932024-02-28T22:17:08.857Z16/30 Write block 226 len 4096 data: 8
103942024-02-28T22:17:08.857Z17/30 Flush
103952024-02-28T22:17:08.857Z18/30 Write block 418 len 28672 data: 10 10 12 12 11 10 9
103962024-02-28T22:17:08.857Z19/30 Read block 226 len 36864
103972024-02-28T22:17:08.857Z20/30 Read block 88 len 16384
103982024-02-28T22:17:08.857Z21/30 Read block 327 len 32768
103992024-02-28T22:17:08.857Z22/30 Read block 383 len 36864
104002024-02-28T22:17:08.857Z23/30 Read block 478 len 32768
104012024-02-28T22:17:08.857Z24/30 Read block 526 len 36864
104022024-02-28T22:17:08.857Z25/30 Read block 9 len 8192
104032024-02-28T22:17:08.857Z26/30 Read block 180 len 32768
104042024-02-28T22:17:08.857Z27/30 Write block 196 len 40960 data: 8 8 8 6 4 6 6 5 5 7
104052024-02-28T22:17:08.857Z28/30 Write block 260 len 16384 data: 9 9 10 8
104062024-02-28T22:17:08.857Z29/30 Write block 229 len 40960 data: 10 9 13 13 12 12 8 7 7 9
104072024-02-28T22:17:08.857Z30/30 Read block 93 len 36864
104082024-02-28T22:17:08.857Z----------------------------------------------------------------
104092024-02-28T22:17:08.858Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
104102024-02-28T22:17:08.858ZStates: Active Active Active
104112024-02-28T22:17:08.858ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104122024-02-28T22:17:08.858Z 1 Acked 1000 Write 3 Done Done Done false
104132024-02-28T22:17:08.858Z 2 Acked 1001 Read 7 Sent Done Done false
104142024-02-28T22:17:08.858ZError: Difference in extent metadata found!
104152024-02-28T22:17:08.858Z 3 Acked 1002 Read 6 Sent Done Done false
104162024-02-28T22:17:08.858Z 4 Acked 1003 Read 1 Sent Done Done false
104172024-02-28T22:17:08.858Z 5 Acked 1004 Write 3 Sent Done Done false
104182024-02-28T22:17:08.858Z 6 Acked 1005 Read 8 Sent Done Done false
104192024-02-28T22:17:08.858Z 7 Acked 1006 Read 3 Sent Done Done false
104202024-02-28T22:17:08.858Z 8 Acked 1007 Write 4 Sent Done Done false
104212024-02-28T22:17:08.858Z 9 Acked 1008 Read 5 Sent Done Done false
104222024-02-28T22:17:08.858Z 10 Acked 1009 Read 10 Sent Done Done false
104232024-02-28T22:17:08.858Z 11 Acked 1010 Read 9 Sent Done Done false
104242024-02-28T22:17:08.858Z 12 Acked 1011 Read 5 Sent Done Done false
104252024-02-28T22:17:08.858Z 13 Acked 1012 Write 2 Sent Done Done false
104262024-02-28T22:17:08.858Z 14 Acked 1013 Read 10 Sent Done Done false
104272024-02-28T22:17:08.858Z 15 Acked 1014 Write 6 Sent Done Done false
104282024-02-28T22:17:08.858Z 16 Acked 1015 Write 1 Sent Done Done false
104292024-02-28T22:17:08.858Z 17 Acked 1016 Flush 0 Sent Done Done false
104302024-02-28T22:17:08.859Z 18 Acked 1017 Write 7 Sent Done Done false
104312024-02-28T22:17:08.859Z 19 Acked 1018 Read 9 Sent Done Done false
104322024-02-28T22:17:08.859Z 20 Acked 1019 Read 4 Sent Done Done false
104332024-02-28T22:17:08.859Z 21 Acked 1020 Read 8 Sent Done Done false
104342024-02-28T22:17:08.859Z 22 Acked 1021 Read 9 Sent Done Done false
104352024-02-28T22:17:08.859Z 23 Acked 1022 Read 8 Sent Done Done false
104362024-02-28T22:17:08.859Z 24 Acked 1023 Read 9 Sent Done Done false
104372024-02-28T22:17:08.859Z 25 Acked 1024 Read 2 Sent Done Done false
104382024-02-28T22:17:08.859Z 26 Acked 1025 Read 8 Sent Done Done false
104392024-02-28T22:17:08.859Z 27 Acked 1026 Write 10 Sent Done Done false
104402024-02-28T22:17:08.859Z 28 Acked 1027 Write 4 Sent Done Done false
104412024-02-28T22:17:08.859Z 29 Acked 1028 Write 10 Sent Done Done false
104422024-02-28T22:17:08.859Z 30 Acked 1029 Read 9 Sent Done Sent false
104432024-02-28T22:17:08.859Z STATES DS:0 DS:1 DS:2 TOTAL
104442024-02-28T22:17:08.859Z New 0 0 0 0
104452024-02-28T22:17:08.859Z Sent 29 0 1 30
104462024-02-28T22:17:08.859Z Done 1 30 29 60
104472024-02-28T22:17:08.859Z Skipped 0 0 0 0
104482024-02-28T22:17:08.859Z Error 0 0 0 0
104492024-02-28T22:17:08.860ZLast Flush: 0 1016 1016
104502024-02-28T22:17:08.860ZDownstairs last five completed:
104512024-02-28T22:17:08.860ZUpstairs last five completed: 30 29 28 27 26
104522024-02-28T22:17:08.867Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:07.549482874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1519}
104532024-02-28T22:17:08.867ZWrote out file "/tmp/test_repair_verify.data"
104542024-02-28T22:17:08.868Z
104552024-02-28T22:17:08.868ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104562024-02-28T22:17:08.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104572024-02-28T22:17:08.868Z 0 000-019 90 90 90 163 163 163 F F F
104582024-02-28T22:17:08.868Z 1 020-039 74 74 74 136 136 136 F F F
104592024-02-28T22:17:08.868Z 2 040-059 92 96 96 166 175 175 F F F <---
104602024-02-28T22:17:08.868Z 3 060-079 94 94 94 171 171 171 F F F
104612024-02-28T22:17:08.868Z 4 080-099 93 96 96 170 175 175 T F F <---
104622024-02-28T22:17:08.868Z 5 100-119 93 96 96 170 175 175 T F F <---
104632024-02-28T22:17:08.868Z 6 120-139 95 95 95 174 174 174 F F F
104642024-02-28T22:17:08.868Z 7 140-159 90 96 96 163 175 175 F F F <---
104652024-02-28T22:17:08.868Z 8 160-179 88 88 88 159 159 159 F F F
104662024-02-28T22:17:08.868Z 9 180-199 92 92 92 169 169 169 F T T <---
104672024-02-28T22:17:08.868Z 10 200-219 92 92 92 165 165 165 F T T <---
104682024-02-28T22:17:08.868Z 11 220-239 92 96 96 169 175 175 F T T <---
104692024-02-28T22:17:08.868Z 12 240-259 92 92 92 168 168 168 F F F
104702024-02-28T22:17:08.868Z 13 260-279 84 84 84 151 151 151 F T T <---
104712024-02-28T22:17:08.869Z 14 280-299 90 90 90 163 163 163 F F F
104722024-02-28T22:17:08.869Z 15 300-319 94 94 94 171 171 171 F F F
104732024-02-28T22:17:08.869Z 16 320-339 94 96 96 171 175 175 F F F <---
104742024-02-28T22:17:08.869Z 17 340-359 84 84 84 154 154 154 F F F
104752024-02-28T22:17:08.869Z 18 360-379 94 94 94 171 171 171 F F F
104762024-02-28T22:17:08.869Z 19 380-399 95 95 95 174 174 174 F F F
104772024-02-28T22:17:08.869Z 20 400-419 94 94 94 171 171 171 F T T <---
104782024-02-28T22:17:08.869Z 21 420-439 90 90 90 163 163 163 F T T <---
104792024-02-28T22:17:08.869Z 22 440-459 94 94 94 173 173 173 F F F
104802024-02-28T22:17:08.869Z 23 460-479 95 95 95 174 174 174 F F F
104812024-02-28T22:17:08.869Z 24 480-499 82 82 82 149 149 149 F F F
104822024-02-28T22:17:08.869Z 25 500-519 94 94 94 171 171 171 F F F
104832024-02-28T22:17:08.869Z 26 520-539 94 94 94 171 171 171 F F F
104842024-02-28T22:17:08.869Z 27 540-559 94 96 96 171 175 175 F F F <---
104852024-02-28T22:17:08.869Z 28 560-579 94 96 96 173 175 175 F F F <---
104862024-02-28T22:17:08.869Z 29 580-599 92 92 92 166 166 166 F F F
104872024-02-28T22:17:08.869ZMax gen: 96, Max flush: 175
104882024-02-28T22:17:08.870ZA Difference in extent metadata is expected here
104892024-02-28T22:17:08.870ZOn loop 48
104902024-02-28T22:17:08.870Z
104912024-02-28T22:17:08.870ZVerifying data now
104922024-02-28T22:17:13.351ZNo change after verify
104932024-02-28T22:17:13.351ZLoop: 48 Downstairs dump after verify (and repair):
104942024-02-28T22:17:13.365ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104952024-02-28T22:17:13.365Z 0 000-019 90 90 90 163 163 163 F F F
104962024-02-28T22:17:13.365Z 1 020-039 74 74 74 136 136 136 F F F
104972024-02-28T22:17:13.365Z 2 040-059 96 96 96 175 175 175 F F F
104982024-02-28T22:17:13.365Z 3 060-079 94 94 94 171 171 171 F F F
104992024-02-28T22:17:13.365Z 4 080-099 96 96 96 175 175 175 F F F
105002024-02-28T22:17:13.365Z 5 100-119 96 96 96 175 175 175 F F F
105012024-02-28T22:17:13.365Z 6 120-139 95 95 95 174 174 174 F F F
105022024-02-28T22:17:13.365Z 7 140-159 96 96 96 175 175 175 F F F
105032024-02-28T22:17:13.365Z 8 160-179 88 88 88 159 159 159 F F F
105042024-02-28T22:17:13.365Z 9 180-199 97 97 97 176 176 176 F F F
105052024-02-28T22:17:13.365Z 10 200-219 97 97 97 176 176 176 F F F
105062024-02-28T22:17:13.366Z 11 220-239 97 97 97 176 176 176 F F F
105072024-02-28T22:17:13.369Z 12 240-259 92 92 92 168 168 168 F F F
105082024-02-28T22:17:13.369Z 13 260-279 97 97 97 176 176 176 F F F
105092024-02-28T22:17:13.369Z 14 280-299 90 90 90 163 163 163 F F F
105102024-02-28T22:17:13.369Z 15 300-319 94 94 94 171 171 171 F F F
105112024-02-28T22:17:13.369Z 16 320-339 96 96 96 175 175 175 F F F
105122024-02-28T22:17:13.369Z 17 340-359 84 84 84 154 154 154 F F F
105132024-02-28T22:17:13.369Z 18 360-379 94 94 94 171 171 171 F F F
105142024-02-28T22:17:13.369Z 19 380-399 95 95 95 174 174 174 F F F
105152024-02-28T22:17:13.369Z 20 400-419 97 97 97 176 176 176 F F F
105162024-02-28T22:17:13.369Z 21 420-439 97 97 97 176 176 176 F F F
105172024-02-28T22:17:13.369Z 22 440-459 94 94 94 173 173 173 F F F
105182024-02-28T22:17:13.369Z 23 460-479 95 95 95 174 174 174 F F F
105192024-02-28T22:17:13.370Z 24 480-499 82 82 82 149 149 149 F F F
105202024-02-28T22:17:13.370Z 25 500-519 94 94 94 171 171 171 F F F
105212024-02-28T22:17:13.370Z 26 520-539 94 94 94 171 171 171 F F F
105222024-02-28T22:17:13.370Z 27 540-559 96 96 96 175 175 175 F F F
105232024-02-28T22:17:13.370Z 28 560-579 96 96 96 175 175 175 F F F
105242024-02-28T22:17:13.370Z 29 580-599 92 92 92 166 166 166 F F F
105252024-02-28T22:17:13.370ZMax gen: 97, Max flush: 176
105262024-02-28T22:17:13.370Z
105272024-02-28T22:17:13.370ZBegin loop 49
105282024-02-28T22:17:13.370ZDownstairs to restart: 1
105292024-02-28T22:17:13.387Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277491612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
105302024-02-28T22:17:13.387Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277769884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
105312024-02-28T22:17:13.394Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277807513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
105322024-02-28T22:17:13.394ZCrucible runtime is spawned
105332024-02-28T22:17:13.394Z{"msg":"Crucible 16d88f87-3a6b-48dd-a7ad-4582be166787 has session id: 27298dae-704c-4bf0-89f5-9fd97dba29c3","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277848852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105342024-02-28T22:17:13.394Z{"msg":"Upstairs opts: Upstairs UUID: 16d88f87-3a6b-48dd-a7ad-4582be166787, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277894961Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105352024-02-28T22:17:13.394Z{"msg":"Crucible stats registered with UUID: 16d88f87-3a6b-48dd-a7ad-4582be166787","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.27792838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105362024-02-28T22:17:13.394Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277956919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
105372024-02-28T22:17:13.394Z{"msg":"16d88f87-3a6b-48dd-a7ad-4582be166787 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277977388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105382024-02-28T22:17:13.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.277999388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105392024-02-28T22:17:13.394Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278029787Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105402024-02-28T22:17:13.394Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278054056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105412024-02-28T22:17:13.395Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278099465Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105422024-02-28T22:17:13.395Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278125594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105432024-02-28T22:17:13.395Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278159323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105442024-02-28T22:17:13.395Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278184033Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105452024-02-28T22:17:13.395Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278230081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105462024-02-28T22:17:13.395Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278255601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105472024-02-28T22:17:13.395Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278338578Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105482024-02-28T22:17:13.395Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.27863809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105492024-02-28T22:17:13.395Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.278690619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105502024-02-28T22:17:13.395Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279048119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105512024-02-28T22:17:13.395Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279088318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105522024-02-28T22:17:13.395Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279193215Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105532024-02-28T22:17:13.395Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279222424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105542024-02-28T22:17:13.395Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279246453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105552024-02-28T22:17:13.395Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279279442Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105562024-02-28T22:17:13.395Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279615853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105572024-02-28T22:17:13.395Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.279811638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105582024-02-28T22:17:13.395Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280034582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105592024-02-28T22:17:13.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280235266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105602024-02-28T22:17:13.395Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280295255Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105612024-02-28T22:17:13.395Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280341253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105622024-02-28T22:17:13.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280373242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105632024-02-28T22:17:13.395Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280389912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105642024-02-28T22:17:13.396Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280411001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105652024-02-28T22:17:13.396Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280568017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105662024-02-28T22:17:13.396Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280612096Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105672024-02-28T22:17:13.396Z{"msg":"[0]R flush_numbers[0..12]: [163, 136, 175, 171, 175, 175, 174, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280639155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105682024-02-28T22:17:13.396Z{"msg":"[0]R generation[0..12]: [90, 74, 96, 94, 96, 96, 95, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280674524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105692024-02-28T22:17:13.396Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280701083Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105702024-02-28T22:17:13.396Z{"msg":"[1]R flush_numbers[0..12]: [163, 136, 175, 171, 175, 175, 174, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280735592Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105712024-02-28T22:17:13.396Z{"msg":"[1]R generation[0..12]: [90, 74, 96, 94, 96, 96, 95, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280771071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
105722024-02-28T22:17:13.396Z,"pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105732024-02-28T22:17:13.396Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.28081351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105742024-02-28T22:17:13.396Z{"msg":"[2]R flush_numbers[0..12]: [163, 136, 175, 171, 175, 175, 174, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280850439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105752024-02-28T22:17:13.396Z{"msg":"[2]R generation[0..12]: [90, 74, 96, 94, 96, 96, 95, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280889598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105762024-02-28T22:17:13.396Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280928737Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105772024-02-28T22:17:13.396Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.280965906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105782024-02-28T22:17:13.396Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281023174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105792024-02-28T22:17:13.396Z{"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281063643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105802024-02-28T22:17:13.396Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281100932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"":"downstairs","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105812024-02-28T22:17:13.396Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281138131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105822024-02-28T22:17:13.396Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.28118162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"0","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105832024-02-28T22:17:13.396Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105842024-02-28T22:17:13.401ZTransition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281208049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"1","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105852024-02-28T22:17:13.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281232169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"client":"2","":"downstairs","session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105862024-02-28T22:17:13.401Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281246448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105872024-02-28T22:17:13.401Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281259128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105882024-02-28T22:17:13.401Z{"msg":"16d88f87-3a6b-48dd-a7ad-4582be166787 is now active with session: 27298dae-704c-4bf0-89f5-9fd97dba29c3","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281272177Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105892024-02-28T22:17:13.401Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281284947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527,"session_id":"27298dae-704c-4bf0-89f5-9fd97dba29c3"}
105902024-02-28T22:17:13.401Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.281297507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
105912024-02-28T22:17:13.401ZLoading write count information from file "/tmp/test_repair_verify.data"
105922024-02-28T22:17:13.401ZRun Repair workload
105932024-02-28T22:17:13.402Z01/30 Write block 370 len 4096 data: 10
105942024-02-28T22:17:13.402Z02/30 Write block 254 len 4096 data: 8
105952024-02-28T22:17:13.402Z03/30 Read block 471 len 20480
105962024-02-28T22:17:13.402Z04/30 Read block 431 len 8192
105972024-02-28T22:17:13.402Z05/30 Read block 58 len 20480
105982024-02-28T22:17:13.402Z06/30 Read block 578 len 20480
105992024-02-28T22:17:13.402Z07/30 Read block 157 len 28672
106002024-02-28T22:17:13.402Z08/30 Read block 397 len 36864
106012024-02-28T22:17:13.402Z09/30 Read block 369 len 12288
106022024-02-28T22:17:13.402Z10/30 Write block 29 len 24576 data: 7 7 6 6 6 5
106032024-02-28T22:17:13.402Z11/30 Read block 61 len 32768
106042024-02-28T22:17:13.405Z12/30 Write block 17 len 24576 data: 9 9 9 6 5 4
106052024-02-28T22:17:13.405Z13/30 Read block 484 len 8192
106062024-02-28T22:17:13.405Z14/30 Flush
106072024-02-28T22:17:13.408Z15/30 Write block 333 len 28672 data: 9 10 10 10 8 9 9
106082024-02-28T22:17:13.408Z16/30 Write block 242 len 8192 data: 7 7
106092024-02-28T22:17:13.409Z17/30 Flush
106102024-02-28T22:17:13.411Z18/30 Write block 446 len 40960 data: 10 10 9 10 11 11 9 10 8 8
106112024-02-28T22:17:13.412Z19/30 Write block 380 len 28672 data: 10 10 14 14 14 12 12
106122024-02-28T22:17:13.412Z20/30 Flush
106132024-02-28T22:17:13.415Z21/30 Write block 31 len 36864 data: 7 7 7 6 7 8 9 10 9
106142024-02-28T22:17:13.415Z22/30 Read block 2 len 8192
106152024-02-28T22:17:13.420Z23/30 Write block 452 len 40960 data: 10 11 9 9 9 8 9 8 5 5
106162024-02-28T22:17:13.421Z24/30 Write block 445 len 28672 data: 9 11 11 10 11 12 12
106172024-02-28T22:17:13.424Z25/30 Write block 127 len 28672 data: 13 14 11 11 12 11 12
106182024-02-28T22:17:13.424Z26/30 Read block 217 len 40960
106192024-02-28T22:17:13.424Z27/30 Write block 121 len 8192 data: 9 10
106202024-02-28T22:17:13.424Z28/30 Write block 81 len 12288 data: 9 11 9
106212024-02-28T22:17:13.424Z29/30 Write block 465 len 32768 data: 7 5 4 7 6 7 8 10
106222024-02-28T22:17:13.424Z30/30 Write block 417 len 24576 data: 8 11 11 13 13 12
106232024-02-28T22:17:13.424Z----------------------------------------------------------------
106242024-02-28T22:17:13.424Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:16
106252024-02-28T22:17:13.425ZStates: Active Active Active
106262024-02-28T22:17:13.425ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106272024-02-28T22:17:13.425Z 15 Acked 1014 Write 7 Done Done Done false
106282024-02-28T22:17:13.425Z 16 Acked 1015 Write 2 Done Done Done false
106292024-02-28T22:17:13.425Z 17 Acked 1016 Flush 0 Done Sent Done false
106302024-02-28T22:17:13.425Z 18 Acked 1017 Write 10 Done Sent Done false
106312024-02-28T22:17:13.425Z 19 Acked 1018 Write 7 Done Sent Done false
106322024-02-28T22:17:13.425Z 20 Acked 1019 Flush 0 Done Sent Done false
106332024-02-28T22:17:13.425Z 21 Acked 1020 Write 9 Done Sent Done false
106342024-02-28T22:17:13.425Z 22 Acked 1021 Read 2 Done Sent Done false
106352024-02-28T22:17:13.425Z 23 Acked 1022 Write 10 Done Sent Done false
106362024-02-28T22:17:13.425Z 24 Acked 1023 Write 7 Done Sent Done false
106372024-02-28T22:17:13.425Z 25 Acked 1024 Write 7 Done Sent Done false
106382024-02-28T22:17:13.425Z 26 Acked 1025 Read 10 Done Sent Done false
106392024-02-28T22:17:13.425Z 27 Acked 1026 Write 2 Done Sent Done false
106402024-02-28T22:17:13.425Z 28 Acked 1027 Write 3 Done Sent Done false
106412024-02-28T22:17:13.425Z 29 Acked 1028 Write 8 Done Sent Done false
106422024-02-28T22:17:13.425Z 30 Acked 1029 Write 6 Sent Sent Sent false
106432024-02-28T22:17:13.425Z STATES DS:0 DS:1 DS:2 TOTAL
106442024-02-28T22:17:13.425Z New 0 0 0 0
106452024-02-28T22:17:13.425Z Sent 1 14 1 16
106462024-02-28T22:17:13.425Z Done 15 2 15 32
106472024-02-28T22:17:13.425Z Skipped 0 0 0 0
106482024-02-28T22:17:13.426Z Error 0 0 0 0
106492024-02-28T22:17:13.426ZLast Flush: 1019 1013 1019
106502024-02-28T22:17:13.426ZDownstairs last five completed: 1013 1012 1011 1010 1009
106512024-02-28T22:17:13.426ZUpstairs last five completed: 30 29 28 27 26
106522024-02-28T22:17:13.426Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:12.313532799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1527}
106532024-02-28T22:17:13.426ZWrote out file "/tmp/test_repair_verify.data"
106542024-02-28T22:17:13.429Z
106552024-02-28T22:17:13.432ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106562024-02-28T22:17:13.448ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106572024-02-28T22:17:13.448Z 0 000-019 98 98 98 177 177 177 F F F
106582024-02-28T22:17:13.448Z 1 020-039 98 98 98 177 177 177 T F T <---
106592024-02-28T22:17:13.448Z 2 040-059 96 96 96 175 175 175 F F F
106602024-02-28T22:17:13.449Z 3 060-079 94 94 94 171 171 171 F F F
106612024-02-28T22:17:13.449Z 4 080-099 96 96 96 175 175 175 T F T <---
106622024-02-28T22:17:13.452Z 5 100-119 96 96 96 175 175 175 F F F
106632024-02-28T22:17:13.452Z 6 120-139 95 95 95 174 174 174 T F T <---
106642024-02-28T22:17:13.452Z 7 140-159 96 96 96 175 175 175 F F F
106652024-02-28T22:17:13.452Z 8 160-179 88 88 88 159 159 159 F F F
106662024-02-28T22:17:13.452Z 9 180-199 97 97 97 176 176 176 F F F
106672024-02-28T22:17:13.452Z 10 200-219 97 97 97 176 176 176 F F F
106682024-02-28T22:17:13.452Z 11 220-239 97 97 97 176 176 176 F F F
106692024-02-28T22:17:13.452Z 12 240-259 98 98 98 178 177 178 F T F <---
106702024-02-28T22:17:13.452Z 13 260-279 97 97 97 176 176 176 F F F
106712024-02-28T22:17:13.452Z 14 280-299 90 90 90 163 163 163 F F F
106722024-02-28T22:17:13.452Z 15 300-319 94 94 94 171 171 171 F F F
106732024-02-28T22:17:13.452Z 16 320-339 98 96 98 178 175 178 F T F <---
106742024-02-28T22:17:13.452Z 17 340-359 84 84 84 154 154 154 F F F
106752024-02-28T22:17:13.452Z 18 360-379 98 98 98 177 177 177 F F F
106762024-02-28T22:17:13.453Z 19 380-399 98 95 98 179 174 179 F F F <---
106772024-02-28T22:17:13.453Z 20 400-419 97 97 97 176 176 176 T F T <---
106782024-02-28T22:17:13.453Z 21 420-439 97 97 97 176 176 176 T F T <---
106792024-02-28T22:17:13.453Z 22 440-459 98 94 98 179 173 179 T F T <---
106802024-02-28T22:17:13.453Z 23 460-479 95 95 95 174 174 174 T F T <---
106812024-02-28T22:17:13.453Z 24 480-499 82 82 82 149 149 149 F F F
106822024-02-28T22:17:13.453Z 25 500-519 94 94 94 171 171 171 F F F
106832024-02-28T22:17:13.453Z 26 520-539 94 94 94 171 171 171 F F F
106842024-02-28T22:17:13.453Z 27 540-559 96 96 96 175 175 175 F F F
106852024-02-28T22:17:13.453Z 28 560-579 96 96 96 175 175 175 F F F
106862024-02-28T22:17:13.453Z 29 580-599 92 92 92 166 166 166 F F F
106872024-02-28T22:17:13.453ZMax gen: 98, Max flush: 179
106882024-02-28T22:17:13.453ZError: Difference in extent metadata found!
106892024-02-28T22:17:13.453ZA Difference in extent metadata is expected here
106902024-02-28T22:17:13.453ZOn loop 49
106912024-02-28T22:17:13.453Z
106922024-02-28T22:17:13.456ZVerifying data now
106932024-02-28T22:17:13.987ZNo change after verify
106942024-02-28T22:17:13.987ZLoop: 49 Downstairs dump after verify (and repair):
106952024-02-28T22:17:14.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106962024-02-28T22:17:14.002Z 0 000-019 98 98 98 177 177 177 F F F
106972024-02-28T22:17:14.002Z 1 020-039 99 99 99 180 180 180 F F F
106982024-02-28T22:17:14.002Z 2 040-059 96 96 96 175 175 175 F F F
106992024-02-28T22:17:14.002Z 3 060-079 94 94 94 171 171 171 F F F
107002024-02-28T22:17:14.002Z 4 080-099 99 99 99 180 180 180 F F F
107012024-02-28T22:17:14.002Z 5 100-119 96 96 96 175 175 175 F F F
107022024-02-28T22:17:14.002Z 6 120-139 99 99 99 180 180 180 F F F
107032024-02-28T22:17:14.002Z 7 140-159 96 96 96 175 175 175 F F F
107042024-02-28T22:17:14.002Z 8 160-179 88 88 88 159 159 159 F F F
107052024-02-28T22:17:14.002Z 9 180-199 97 97 97 176 176 176 F F F
107062024-02-28T22:17:14.002Z 10 200-219 97 97 97 176 176 176 F F F
107072024-02-28T22:17:14.002Z 11 220-239 97 97 97 176 176 176 F F F
107082024-02-28T22:17:14.002Z 12 240-259 98 98 98 178 178 178 F F F
107092024-02-28T22:17:14.002Z 13 260-279 97 97 97 176 176 176 F F F
107102024-02-28T22:17:14.002Z 14 280-299 90 90 90 163 163 163 F F F
107112024-02-28T22:17:14.002Z 15 300-319 94 94 94 171 171 171 F F F
107122024-02-28T22:17:14.006Z 16 320-339 98 98 98 178 178 178 F F F
107132024-02-28T22:17:14.006Z 17 340-359 84 84 84 154 154 154 F F F
107142024-02-28T22:17:14.006Z 18 360-379 98 98 98 177 177 177 F F F
107152024-02-28T22:17:14.006Z 19 380-399 98 98 98 179 179 179 F F F
107162024-02-28T22:17:14.006Z 20 400-419 99 99 99 180 180 180 F F F
107172024-02-28T22:17:14.006Z 21 420-439 99 99 99 180 180 180 F F F
107182024-02-28T22:17:14.006Z 22 440-459 99 99 99 180 180 180 F F F
107192024-02-28T22:17:14.006Z 23 460-479 99 99 99 180 180 180 F F F
107202024-02-28T22:17:14.006Z 24 480-499 82 82 82 149 149 149 F F F
107212024-02-28T22:17:14.006Z 25 500-519 94 94 94 171 171 171 F F F
107222024-02-28T22:17:14.006Z 26 520-539 94 94 94 171 171 171 F F F
107232024-02-28T22:17:14.007Z 27 540-559 96 96 96 175 175 175 F F F
107242024-02-28T22:17:14.007Z 28 560-579 96 96 96 175 175 175 F F F
107252024-02-28T22:17:14.007Z 29 580-599 92 92 92 166 166 166 F F F
107262024-02-28T22:17:14.007ZMax gen: 99, Max flush: 180
107272024-02-28T22:17:14.007Z
107282024-02-28T22:17:14.007ZBegin loop 50
107292024-02-28T22:17:14.007ZDownstairs to restart: 2
107302024-02-28T22:17:14.023Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.913556319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
107312024-02-28T22:17:14.024Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
107322024-02-28T22:17:14.024Z,"time":"2024-02-28T22:17:12.91386675Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
107332024-02-28T22:17:14.030Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.913915109Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
107342024-02-28T22:17:14.030Z{"msg":"Crucible 2e8b1d0b-1e77-4f0a-89c0-0cc11da12f63 has session id: f4e474b3-a937-44ef-8087-a76c39512b67","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.913948548Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107352024-02-28T22:17:14.030Z{"msg":"Upstairs opts: Upstairs UUID: 2e8b1d0b-1e77-4f0a-89c0-0cc11da12f63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914014246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107362024-02-28T22:17:14.030Z{"msg":"Crucible stats registered with UUID: 2e8b1d0b-1e77-4f0a-89c0-0cc11da12f63","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914057645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107372024-02-28T22:17:14.030Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914077504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
107382024-02-28T22:17:14.030Z{"msg":"2e8b1d0b-1e77-4f0a-89c0-0cc11da12f63 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914155952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107392024-02-28T22:17:14.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914182421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107402024-02-28T22:17:14.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914202221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107412024-02-28T22:17:14.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91422685Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107422024-02-28T22:17:14.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914262089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107432024-02-28T22:17:14.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914288558Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107442024-02-28T22:17:14.031Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914324997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107452024-02-28T22:17:14.031Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914349877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107462024-02-28T22:17:14.031Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914391896Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107472024-02-28T22:17:14.031Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914419645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107482024-02-28T22:17:14.031Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91458683Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107492024-02-28T22:17:14.031Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914635749Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107502024-02-28T22:17:14.031Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.914720317Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107512024-02-28T22:17:14.031Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915271362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107522024-02-28T22:17:14.031Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915300371Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107532024-02-28T22:17:14.031Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915361179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107542024-02-28T22:17:14.031Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915421677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107552024-02-28T22:17:14.031Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915453537Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107562024-02-28T22:17:14.031Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915481146Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107572024-02-28T22:17:14.031Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915947533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107582024-02-28T22:17:14.031Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.915994751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107592024-02-28T22:17:14.031Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916073259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107602024-02-28T22:17:14.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916598805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107612024-02-28T22:17:14.031Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916643244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107622024-02-28T22:17:14.031Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916664383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107632024-02-28T22:17:14.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916677953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107642024-02-28T22:17:14.031Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916692432Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107652024-02-28T22:17:14.031Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916705972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107662024-02-28T22:17:14.031Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916727541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107672024-02-28T22:17:14.032Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916741841Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107682024-02-28T22:17:14.032Z{"msg":"[0]R flush_numbers[0..12]: [177, 180, 175, 171, 180, 175, 180, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91676336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107692024-02-28T22:17:14.032Z{"msg":"[0]R generation[0..12]: [98, 99, 96, 94, 99, 96, 99, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91677794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107702024-02-28T22:17:14.032Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916809399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO
107712024-02-28T22:17:14.032Z"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107722024-02-28T22:17:14.032Z{"msg":"[1]R flush_numbers[0..12]: [177, 180, 175, 171, 180, 175, 180, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916857388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107732024-02-28T22:17:14.032Z{"msg":"[1]R generation[0..12]: [98, 99, 96, 94, 99, 96, 99, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916881867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107742024-02-28T22:17:14.032Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916909066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107752024-02-28T22:17:14.032Z{"msg":"[2]R flush_numbers[0..12]: [177, 180, 175, 171, 180, 175, 180, 175, 159, 176, 176, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916924276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107762024-02-28T22:17:14.032Z{"msg":"[2]R generation[0..12]: [98, 99, 96, 94, 99, 96, 99, 96, 88, 97, 97, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916951355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107772024-02-28T22:17:14.032Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.916989754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107782024-02-28T22:17:14.032Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917014783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107792024-02-28T22:17:14.032Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917030003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107802024-02-28T22:17:14.032Z{"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917055792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107812024-02-28T22:17:14.032Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917069692Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"":"downstairs","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107822024-02-28T22:17:14.032Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917090211Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107832024-02-28T22:17:14.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917102841Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"0","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107842024-02-28T22:17:14.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91712332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"1","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107852024-02-28T22:17:14.032Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91713771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"client":"2","":"downstairs","session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107862024-02-28T22:17:14.032Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.91715155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107872024-02-28T22:17:14.032Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917171009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107882024-02-28T22:17:14.032Z{"msg":"2e8b1d0b-1e77-4f0a-89c0-0cc11da12f63 is now active with session: f4e474b3-a937-44ef-8087-a76c39512b67","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917184259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107892024-02-28T22:17:14.032Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917203478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535,"session_id":"f4e474b3-a937-44ef-8087-a76c39512b67"}
107902024-02-28T22:17:14.032Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:12.917216148Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
107912024-02-28T22:17:14.033ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
107922024-02-28T22:17:14.041ZLoading write count information from file "/tmp/test_repair_verify.data"
107932024-02-28T22:17:14.041ZRun Repair workload
107942024-02-28T22:17:14.041Z01/30 Flush
107952024-02-28T22:17:14.041Z02/30 Write block 341 len 16384 data: 8 9 8 6
107962024-02-28T22:17:14.041Z03/30 Read block 490 len 24576
107972024-02-28T22:17:14.041Z04/30 Read block 520 len 4096
107982024-02-28T22:17:14.041Z05/30 Read block 293 len 24576
107992024-02-28T22:17:14.041Z06/30 Flush
108002024-02-28T22:17:14.041Z07/30 Write block 108 len 20480 data: 10 12 11 10 10
108012024-02-28T22:17:14.041Z08/30 Read block 98 len 32768
108022024-02-28T22:17:14.041Z09/30 Read block 211 len 28672
108032024-02-28T22:17:14.041Z10/30 Read block 585 len 36864
108042024-02-28T22:17:14.041Z11/30 Read block 125 len 40960
108052024-02-28T22:17:14.045Z12/30 Write block 254 len 16384 data: 9 7 10 10
108062024-02-28T22:17:14.045Z13/30 Write block 56 len 8192 data: 8 8
108072024-02-28T22:17:14.045Z14/30 Flush
108082024-02-28T22:17:14.048Z15/30 Write block 453 len 12288 data: 12 10 10
108092024-02-28T22:17:14.048Z16/30 Read block 237 len 24576
108102024-02-28T22:17:14.048Z17/30 Read block 295 len 32768
108112024-02-28T22:17:14.048Z18/30 Read block 338 len 24576
108122024-02-28T22:17:14.051Z19/30 Write block 391 len 20480 data: 9 8 7 6 8
108132024-02-28T22:17:14.051Z20/30 Read block 242 len 28672
108142024-02-28T22:17:14.051Z21/30 Read block 471 len 36864
108152024-02-28T22:17:14.054Z22/30 Read block 482 len 20480
108162024-02-28T22:17:14.054Z23/30 Write block 59 len 32768 data: 11 12 11 13 11 11 11 10
108172024-02-28T22:17:14.054Z24/30 Write block 566 len 12288 data: 11 13 12
108182024-02-28T22:17:14.054Z25/30 Read block 102 len 24576
108192024-02-28T22:17:14.057Z26/30 Write block 563 len 32768 data: 9 9 10 12 14 13 11 10
108202024-02-28T22:17:14.057Z27/30 Write block 107 len 16384 data: 8 11 13 12
108212024-02-28T22:17:14.057Z28/30 Write block 204 len 12288 data: 6 8 6
108222024-02-28T22:17:14.057Z29/30 Read block 185 len 16384
108232024-02-28T22:17:14.061Z30/30 Read block 355 len 36864
108242024-02-28T22:17:14.061Z----------------------------------------------------------------
108252024-02-28T22:17:14.061Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:24
108262024-02-28T22:17:14.061ZStates: Active Active Active
108272024-02-28T22:17:14.062ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108282024-02-28T22:17:14.062Z 7 Acked 1006 Write 5 Done Done Sent false
108292024-02-28T22:17:14.062Z 8 Acked 1007 Read 8 Done Done Sent false
108302024-02-28T22:17:14.062Z 9 Acked 1008 Read 7 Done Done Sent false
108312024-02-28T22:17:14.062Z 10 Acked 1009 Read 9 Done Done Sent false
108322024-02-28T22:17:14.062Z 11 Acked 1010 Read 10 Done Done Sent false
108332024-02-28T22:17:14.062Z 12 Acked 1011 Write 4 Done Done Sent false
108342024-02-28T22:17:14.062Z 13 Acked 1012 Write 2 Done Done Sent false
108352024-02-28T22:17:14.062Z 14 Acked 1013 Flush 0 Done Done Sent false
108362024-02-28T22:17:14.062Z 15 Acked 1014 Write 3 Done Done Sent false
108372024-02-28T22:17:14.062Z 16 Acked 1015 Read 6 Done Done Sent false
108382024-02-28T22:17:14.062Z 17 Acked 1016 Read 8 Done Done Sent false
108392024-02-28T22:17:14.062Z 18 Acked 1017 Read 6 Done Done Sent false
108402024-02-28T22:17:14.062Z 19 Acked 1018 Write 5 Done Done Sent false
108412024-02-28T22:17:14.062Z 20 Acked 1019 Read 7 Done Done Sent false
108422024-02-28T22:17:14.062Z 21 Acked 1020 Read 9 Done Done Sent false
108432024-02-28T22:17:14.062Z 22 Acked 1021 Read 5 Done Done Sent false
108442024-02-28T22:17:14.062Z 23 Acked 1022 Write 8 Done Done Sent false
108452024-02-28T22:17:14.062Z 24 Acked 1023 Write 3 Done Done Sent false
108462024-02-28T22:17:14.062Z 25 Acked 1024 Read 6 Done Done Sent false
108472024-02-28T22:17:14.062Z 26 Acked 1025 Write 8 Done Done Sent false
108482024-02-28T22:17:14.062Z 27 Acked 1026 Write 4 Done Done Sent false
108492024-02-28T22:17:14.063Z 28 Acked 1027 Write 3 Done Done Sent false
108502024-02-28T22:17:14.063Z 29 Acked 1028 Read 4 Done Done Sent false
108512024-02-28T22:17:14.063Z 30 Acked 1029 Read 9 Done Sent Sent false
108522024-02-28T22:17:14.063Z STATES DS:0 DS:1 DS:2 TOTAL
108532024-02-28T22:17:14.063Z New 0 0 0 0
108542024-02-28T22:17:14.063Z Sent 0 1 24 25
108552024-02-28T22:17:14.063Z Done 24 23 0 47
108562024-02-28T22:17:14.063Z Skipped 0 0 0 0
108572024-02-28T22:17:14.063Z Error 0 0 0 0
108582024-02-28T22:17:14.063ZLast Flush: 1013 1013 1005
108592024-02-28T22:17:14.063ZDownstairs last five completed: 1005 1004 1003 1002 1001
108602024-02-28T22:17:14.063ZUpstairs last five completed: 30 29 28 27 26
108612024-02-28T22:17:14.063Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:12.949615745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1535}
108622024-02-28T22:17:14.063ZWrote out file "/tmp/test_repair_verify.data"
108632024-02-28T22:17:14.067Z
108642024-02-28T22:17:14.073ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108652024-02-28T22:17:14.084ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108662024-02-28T22:17:14.084Z 0 000-019 98 98 98 177 177 177 F F F
108672024-02-28T22:17:14.084Z 1 020-039 99 99 99 180 180 180 F F F
108682024-02-28T22:17:14.084Z 2 040-059 100 100 96 183 183 175 T T F <---
108692024-02-28T22:17:14.084Z 3 060-079 94 94 94 171 171 171 T T F <---
108702024-02-28T22:17:14.084Z 4 080-099 99 99 99 180 180 180 F F F
108712024-02-28T22:17:14.084Z 5 100-119 100 100 96 183 183 175 T T F <---
108722024-02-28T22:17:14.084Z 6 120-139 99 99 99 180 180 180 F F F
108732024-02-28T22:17:14.084Z 7 140-159 96 96 96 175 175 175 F F F
108742024-02-28T22:17:14.084Z 8 160-179 88 88 88 159 159 159 F F F
108752024-02-28T22:17:14.084Z 9 180-199 97 97 97 176 176 176 F F F
108762024-02-28T22:17:14.088Z 10 200-219 97 97 97 176 176 176 T T F <---
108772024-02-28T22:17:14.088Z 11 220-239 97 97 97 176 176 176 F F F
108782024-02-28T22:17:14.088Z 12 240-259 100 100 98 183 183 178 F F F <---
108792024-02-28T22:17:14.088Z 13 260-279 97 97 97 176 176 176 F F F
108802024-02-28T22:17:14.088Z 14 280-299 90 90 90 163 163 163 F F F
108812024-02-28T22:17:14.088Z 15 300-319 94 94 94 171 171 171 F F F
108822024-02-28T22:17:14.088Z 16 320-339 98 98 98 178 178 178 F F F
108832024-02-28T22:17:14.088Z 17 340-359 100 100 100 182 182 182 F F F
108842024-02-28T22:17:14.088Z 18 360-379 98 98 98 177 177 177 F F F
108852024-02-28T22:17:14.088Z 19 380-399 98 98 98 179 179 179 T T F <---
108862024-02-28T22:17:14.088Z 20 400-419 99 99 99 180 180 180 F F F
108872024-02-28T22:17:14.088Z 21 420-439 99 99 99 180 180 180 F F F
108882024-02-28T22:17:14.088Z 22 440-459 99 99 99 180 180 180 T T F <---
108892024-02-28T22:17:14.088Z 23 460-479 99 99 99 180 180 180 F F F
108902024-02-28T22:17:14.088Z 24 480-499 82 82 82 149 149 149 F F F
108912024-02-28T22:17:14.088Z 25 500-519 94 94 94 171 171 171 F F F
108922024-02-28T22:17:14.088Z 26 520-539 94 94 94 171 171 171 F F F
108932024-02-28T22:17:14.088Z 27 540-559 96 96 96 175 175 175 F F F
108942024-02-28T22:17:14.088Z 28 560-579 96 96 96 175 175 175 T T F <---
108952024-02-28T22:17:14.089Z 29 580-599 92 92 92 166 166 166 F F F
108962024-02-28T22:17:14.089ZMax gen: 100, Max flush: 183
108972024-02-28T22:17:14.089ZError: Difference in extent metadata found!
108982024-02-28T22:17:14.089ZA Difference in extent metadata is expected here
108992024-02-28T22:17:14.089ZOn loop 50
109002024-02-28T22:17:14.089Z
109012024-02-28T22:17:14.092ZVerifying data now
109022024-02-28T22:17:18.549ZNo change after verify
109032024-02-28T22:17:18.549ZLoop: 50 Downstairs dump after verify (and repair):
109042024-02-28T22:17:18.561ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109052024-02-28T22:17:18.561Z 0 000-019 98 98 98 177 177 177 F F F
109062024-02-28T22:17:18.561Z 1 020-039 99 99 99 180 180 180 F F F
109072024-02-28T22:17:18.561Z 2 040-059 101 101 101 184 184 184 F F F
109082024-02-28T22:17:18.561Z 3 060-079 101 101 101 184 184 184 F F F
109092024-02-28T22:17:18.561Z 4 080-099 99 99 99 180 180 180 F F F
109102024-02-28T22:17:18.561Z 5 100-119 101 101 101 184 184 184 F F F
109112024-02-28T22:17:18.561Z 6 120-139 99 99 99 180 180 180 F F F
109122024-02-28T22:17:18.561Z 7 140-159 96 96 96 175 175 175 F F F
109132024-02-28T22:17:18.561Z 8 160-179 88 88 88 159 159 159 F F F
109142024-02-28T22:17:18.561Z 9 180-199 97 97 97 176 176 176 F F F
109152024-02-28T22:17:18.561Z 10 200-219 101 101 101 184 184 184 F F F
109162024-02-28T22:17:18.561Z 11 220-239 97 97 97 176 176 176 F F F
109172024-02-28T22:17:18.561Z 12 240-259 100 100 100 183 183 183 F F F
109182024-02-28T22:17:18.561Z 13 260-279 97 97 97 176 176 176 F F F
109192024-02-28T22:17:18.561Z 14 280-299 90 90 90 163 163 163 F F F
109202024-02-28T22:17:18.561Z 15 300-319 94 94 94 171 171 171 F F F
109212024-02-28T22:17:18.561Z 16 320-339 98 98 98 178 178 178 F F F
109222024-02-28T22:17:18.561Z 17 340-359 100 100 100 182 182 182 F F F
109232024-02-28T22:17:18.567Z 18 360-379 98 98 98 177 177 177 F F F
109242024-02-28T22:17:18.567Z 19 380-399 101 101 101 184 184 184 F F F
109252024-02-28T22:17:18.567Z 20 400-419 99 99 99 180 180 180 F F F
109262024-02-28T22:17:18.567Z 21 420-439 99 99 99 180 180 180 F F F
109272024-02-28T22:17:18.567Z 22 440-459 101 101 101 184 184 184 F F F
109282024-02-28T22:17:18.567Z 23 460-479 99 99 99 180 180 180 F F F
109292024-02-28T22:17:18.567Z 24 480-499 82 82 82 149 149 149 F F F
109302024-02-28T22:17:18.567Z 25 500-519 94 94 94 171 171 171 F F F
109312024-02-28T22:17:18.567Z 26 520-539 94 94 94 171 171 171 F F F
109322024-02-28T22:17:18.567Z 27 540-559 96 96 96 175 175 175 F F F
109332024-02-28T22:17:18.567Z 28 560-579 101 101 101 184 184 184 F F F
109342024-02-28T22:17:18.567Z 29 580-599 92 92 92 166 166 166 F F F
109352024-02-28T22:17:18.567ZMax gen: 101, Max flush: 184
109362024-02-28T22:17:18.567Z
109372024-02-28T22:17:18.567ZBegin loop 51
109382024-02-28T22:17:18.568ZDownstairs to restart: 1
109392024-02-28T22:17:18.583Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473152397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
109402024-02-28T22:17:18.589Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473436679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
109412024-02-28T22:17:18.590Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible"Crucible runtime is spawned
109422024-02-28T22:17:18.590Z,"level":30,"time":"2024-02-28T22:17:17.473481788Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
109432024-02-28T22:17:18.590Z{"msg":"Crucible ca73b9c6-cbfc-4013-8ab1-600a2e123620 has session id: dd59e3e7-403a-4d00-8197-388c52ed55f6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473516197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109442024-02-28T22:17:18.590Z{"msg":"Upstairs opts: Upstairs UUID: ca73b9c6-cbfc-4013-8ab1-600a2e123620, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473560395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109452024-02-28T22:17:18.590Z{"msg":"Crucible stats registered with UUID: ca73b9c6-cbfc-4013-8ab1-600a2e123620","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473594554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109462024-02-28T22:17:18.590Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473619054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
109472024-02-28T22:17:18.590Z{"msg":"ca73b9c6-cbfc-4013-8ab1-600a2e123620 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473690982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109482024-02-28T22:17:18.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473727221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109492024-02-28T22:17:18.590Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.4737454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109502024-02-28T22:17:18.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473787069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109512024-02-28T22:17:18.590Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473816018Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109522024-02-28T22:17:18.590Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473858827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109532024-02-28T22:17:18.590Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473875147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109542024-02-28T22:17:18.590Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473888716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109552024-02-28T22:17:18.590Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473913496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109562024-02-28T22:17:18.590Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.473949845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109572024-02-28T22:17:18.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474086981Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109582024-02-28T22:17:18.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.47412967Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109592024-02-28T22:17:18.590Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474158099Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109602024-02-28T22:17:18.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474882009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109612024-02-28T22:17:18.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474914988Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109622024-02-28T22:17:18.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474930118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109632024-02-28T22:17:18.591Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474944377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109642024-02-28T22:17:18.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474967117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109652024-02-28T22:17:18.591Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.474982246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109662024-02-28T22:17:18.591Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.475488372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109672024-02-28T22:17:18.591Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.475517912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109682024-02-28T22:17:18.591Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.475751245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109692024-02-28T22:17:18.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476073446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109702024-02-28T22:17:18.591Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476111095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109712024-02-28T22:17:18.591Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476128405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109722024-02-28T22:17:18.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476170674Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109732024-02-28T22:17:18.591Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476199413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109742024-02-28T22:17:18.591Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476214252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109752024-02-28T22:17:18.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476246352Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109762024-02-28T22:17:18.591Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476273131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109772024-02-28T22:17:18.591Z{"msg":"[0]R flush_numbers[0..12]: [177, 180, 184, 184, 180, 184, 180, 175, 159, 176, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.47629974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109782024-02-28T22:17:18.591Z{"msg":"[0]R generation[0..12]: [98, 99, 101, 101, 99, 101, 99, 96, 88, 97, 101, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476318579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109792024-02-28T22:17:18.591Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476342709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109802024-02-28T22:17:18.591Z{"msg":"[1]R flush_numbers[0..12]: [177, 180, 184, 184, 180, 184, 180, 175, 159, 176, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476357498Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109812024-02-28T22:17:18.591Z{"msg":"[1]R generation[0..12]: [98, 99, 101, 101, 99, 101, 99, 96, 88, 97, 101, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476371368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109822024-02-28T22:17:18.591Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476392937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109832024-02-28T22:17:18.591Z{"msg":"[2]R flush_numbers[0..12]: [177, 180, 184, 184, 180, 184, 180, 175, 159, 176, 184, 176]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476407557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109842024-02-28T22:17:18.591ZWait for a query_work_queue command to finish before sending IO
109852024-02-28T22:17:18.592Z{"msg":"[2]R generation[0..12]: [98, 99, 101, 101, 99, 101, 99, 96, 88, 97, 101, 97]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476428327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109862024-02-28T22:17:18.592Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476445216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109872024-02-28T22:17:18.592Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476463695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109882024-02-28T22:17:18.592Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476485015Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109892024-02-28T22:17:18.592Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476498525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109902024-02-28T22:17:18.592Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476511724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"":"downstairs","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109912024-02-28T22:17:18.592Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476530744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109922024-02-28T22:17:18.592Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476560053Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"0","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109932024-02-28T22:17:18.592Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476575972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"1","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109942024-02-28T22:17:18.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476611881Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"client":"2","":"downstairs","session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109952024-02-28T22:17:18.596Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.47664357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109962024-02-28T22:17:18.596Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.47666279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109972024-02-28T22:17:18.596Z{"msg":"ca73b9c6-cbfc-4013-8ab1-600a2e123620 is now active with session: dd59e3e7-403a-4d00-8197-388c52ed55f6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476695589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109982024-02-28T22:17:18.596Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476722178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543,"session_id":"dd59e3e7-403a-4d00-8197-388c52ed55f6"}
109992024-02-28T22:17:18.596Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:17.476744028Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
110002024-02-28T22:17:18.596ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110012024-02-28T22:17:18.596ZLoading write count information from file "/tmp/test_repair_verify.data"
110022024-02-28T22:17:18.597ZRun Repair workload
110032024-02-28T22:17:18.597Z01/30 Write block 549 len 24576 data: 7 7 6 5 5 4
110042024-02-28T22:17:18.597Z02/30 Read block 376 len 24576
110052024-02-28T22:17:18.597Z03/30 Write block 48 len 28672 data: 7 8 6 7 4 4 4
110062024-02-28T22:17:18.597Z04/30 Write block 229 len 36864 data: 11 10 14 14 13 13 9 8 8
110072024-02-28T22:17:18.597Z05/30 Read block 354 len 24576
110082024-02-28T22:17:18.597Z06/30 Read block 269 len 8192
110092024-02-28T22:17:18.597Z07/30 Write block 312 len 16384 data: 11 9 9 6
110102024-02-28T22:17:18.597Z08/30 Write block 243 len 12288 data: 8 8 8
110112024-02-28T22:17:18.597Z09/30 Read block 550 len 8192
110122024-02-28T22:17:18.607Z10/30 Write block 2 len 28672 data: 5 7 9 9 11 14 15
110132024-02-28T22:17:18.607Z11/30 Read block 352 len 36864
110142024-02-28T22:17:18.607Z12/30 Read block 539 len 20480
110152024-02-28T22:17:18.607Z13/30 Write block 485 len 8192 data: 10 7
110162024-02-28T22:17:18.607Z14/30 Flush
110172024-02-28T22:17:18.612Z15/30 Write block 367 len 12288 data: 5 7 9
110182024-02-28T22:17:18.612Z16/30 Read block 291 len 4096
110192024-02-28T22:17:18.612Z17/30 Write block 248 len 4096 data: 12
110202024-02-28T22:17:18.612Z18/30 Flush
110212024-02-28T22:17:18.612Z19/30 Flush
110222024-02-28T22:17:18.613Z20/30 Write block 405 len 12288 data: 9 9 9
110232024-02-28T22:17:18.613Z21/30 Write block 122 len 8192 data: 11 9
110242024-02-28T22:17:18.613Z22/30 Read block 148 len 32768
110252024-02-28T22:17:18.613Z23/30 Read block 133 len 28672
110262024-02-28T22:17:18.613Z24/30 Write block 347 len 8192 data: 5 7
110272024-02-28T22:17:18.613Z25/30 Write block 465 len 12288 data: 8 6 5
110282024-02-28T22:17:18.617Z26/30 Read block 221 len 8192
110292024-02-28T22:17:18.617Z27/30 Read block 29 len 28672
110302024-02-28T22:17:18.617Z28/30 Read block 56 len 16384
110312024-02-28T22:17:18.617Z29/30 Read block 243 len 20480
110322024-02-28T22:17:18.617Z30/30 Read block 360 len 4096
110332024-02-28T22:17:18.617Z----------------------------------------------------------------
110342024-02-28T22:17:18.617Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110352024-02-28T22:17:18.617ZStates: Active Active Active
110362024-02-28T22:17:18.617ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110372024-02-28T22:17:18.617Z 1 Acked 1000 Write 6 Done Done Done false
110382024-02-28T22:17:18.617Z 2 Acked 1001 Read 6 Done Done Done false
110392024-02-28T22:17:18.617Z 3 Acked 1002 Write 7 Done Done Done false
110402024-02-28T22:17:18.617Z 4 Acked 1003 Write 9 Done Done Done false
110412024-02-28T22:17:18.617Z 5 Acked 1004 Read 6 Done Sent Done false
110422024-02-28T22:17:18.617Z 6 Acked 1005 Read 2 Done Sent Done false
110432024-02-28T22:17:18.618Z 7 Acked 1006 Write 4 Done Sent Done false
110442024-02-28T22:17:18.618Z 8 Acked 1007 Write 3 Done Sent Done false
110452024-02-28T22:17:18.618Z 9 Acked 1008 Read 2 Done Sent Done false
110462024-02-28T22:17:18.618Z 10 Acked 1009 Write 7 Done Sent Done false
110472024-02-28T22:17:18.618Z 11 Acked 1010 Read 9 Done Sent Done false
110482024-02-28T22:17:18.618Z 12 Acked 1011 Read 5 Done Sent Done false
110492024-02-28T22:17:18.618Z 13 Acked 1012 Write 2 Done Sent Done false
110502024-02-28T22:17:18.618Z 14 Acked 1013 Flush 0 Done Sent Done false
110512024-02-28T22:17:18.618Z 15 Acked 1014 Write 3 Done Sent Done false
110522024-02-28T22:17:18.618Z 16 Acked 1015 Read 1 Done Sent Done false
110532024-02-28T22:17:18.618Z 17 Acked 1016 Write 1 Done Sent Done false
110542024-02-28T22:17:18.618Z 18 Acked 1017 Flush 0 Done Sent Done false
110552024-02-28T22:17:18.618Z 19 Acked 1018 Flush 0 Done Sent Done false
110562024-02-28T22:17:18.618Z 20 Acked 1019 Write 3 Done Sent Done false
110572024-02-28T22:17:18.618Z 21 Acked 1020 Write 2 Done Sent Done false
110582024-02-28T22:17:18.618Z 22 Acked 1021 Read 8 Done Sent Done false
110592024-02-28T22:17:18.618Z 23 Acked 1022 Read 7 Done Sent Done false
110602024-02-28T22:17:18.618Z 24 Acked 1023 Write 2 Done Sent Done false
110612024-02-28T22:17:18.618Z 25 Acked 1024 Write 3 Done Sent Done false
110622024-02-28T22:17:18.618Z 26 Acked 1025 Read 2 Done Sent Done false
110632024-02-28T22:17:18.618Z 27 Acked 1026 Read 7 Done Sent Done false
110642024-02-28T22:17:18.618Z 28 Acked 1027 Read 4 Done Sent Done false
110652024-02-28T22:17:18.618Z 29 Acked 1028 Read 5 Done Sent Done false
110662024-02-28T22:17:18.618Z 30 Acked 1029 Read 1 Done Sent Done false
110672024-02-28T22:17:18.619Z STATES DS:0 DS:1 DS:2 TOTAL
110682024-02-28T22:17:18.619Z New 0 0 0 0
110692024-02-28T22:17:18.619Z Sent 0 26 0 26
110702024-02-28T22:17:18.619Z Done 30 4 30 64
110712024-02-28T22:17:18.619Z Skipped 0 0 0 0
110722024-02-28T22:17:18.619Z Error 0 0 0 0
110732024-02-28T22:17:18.619ZLast Flush: 1018 0 1018
110742024-02-28T22:17:18.619ZDownstairs last five completed:
110752024-02-28T22:17:18.619ZUpstairs last five completed: 30 29 28 27 26
110762024-02-28T22:17:18.619Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:17.504938671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1543}
110772024-02-28T22:17:18.619ZWrote out file "/tmp/test_repair_verify.data"
110782024-02-28T22:17:18.622Z
110792024-02-28T22:17:18.626ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
110802024-02-28T22:17:18.639ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110812024-02-28T22:17:18.639Z 0 000-019 102 98 102 185 177 185 F F F <---
110822024-02-28T22:17:18.639Z 1 020-039 99 99 99 180 180 180 F F F
110832024-02-28T22:17:18.639Z 2 040-059 102 101 102 185 184 185 F T F <---
110842024-02-28T22:17:18.639Z 3 060-079 101 101 101 184 184 184 F F F
110852024-02-28T22:17:18.639Z 4 080-099 99 99 99 180 180 180 F F F
110862024-02-28T22:17:18.639Z 5 100-119 101 101 101 184 184 184 F F F
110872024-02-28T22:17:18.639Z 6 120-139 99 99 99 180 180 180 T F T <---
110882024-02-28T22:17:18.639Z 7 140-159 96 96 96 175 175 175 F F F
110892024-02-28T22:17:18.639Z 8 160-179 88 88 88 159 159 159 F F F
110902024-02-28T22:17:18.639Z 9 180-199 97 97 97 176 176 176 F F F
110912024-02-28T22:17:18.639Z 10 200-219 101 101 101 184 184 184 F F F
110922024-02-28T22:17:18.639Z 11 220-239 102 97 102 185 176 185 F T F <---
110932024-02-28T22:17:18.640Z 12 240-259 102 100 102 186 183 186 F F F <---
110942024-02-28T22:17:18.643Z 13 260-279 97 97 97 176 176 176 F F F
110952024-02-28T22:17:18.643Z 14 280-299 90 90 90 163 163 163 F F F
110962024-02-28T22:17:18.643Z 15 300-319 102 94 102 185 171 185 F F F <---
110972024-02-28T22:17:18.643Z 16 320-339 98 98 98 178 178 178 F F F
110982024-02-28T22:17:18.643Z 17 340-359 100 100 100 182 182 182 T F T <---
110992024-02-28T22:17:18.643Z 18 360-379 102 98 102 186 177 186 F F F <---
111002024-02-28T22:17:18.643Z 19 380-399 101 101 101 184 184 184 F F F
111012024-02-28T22:17:18.643Z 20 400-419 99 99 99 180 180 180 T F T <---
111022024-02-28T22:17:18.643Z 21 420-439 99 99 99 180 180 180 F F F
111032024-02-28T22:17:18.644Z 22 440-459 101 101 101 184 184 184 F F F
111042024-02-28T22:17:18.644Z 23 460-479 99 99 99 180 180 180 T F T <---
111052024-02-28T22:17:18.644Z 24 480-499 102 82 102 185 149 185 F F F <---
111062024-02-28T22:17:18.644Z 25 500-519 94 94 94 171 171 171 F F F
111072024-02-28T22:17:18.644Z 26 520-539 94 94 94 171 171 171 F F F
111082024-02-28T22:17:18.644Z 27 540-559 102 96 102 185 175 185 F T F <---
111092024-02-28T22:17:18.644Z 28 560-579 101 101 101 184 184 184 F F F
111102024-02-28T22:17:18.644Z 29 580-599 92 92 92 166 166 166 F F F
111112024-02-28T22:17:18.644ZMax gen: 102, Max flush: 186
111122024-02-28T22:17:18.644ZError: Difference in extent metadata found!
111132024-02-28T22:17:18.644ZA Difference in extent metadata is expected here
111142024-02-28T22:17:18.644ZOn loop 51
111152024-02-28T22:17:18.644Z
111162024-02-28T22:17:18.648ZVerifying data now
111172024-02-28T22:17:24.267ZNo change after verify
111182024-02-28T22:17:24.268ZLoop: 51 Downstairs dump after verify (and repair):
111192024-02-28T22:17:24.282ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111202024-02-28T22:17:24.282Z 0 000-019 102 102 102 185 185 185 F F F
111212024-02-28T22:17:24.282Z 1 020-039 99 99 99 180 180 180 F F F
111222024-02-28T22:17:24.282Z 2 040-059 102 102 102 185 185 185 F F F
111232024-02-28T22:17:24.282Z 3 060-079 101 101 101 184 184 184 F F F
111242024-02-28T22:17:24.282Z 4 080-099 99 99 99 180 180 180 F F F
111252024-02-28T22:17:24.282Z 5 100-119 101 101 101 184 184 184 F F F
111262024-02-28T22:17:24.282Z 6 120-139 103 103 103 187 187 187 F F F
111272024-02-28T22:17:24.282Z 7 140-159 96 96 96 175 175 175 F F F
111282024-02-28T22:17:24.282Z 8 160-179 88 88 88 159 159 159 F F F
111292024-02-28T22:17:24.282Z 9 180-199 97 97 97 176 176 176 F F F
111302024-02-28T22:17:24.282Z 10 200-219 101 101 101 184 184 184 F F F
111312024-02-28T22:17:24.282Z 11 220-239 102 102 102 185 185 185 F F F
111322024-02-28T22:17:24.282Z 12 240-259 102 102 102 186 186 186 F F F
111332024-02-28T22:17:24.282Z 13 260-279 97 97 97 176 176 176 F F F
111342024-02-28T22:17:24.282Z 14 280-299 90 90 90 163 163 163 F F F
111352024-02-28T22:17:24.282Z 15 300-319 102 102 102 185 185 185 F F F
111362024-02-28T22:17:24.282Z 16 320-339 98 98 98 178 178 178 F F F
111372024-02-28T22:17:24.282Z 17 340-359 103 103 103 187 187 187 F F F
111382024-02-28T22:17:24.286Z 18 360-379 102 102 102 186 186 186 F F F
111392024-02-28T22:17:24.286Z 19 380-399 101 101 101 184 184 184 F F F
111402024-02-28T22:17:24.286Z 20 400-419 103 103 103 187 187 187 F F F
111412024-02-28T22:17:24.286Z 21 420-439 99 99 99 180 180 180 F F F
111422024-02-28T22:17:24.286Z 22 440-459 101 101 101 184 184 184 F F F
111432024-02-28T22:17:24.286Z 23 460-479 103 103 103 187 187 187 F F F
111442024-02-28T22:17:24.286Z 24 480-499 102 102 102 185 185 185 F F F
111452024-02-28T22:17:24.286Z 25 500-519 94 94 94 171 171 171 F F F
111462024-02-28T22:17:24.286Z 26 520-539 94 94 94 171 171 171 F F F
111472024-02-28T22:17:24.286Z 27 540-559 102 102 102 185 185 185 F F F
111482024-02-28T22:17:24.287Z 28 560-579 101 101 101 184 184 184 F F F
111492024-02-28T22:17:24.287Z 29 580-599 92 92 92 166 166 166 F F F
111502024-02-28T22:17:24.287ZMax gen: 103, Max flush: 187
111512024-02-28T22:17:24.287Z
111522024-02-28T22:17:24.287ZBegin loop 52
111532024-02-28T22:17:24.287ZDownstairs to restart: 1
111542024-02-28T22:17:24.303Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192321889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
111552024-02-28T22:17:24.303Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192588172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
111562024-02-28T22:17:24.303Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192606251Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
111572024-02-28T22:17:24.310Z{"msg":"Crucible 78911f2c-138c-47f8-a1f6-84b31cd8654e has session id: d90db966-99b0-44f6-b4a4-be546801593e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192629561Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111582024-02-28T22:17:24.310Z{"msg":"Crucible runtime is spawned
111592024-02-28T22:17:24.310ZUpstairs opts: Upstairs UUID: 78911f2c-138c-47f8-a1f6-84b31cd8654e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192678639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111602024-02-28T22:17:24.310Z{"msg":"Crucible stats registered with UUID: 78911f2c-138c-47f8-a1f6-84b31cd8654e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192704149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111612024-02-28T22:17:24.310Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192735628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
111622024-02-28T22:17:24.310Z{"msg":"78911f2c-138c-47f8-a1f6-84b31cd8654e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192922192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111632024-02-28T22:17:24.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192961402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111642024-02-28T22:17:24.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.192987731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111652024-02-28T22:17:24.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193030029Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111662024-02-28T22:17:24.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193069308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111672024-02-28T22:17:24.310Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193089478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111682024-02-28T22:17:24.310Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193112777Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111692024-02-28T22:17:24.310Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193136397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111702024-02-28T22:17:24.310Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193180036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111712024-02-28T22:17:24.311Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193207775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111722024-02-28T22:17:24.311Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193295282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111732024-02-28T22:17:24.311Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193394199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111742024-02-28T22:17:24.311Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.193426429Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111752024-02-28T22:17:24.311Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194045972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111762024-02-28T22:17:24.311Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.19408632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111772024-02-28T22:17:24.311Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.19411963Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111782024-02-28T22:17:24.311Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194157058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111792024-02-28T22:17:24.311Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194187338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111802024-02-28T22:17:24.311Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194223627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111812024-02-28T22:17:24.311Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194593386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111822024-02-28T22:17:24.311Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194628645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111832024-02-28T22:17:24.311Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.194889218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111842024-02-28T22:17:24.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195113672Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111852024-02-28T22:17:24.311Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195150981Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111862024-02-28T22:17:24.311Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195167591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111872024-02-28T22:17:24.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.19518832Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111882024-02-28T22:17:24.311Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.19520305Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111892024-02-28T22:17:24.311Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195223819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111902024-02-28T22:17:24.311Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195375635Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111912024-02-28T22:17:24.311Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195395615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111922024-02-28T22:17:24.311Z{"msg":"[0]R flush_numbers[0..12]: [185, 180, 185, 184, 180, 184, 187, 175, 159, 176, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195410424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111932024-02-28T22:17:24.312Z{"msg":"[0]R generation[0..12]: [102, 99, 102, 101, 99, 101, 103, 96, 88, 97, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195424904Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111942024-02-28T22:17:24.312Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195448313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111952024-02-28T22:17:24.312Z{"msg":"[1]R flush_numbers[0..12]: [185, 180, 185, 184, 180, 184, 187, 175, 159, 176, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195463622Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111962024-02-28T22:17:24.312Z{"msg":"[1]R generation[0..12]: [102, 99, 102, 101, 99, 101, 103, 96, 88, 97, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195485422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111972024-02-28T22:17:24.312Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195500061Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111982024-02-28T22:17:24.312Z{"msg":"[2]R flush_numbers[0..12]: [185, 180, 185, 184, 180, 184, 187, 175, 159, 176, 184, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195520621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
111992024-02-28T22:17:24.312Z{"msg":"[2]R generation[0..12]: [102, 99, 102, 101, 99, 101, 103, 96, 88, 97, 101, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195534661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112002024-02-28T22:17:24.312ZWait for a query_work_queue command to finish before sending IO
112012024-02-28T22:17:24.312Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195590039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112022024-02-28T22:17:24.312Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195616118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112032024-02-28T22:17:24.312Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195630418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112042024-02-28T22:17:24.312Z{"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195654117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112052024-02-28T22:17:24.312Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195679936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"":"downstairs","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112062024-02-28T22:17:24.312Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195694326Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112072024-02-28T22:17:24.312Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195712876Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"0","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112082024-02-28T22:17:24.312Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195726665Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"1","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112092024-02-28T22:17:24.312Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195746295Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"client":"2","":"downstairs","session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112102024-02-28T22:17:24.312Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195760034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112112024-02-28T22:17:24.312Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195779064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112122024-02-28T22:17:24.312Z{"msg":"78911f2c-138c-47f8-a1f6-84b31cd8654e is now active with session: d90db966-99b0-44f6-b4a4-be546801593e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195792533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112132024-02-28T22:17:24.312Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195805433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551,"session_id":"d90db966-99b0-44f6-b4a4-be546801593e"}
112142024-02-28T22:17:24.313Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:23.195817863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
112152024-02-28T22:17:24.313ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112162024-02-28T22:17:24.313ZLoading write count information from file "/tmp/test_repair_verify.data"
112172024-02-28T22:17:24.313ZRun Repair workload
112182024-02-28T22:17:24.316Z01/30 Write block 395 len 12288 data: 9 8 8
112192024-02-28T22:17:24.316Z02/30 Write block 499 len 36864 data: 8 9 10 10 12 13 12 13 11
112202024-02-28T22:17:24.316Z03/30 Write block 416 len 4096 data: 11
112212024-02-28T22:17:24.317Z04/30 Read block 442 len 36864
112222024-02-28T22:17:24.317Z05/30 Read block 166 len 28672
112232024-02-28T22:17:24.317Z06/30 Write block 278 len 36864 data: 5 6 6 5 6 5 5 4 6
112242024-02-28T22:17:24.317Z07/30 Read block 302 len 36864
112252024-02-28T22:17:24.317Z08/30 Read block 146 len 16384
112262024-02-28T22:17:24.317Z09/30 Read block 35 len 8192
112272024-02-28T22:17:24.317Z10/30 Write block 173 len 20480 data: 12 12 10 9 8
112282024-02-28T22:17:24.317Z11/30 Read block 262 len 4096
112292024-02-28T22:17:24.321Z12/30 Read block 365 len 4096
112302024-02-28T22:17:24.321Z13/30 Read block 38 len 4096
112312024-02-28T22:17:24.321Z14/30 Write block 36 len 16384 data: 9 10 11 10
112322024-02-28T22:17:24.321Z15/30 Read block 542 len 12288
112332024-02-28T22:17:24.321Z16/30 Write block 322 len 4096 data: 7
112342024-02-28T22:17:24.321Z17/30 Read block 321 len 28672
112352024-02-28T22:17:24.324Z18/30 Read block 308 len 12288
112362024-02-28T22:17:24.324Z19/30 Read block 103 len 32768
112372024-02-28T22:17:24.324Z20/30 Write block 99 len 16384 data: 7 8 7 7
112382024-02-28T22:17:24.324Z21/30 Read block 212 len 40960
112392024-02-28T22:17:24.327Z22/30 Flush
112402024-02-28T22:17:24.330Z23/30 Write block 458 len 28672 data: 10 9 6 6 6 7 7
112412024-02-28T22:17:24.330Z24/30 Write block 302 len 16384 data: 6 9 9 8
112422024-02-28T22:17:24.330Z25/30 Read block 156 len 32768
112432024-02-28T22:17:24.334Z26/30 Write block 273 len 8192 data: 3 2
112442024-02-28T22:17:24.334Z27/30 Read block 213 len 40960
112452024-02-28T22:17:24.334Z28/30 Write block 212 len 12288 data: 9 8 7
112462024-02-28T22:17:24.334Z29/30 Read block 363 len 36864
112472024-02-28T22:17:24.338Z30/30 Read block 19 len 16384
112482024-02-28T22:17:24.338Z----------------------------------------------------------------
112492024-02-28T22:17:24.338Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
112502024-02-28T22:17:24.338ZStates: Active Active Active
112512024-02-28T22:17:24.338ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112522024-02-28T22:17:24.338Z 1 Acked 1000 Write 3 Done Done Done false
112532024-02-28T22:17:24.338Z 2 Acked 1001 Write 9 Done Done Done false
112542024-02-28T22:17:24.338Z 3 Acked 1002 Write 1 Done Done Done false
112552024-02-28T22:17:24.338Z 4 Acked 1003 Read 9 Done Sent Done false
112562024-02-28T22:17:24.339Z 5 Acked 1004 Read 7 Done Sent Done false
112572024-02-28T22:17:24.339Z 6 Acked 1005 Write 9 Done Sent Done false
112582024-02-28T22:17:24.339Z 7 Acked 1006 Read 9 Done Sent Done false
112592024-02-28T22:17:24.339Z 8 Acked 1007 Read 4 Done Sent Done false
112602024-02-28T22:17:24.339Z 9 Acked 1008 Read 2 Done Sent Done false
112612024-02-28T22:17:24.339Z 10 Acked 1009 Write 5 Done Sent Done false
112622024-02-28T22:17:24.339Z 11 Acked 1010 Read 1 Done Sent Done false
112632024-02-28T22:17:24.339Z 12 Acked 1011 Read 1 Done Sent Done false
112642024-02-28T22:17:24.339Z 13 Acked 1012 Read 1 Done Sent Done false
112652024-02-28T22:17:24.339Z 14 Acked 1013 Write 4 Done Sent Done false
112662024-02-28T22:17:24.339Z 15 Acked 1014 Read 3 Done Sent Done false
112672024-02-28T22:17:24.339Z 16 Acked 1015 Write 1 Done Sent Done false
112682024-02-28T22:17:24.339Z 17 Acked 1016 Read 7 Done Sent Done false
112692024-02-28T22:17:24.339Z 18 Acked 1017 Read 3 Done Sent Done false
112702024-02-28T22:17:24.339Z 19 Acked 1018 Read 8 Done Sent Done false
112712024-02-28T22:17:24.339Z 20 Acked 1019 Write 4 Done Sent Done false
112722024-02-28T22:17:24.339Z 21 Acked 1020 Read 10 Done Sent Done false
112732024-02-28T22:17:24.339Z 22 Acked 1021 Flush 0 Done Sent Done false
112742024-02-28T22:17:24.339Z 23 Acked 1022 Write 7 Done Sent Done false
112752024-02-28T22:17:24.339Z 24 Acked 1023 Write 4 Done Sent Done false
112762024-02-28T22:17:24.339Z 25 Acked 1024 Read 8 Done Sent Done false
112772024-02-28T22:17:24.340Z 26 Acked 1025 Write 2 Done Sent Done false
112782024-02-28T22:17:24.340Z 27 Acked 1026 Read 10 Done Sent Done false
112792024-02-28T22:17:24.340Z 28 Acked 1027 Write 3 Done Sent Done false
112802024-02-28T22:17:24.340Z 29 Acked 1028 Read 9 Done Sent Done false
112812024-02-28T22:17:24.340Z 30 Acked 1029 Read 4 Done Sent Done false
112822024-02-28T22:17:24.340Z STATES DS:0 DS:1 DS:2 TOTAL
112832024-02-28T22:17:24.340Z New 0 0 0 0
112842024-02-28T22:17:24.340Z Sent 0 27 0 27
112852024-02-28T22:17:24.340Z Done 30 3 30 63
112862024-02-28T22:17:24.340Z Skipped 0 0 0 0
112872024-02-28T22:17:24.340Z Error 0 0 0 0
112882024-02-28T22:17:24.340ZLast Flush: 1021 0 1021
112892024-02-28T22:17:24.340ZDownstairs last five completed:
112902024-02-28T22:17:24.340ZUpstairs last five completed: 30 29 28 27 26
112912024-02-28T22:17:24.340Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:23.225401408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1551}
112922024-02-28T22:17:24.340ZWrote out file "/tmp/test_repair_verify.data"
112932024-02-28T22:17:24.344Z
112942024-02-28T22:17:24.344ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112952024-02-28T22:17:24.360ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112962024-02-28T22:17:24.360Z 0 000-019 102 102 102 185 185 185 F F F
112972024-02-28T22:17:24.360Z 1 020-039 104 99 104 188 180 188 F F F <---
112982024-02-28T22:17:24.360Z 2 040-059 102 102 102 185 185 185 F F F
112992024-02-28T22:17:24.360Z 3 060-079 101 101 101 184 184 184 F F F
113002024-02-28T22:17:24.360Z 4 080-099 104 99 104 188 180 188 F F F <---
113012024-02-28T22:17:24.360Z 5 100-119 104 101 104 188 184 188 F F F <---
113022024-02-28T22:17:24.363Z 6 120-139 103 103 103 187 187 187 F F F
113032024-02-28T22:17:24.363Z 7 140-159 96 96 96 175 175 175 F F F
113042024-02-28T22:17:24.363Z 8 160-179 104 88 104 188 159 188 F F F <---
113052024-02-28T22:17:24.364Z 9 180-199 97 97 97 176 176 176 F F F
113062024-02-28T22:17:24.364Z 10 200-219 101 101 101 184 184 184 T F T <---
113072024-02-28T22:17:24.364Z 11 220-239 102 102 102 185 185 185 F F F
113082024-02-28T22:17:24.364Z 12 240-259 102 102 102 186 186 186 F F F
113092024-02-28T22:17:24.364Z 13 260-279 104 97 104 188 176 188 T F T <---
113102024-02-28T22:17:24.364Z 14 280-299 104 90 104 188 163 188 F F F <---
113112024-02-28T22:17:24.364Z 15 300-319 102 102 102 185 185 185 T F T <---
113122024-02-28T22:17:24.364Z 16 320-339 104 98 104 188 178 188 F F F <---
113132024-02-28T22:17:24.364Z 17 340-359 103 103 103 187 187 187 F F F
113142024-02-28T22:17:24.364Z 18 360-379 102 102 102 186 186 186 F F F
113152024-02-28T22:17:24.364Z 19 380-399 104 101 104 188 184 188 F T F <---
113162024-02-28T22:17:24.364Z 20 400-419 104 103 104 188 187 188 F T F <---
113172024-02-28T22:17:24.364Z 21 420-439 99 99 99 180 180 180 F F F
113182024-02-28T22:17:24.364Z 22 440-459 101 101 101 184 184 184 T F T <---
113192024-02-28T22:17:24.364Z 23 460-479 103 103 103 187 187 187 T F T <---
113202024-02-28T22:17:24.364Z 24 480-499 104 102 104 188 185 188 F T F <---
113212024-02-28T22:17:24.364Z 25 500-519 104 94 104 188 171 188 F T F <---
113222024-02-28T22:17:24.364Z 26 520-539 94 94 94 171 171 171 F F F
113232024-02-28T22:17:24.364Z 27 540-559 102 102 102 185 185 185 F F F
113242024-02-28T22:17:24.364Z 28 560-579 101 101 101 184 184 184 F F F
113252024-02-28T22:17:24.364Z 29 580-599 92 92 92 166 166 166 F F F
113262024-02-28T22:17:24.364ZMax gen: 104, Max flush: 188
113272024-02-28T22:17:24.365ZError: Difference in extent metadata found!
113282024-02-28T22:17:24.365ZA Difference in extent metadata is expected here
113292024-02-28T22:17:24.365ZOn loop 52
113302024-02-28T22:17:24.365Z
113312024-02-28T22:17:24.368ZVerifying data now
113322024-02-28T22:17:30.108ZNo change after verify
113332024-02-28T22:17:30.108ZLoop: 52 Downstairs dump after verify (and repair):
113342024-02-28T22:17:30.122ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113352024-02-28T22:17:30.122Z 0 000-019 102 102 102 185 185 185 F F F
113362024-02-28T22:17:30.122Z 1 020-039 104 104 104 188 188 188 F F F
113372024-02-28T22:17:30.122Z 2 040-059 102 102 102 185 185 185 F F F
113382024-02-28T22:17:30.122Z 3 060-079 101 101 101 184 184 184 F F F
113392024-02-28T22:17:30.122Z 4 080-099 104 104 104 188 188 188 F F F
113402024-02-28T22:17:30.122Z 5 100-119 104 104 104 188 188 188 F F F
113412024-02-28T22:17:30.122Z 6 120-139 103 103 103 187 187 187 F F F
113422024-02-28T22:17:30.122Z 7 140-159 96 96 96 175 175 175 F F F
113432024-02-28T22:17:30.122Z 8 160-179 104 104 104 188 188 188 F F F
113442024-02-28T22:17:30.122Z 9 180-199 97 97 97 176 176 176 F F F
113452024-02-28T22:17:30.136Z 10 200-219 105 105 105 189 189 189 F F F
113462024-02-28T22:17:30.136Z 11 220-239 102 102 102 185 185 185 F F F
113472024-02-28T22:17:30.136Z 12 240-259 102 102 102 186 186 186 F F F
113482024-02-28T22:17:30.136Z 13 260-279 105 105 105 189 189 189 F F F
113492024-02-28T22:17:30.136Z 14 280-299 104 104 104 188 188 188 F F F
113502024-02-28T22:17:30.136Z 15 300-319 105 105 105 189 189 189 F F F
113512024-02-28T22:17:30.137Z 16 320-339 104 104 104 188 188 188 F F F
113522024-02-28T22:17:30.137Z 17 340-359 103 103 103 187 187 187 F F F
113532024-02-28T22:17:30.137Z 18 360-379 102 102 102 186 186 186 F F F
113542024-02-28T22:17:30.137Z 19 380-399 104 104 104 188 188 188 F F F
113552024-02-28T22:17:30.137Z 20 400-419 104 104 104 188 188 188 F F F
113562024-02-28T22:17:30.137Z 21 420-439 99 99 99 180 180 180 F F F
113572024-02-28T22:17:30.137Z 22 440-459 105 105 105 189 189 189 F F F
113582024-02-28T22:17:30.137Z 23 460-479 105 105 105 189 189 189 F F F
113592024-02-28T22:17:30.137Z 24 480-499 104 104 104 188 188 188 F F F
113602024-02-28T22:17:30.137Z 25 500-519 104 104 104 188 188 188 F F F
113612024-02-28T22:17:30.137Z 26 520-539 94 94 94 171 171 171 F F F
113622024-02-28T22:17:30.137Z 27 540-559 102 102 102 185 185 185 F F F
113632024-02-28T22:17:30.137Z 28 560-579 101 101 101 184 184 184 F F F
113642024-02-28T22:17:30.137Z 29 580-599 92 92 92 166 166 166 F F F
113652024-02-28T22:17:30.137ZMax gen: 105, Max flush: 189
113662024-02-28T22:17:30.137Z
113672024-02-28T22:17:30.137ZBegin loop 53
113682024-02-28T22:17:30.137ZDownstairs to restart: 1
113692024-02-28T22:17:30.142Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031303091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
113702024-02-28T22:17:30.142ZCrucible runtime is spawned
113712024-02-28T22:17:30.149Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031649281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
113722024-02-28T22:17:30.149Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03169542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
113732024-02-28T22:17:30.149Z{"msg":"Crucible a97a623d-0b72-4cc6-8707-13eeba4db80d has session id: 5c1e882b-3d08-4618-9130-472cf33d48b6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031764608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113742024-02-28T22:17:30.149Z{"msg":"Upstairs opts: Upstairs UUID: a97a623d-0b72-4cc6-8707-13eeba4db80d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031820677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113752024-02-28T22:17:30.149Z{"msg":"Crucible stats registered with UUID: a97a623d-0b72-4cc6-8707-13eeba4db80d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031860786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113762024-02-28T22:17:30.149Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031896805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
113772024-02-28T22:17:30.149Z{"msg":"a97a623d-0b72-4cc6-8707-13eeba4db80d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031938313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113782024-02-28T22:17:30.149Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.031964713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113792024-02-28T22:17:30.149Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032017681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113802024-02-28T22:17:30.149Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03205211Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113812024-02-28T22:17:30.149Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03207068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113822024-02-28T22:17:30.150Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032109599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113832024-02-28T22:17:30.150Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032128698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113842024-02-28T22:17:30.150Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032169027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113852024-02-28T22:17:30.150Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032222396Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113862024-02-28T22:17:30.150Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032258775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113872024-02-28T22:17:30.150Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032300403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113882024-02-28T22:17:30.150Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032334533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113892024-02-28T22:17:30.150Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.032378501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113902024-02-28T22:17:30.150Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033046893Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113912024-02-28T22:17:30.150Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033087772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113922024-02-28T22:17:30.150Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033237198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113932024-02-28T22:17:30.150Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033288536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113942024-02-28T22:17:30.150Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033307176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113952024-02-28T22:17:30.150Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033368744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113962024-02-28T22:17:30.150Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033600458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113972024-02-28T22:17:30.150Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.033962218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113982024-02-28T22:17:30.150Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034011536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
113992024-02-28T22:17:30.150Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034192571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114002024-02-28T22:17:30.150Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03423004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114012024-02-28T22:17:30.150Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03424688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114022024-02-28T22:17:30.151Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034445914Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114032024-02-28T22:17:30.151Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034506113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114042024-02-28T22:17:30.151Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034545892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114052024-02-28T22:17:30.151Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034561281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114062024-02-28T22:17:30.151Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034577071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114072024-02-28T22:17:30.151Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 185, 184, 188, 188, 187, 175, 188, 176, 189, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.0345923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114082024-02-28T22:17:30.151Z{"msg":"[0]R generation[0..12]: [102, 104, 102, 101, 104, 104, 103, 96, 104, 97, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03461441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114092024-02-28T22:17:30.151Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034630459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114102024-02-28T22:17:30.151Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 185, 184, 188, 188, 187, 175, 188, 176, 189, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034652049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114112024-02-28T22:17:30.151Z{"msg":"[1]R generation[0..12]: [102, 104, 102, 101, 104, 104, 103, 96, 104, 97, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034666628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114122024-02-28T22:17:30.151Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034688168Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
114132024-02-28T22:17:30.151Zsession_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114142024-02-28T22:17:30.151Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 185, 184, 188, 188, 187, 175, 188, 176, 189, 185]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034707627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114152024-02-28T22:17:30.151Z{"msg":"[2]R generation[0..12]: [102, 104, 102, 101, 104, 104, 103, 96, 104, 97, 105, 102]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034729056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114162024-02-28T22:17:30.151Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034743736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114172024-02-28T22:17:30.151Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034778385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114182024-02-28T22:17:30.151Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034793145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114192024-02-28T22:17:30.151Z{"msg":"Next flush: 190","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034813944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114202024-02-28T22:17:30.151Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034828244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"":"downstairs","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114212024-02-28T22:17:30.151Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034842793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114222024-02-28T22:17:30.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034855573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"0","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114232024-02-28T22:17:30.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034881582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"1","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114242024-02-28T22:17:30.151Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034895652Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"client":"2","":"downstairs","session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114252024-02-28T22:17:30.151Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034916322Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114262024-02-28T22:17:30.152Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034929921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114272024-02-28T22:17:30.152Z{"msg":"a97a623d-0b72-4cc6-8707-13eeba4db80d is now active with session: 5c1e882b-3d08-4618-9130-472cf33d48b6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.034942991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114282024-02-28T22:17:30.152Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03496306Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559,"session_id":"5c1e882b-3d08-4618-9130-472cf33d48b6"}
114292024-02-28T22:17:30.152Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:29.03497609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
114302024-02-28T22:17:30.152ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114312024-02-28T22:17:30.156ZLoading write count information from file "/tmp/test_repair_verify.data"
114322024-02-28T22:17:30.156ZRun Repair workload
114332024-02-28T22:17:30.156Z01/30 Write block 237 len 4096 data: 9
114342024-02-28T22:17:30.156Z02/30 Write block 342 len 24576 data: 10 9 7 7 6 6
114352024-02-28T22:17:30.156Z03/30 Flush
114362024-02-28T22:17:30.156Z04/30 Write block 517 len 24576 data: 8 7 6 7 6 5
114372024-02-28T22:17:30.156Z05/30 Read block 143 len 16384
114382024-02-28T22:17:30.156Z06/30 Write block 513 len 36864 data: 9 8 9 9 9 8 7 8 7
114392024-02-28T22:17:30.156Z07/30 Read block 225 len 24576
114402024-02-28T22:17:30.156Z08/30 Write block 266 len 12288 data: 9 8 7
114412024-02-28T22:17:30.156Z09/30 Read block 354 len 4096
114422024-02-28T22:17:30.161Z10/30 Flush
114432024-02-28T22:17:30.161Z11/30 Write block 91 len 16384 data: 8 10 7 7
114442024-02-28T22:17:30.161Z12/30 Write block 583 len 12288 data: 4 4 6
114452024-02-28T22:17:30.161Z13/30 Read block 396 len 16384
114462024-02-28T22:17:30.161Z14/30 Flush
114472024-02-28T22:17:30.165Z15/30 Write block 552 len 8192 data: 6 6
114482024-02-28T22:17:30.165Z16/30 Read block 384 len 24576
114492024-02-28T22:17:30.165Z17/30 Read block 437 len 12288
114502024-02-28T22:17:30.168Z18/30 Flush
114512024-02-28T22:17:30.168Z19/30 Write block 374 len 16384 data: 10 9 9 9
114522024-02-28T22:17:30.171Z20/30 Write block 519 len 4096 data: 8
114532024-02-28T22:17:30.171Z21/30 Read block 284 len 16384
114542024-02-28T22:17:30.172Z22/30 Read block 592 len 24576
114552024-02-28T22:17:30.172Z23/30 Write block 500 len 40960 data: 10 11 11 13 14 13 14 12 11 13
114562024-02-28T22:17:30.175Z24/30 Read block 230 len 36864
114572024-02-28T22:17:30.175Z25/30 Read block 121 len 24576
114582024-02-28T22:17:30.175Z26/30 Read block 41 len 40960
114592024-02-28T22:17:30.175Z27/30 Read block 291 len 40960
114602024-02-28T22:17:30.179Z28/30 Write block 363 len 32768 data: 7 5 5 4 6 8 10 11
114612024-02-28T22:17:30.180Z29/30 Write block 422 len 8192 data: 13 11
114622024-02-28T22:17:30.180Z30/30 Read block 579 len 32768
114632024-02-28T22:17:30.180Z----------------------------------------------------------------
114642024-02-28T22:17:30.180Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:27
114652024-02-28T22:17:30.180ZStates: Active Active Active
114662024-02-28T22:17:30.180ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114672024-02-28T22:17:30.180Z 4 Acked 1003 Write 6 Done Done Done false
114682024-02-28T22:17:30.180Z 5 Acked 1004 Read 4 Done Done Done false
114692024-02-28T22:17:30.180Z 6 Acked 1005 Write 9 Done Sent Done false
114702024-02-28T22:17:30.180Z 7 Acked 1006 Read 6 Done Sent Done false
114712024-02-28T22:17:30.180Z 8 Acked 1007 Write 3 Done Sent Done false
114722024-02-28T22:17:30.180Z 9 Acked 1008 Read 1 Done Sent Done false
114732024-02-28T22:17:30.180Z 10 Acked 1009 Flush 0 Done Sent Done false
114742024-02-28T22:17:30.181Z 11 Acked 1010 Write 4 Done Sent Done false
114752024-02-28T22:17:30.181Z 12 Acked 1011 Write 3 Done Sent Done false
114762024-02-28T22:17:30.181Z 13 Acked 1012 Read 4 Done Sent Done false
114772024-02-28T22:17:30.181Z 14 Acked 1013 Flush 0 Done Sent Done false
114782024-02-28T22:17:30.181Z 15 Acked 1014 Write 2 Done Sent Done false
114792024-02-28T22:17:30.181Z 16 Acked 1015 Read 6 Done Sent Done false
114802024-02-28T22:17:30.181Z 17 Acked 1016 Read 3 Done Sent Done false
114812024-02-28T22:17:30.181Z 18 Acked 1017 Flush 0 Done Sent Done false
114822024-02-28T22:17:30.181Z 19 Acked 1018 Write 4 Done Sent Done false
114832024-02-28T22:17:30.181Z 20 Acked 1019 Write 1 Done Sent Done false
114842024-02-28T22:17:30.181Z 21 Acked 1020 Read 4 Done Sent Done false
114852024-02-28T22:17:30.181Z 22 Acked 1021 Read 6 Done Sent Done false
114862024-02-28T22:17:30.182Z 23 Acked 1022 Write 10 Done Sent Done false
114872024-02-28T22:17:30.182Z 24 Acked 1023 Read 9 Done Sent Done false
114882024-02-28T22:17:30.182Z 25 Acked 1024 Read 6 Done Sent Done false
114892024-02-28T22:17:30.182Z 26 Acked 1025 Read 10 Done Sent Done false
114902024-02-28T22:17:30.182Z 27 Acked 1026 Read 10 Done Sent Done false
114912024-02-28T22:17:30.182Z 28 Acked 1027 Write 8 Done Sent Done false
114922024-02-28T22:17:30.182Z 29 Acked 1028 Write 2 Done Sent Done false
114932024-02-28T22:17:30.182Z 30 Acked 1029 Read 8 Done Sent Sent false
114942024-02-28T22:17:30.182Z STATES DS:0 DS:1 DS:2 TOTAL
114952024-02-28T22:17:30.182Z New 0 0 0 0
114962024-02-28T22:17:30.182Z Sent 0 25 1 26
114972024-02-28T22:17:30.182Z Done 27 2 26 55
114982024-02-28T22:17:30.182Z Skipped 0 0 0 0
114992024-02-28T22:17:30.182Z Error 0 0 0 0
115002024-02-28T22:17:30.183ZLast Flush: 1017 1002 1017
115012024-02-28T22:17:30.183ZDownstairs last five completed: 1002 1001 1000
115022024-02-28T22:17:30.183ZUpstairs last five completed: 30 29 28 27 26
115032024-02-28T22:17:30.188Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:29.068435628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1559}
115042024-02-28T22:17:30.189ZWrote out file "/tmp/test_repair_verify.data"
115052024-02-28T22:17:30.189Z
115062024-02-28T22:17:30.192ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115072024-02-28T22:17:30.204ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115082024-02-28T22:17:30.204Z 0 000-019 102 102 102 185 185 185 F F F
115092024-02-28T22:17:30.204Z 1 020-039 104 104 104 188 188 188 F F F
115102024-02-28T22:17:30.204Z 2 040-059 102 102 102 185 185 185 F F F
115112024-02-28T22:17:30.205Z 3 060-079 101 101 101 184 184 184 F F F
115122024-02-28T22:17:30.205Z 4 080-099 106 104 106 192 188 192 F F F <---
115132024-02-28T22:17:30.205Z 5 100-119 104 104 104 188 188 188 F F F
115142024-02-28T22:17:30.208Z 6 120-139 103 103 103 187 187 187 F F F
115152024-02-28T22:17:30.208Z 7 140-159 96 96 96 175 175 175 F F F
115162024-02-28T22:17:30.208Z 8 160-179 104 104 104 188 188 188 F F F
115172024-02-28T22:17:30.208Z 9 180-199 97 97 97 176 176 176 F F F
115182024-02-28T22:17:30.209Z 10 200-219 105 105 105 189 189 189 F F F
115192024-02-28T22:17:30.209Z 11 220-239 106 106 106 190 190 190 F F F
115202024-02-28T22:17:30.209Z 12 240-259 102 102 102 186 186 186 F F F
115212024-02-28T22:17:30.209Z 13 260-279 106 105 106 191 189 191 F F F <---
115222024-02-28T22:17:30.209Z 14 280-299 104 104 104 188 188 188 F F F
115232024-02-28T22:17:30.209Z 15 300-319 105 105 105 189 189 189 F F F
115242024-02-28T22:17:30.209Z 16 320-339 104 104 104 188 188 188 F F F
115252024-02-28T22:17:30.209Z 17 340-359 106 106 106 190 190 190 F F F
115262024-02-28T22:17:30.209Z 18 360-379 102 102 102 186 186 186 T F T <---
115272024-02-28T22:17:30.209Z 19 380-399 104 104 104 188 188 188 F F F
115282024-02-28T22:17:30.209Z 20 400-419 104 104 104 188 188 188 F F F
115292024-02-28T22:17:30.209Z 21 420-439 99 99 99 180 180 180 T F T <---
115302024-02-28T22:17:30.209Z 22 440-459 105 105 105 189 189 189 F F F
115312024-02-28T22:17:30.210Z 23 460-479 105 105 105 189 189 189 F F F
115322024-02-28T22:17:30.210Z 24 480-499 104 104 104 188 188 188 F F F
115332024-02-28T22:17:30.210Z 25 500-519 106 104 106 191 188 191 T T T <---
115342024-02-28T22:17:30.210Z 26 520-539 106 94 106 191 171 191 F T F <---
115352024-02-28T22:17:30.210Z 27 540-559 106 102 106 193 185 193 F F F <---
115362024-02-28T22:17:30.210Z 28 560-579 101 101 101 184 184 184 F F F
115372024-02-28T22:17:30.210Z 29 580-599 106 92 106 192 166 192 F F F <---
115382024-02-28T22:17:30.210ZMax gen: 106, Max flush: 193
115392024-02-28T22:17:30.210ZError: Difference in extent metadata found!
115402024-02-28T22:17:30.210ZA Difference in extent metadata is expected here
115412024-02-28T22:17:30.210ZOn loop 53
115422024-02-28T22:17:30.210Z
115432024-02-28T22:17:30.214ZVerifying data now
115442024-02-28T22:17:34.662ZNo change after verify
115452024-02-28T22:17:34.662ZLoop: 53 Downstairs dump after verify (and repair):
115462024-02-28T22:17:34.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115472024-02-28T22:17:34.674Z 0 000-019 102 102 102 185 185 185 F F F
115482024-02-28T22:17:34.674Z 1 020-039 104 104 104 188 188 188 F F F
115492024-02-28T22:17:34.675Z 2 040-059 102 102 102 185 185 185 F F F
115502024-02-28T22:17:34.675Z 3 060-079 101 101 101 184 184 184 F F F
115512024-02-28T22:17:34.675Z 4 080-099 106 106 106 192 192 192 F F F
115522024-02-28T22:17:34.675Z 5 100-119 104 104 104 188 188 188 F F F
115532024-02-28T22:17:34.675Z 6 120-139 103 103 103 187 187 187 F F F
115542024-02-28T22:17:34.675Z 7 140-159 96 96 96 175 175 175 F F F
115552024-02-28T22:17:34.675Z 8 160-179 104 104 104 188 188 188 F F F
115562024-02-28T22:17:34.675Z 9 180-199 97 97 97 176 176 176 F F F
115572024-02-28T22:17:34.686Z 10 200-219 105 105 105 189 189 189 F F F
115582024-02-28T22:17:34.686Z 11 220-239 106 106 106 190 190 190 F F F
115592024-02-28T22:17:34.686Z 12 240-259 102 102 102 186 186 186 F F F
115602024-02-28T22:17:34.686Z 13 260-279 106 106 106 191 191 191 F F F
115612024-02-28T22:17:34.686Z 14 280-299 104 104 104 188 188 188 F F F
115622024-02-28T22:17:34.686Z 15 300-319 105 105 105 189 189 189 F F F
115632024-02-28T22:17:34.686Z 16 320-339 104 104 104 188 188 188 F F F
115642024-02-28T22:17:34.686Z 17 340-359 106 106 106 190 190 190 F F F
115652024-02-28T22:17:34.686Z 18 360-379 107 107 107 194 194 194 F F F
115662024-02-28T22:17:34.687Z 19 380-399 104 104 104 188 188 188 F F F
115672024-02-28T22:17:34.687Z 20 400-419 104 104 104 188 188 188 F F F
115682024-02-28T22:17:34.687Z 21 420-439 107 107 107 194 194 194 F F F
115692024-02-28T22:17:34.687Z 22 440-459 105 105 105 189 189 189 F F F
115702024-02-28T22:17:34.687Z 23 460-479 105 105 105 189 189 189 F F F
115712024-02-28T22:17:34.687Z 24 480-499 104 104 104 188 188 188 F F F
115722024-02-28T22:17:34.687Z 25 500-519 107 107 107 194 194 194 F F F
115732024-02-28T22:17:34.687Z 26 520-539 106 106 106 191 191 191 F F F
115742024-02-28T22:17:34.687Z 27 540-559 106 106 106 193 193 193 F F F
115752024-02-28T22:17:34.687Z 28 560-579 101 101 101 184 184 184 F F F
115762024-02-28T22:17:34.687Z 29 580-599 106 106 106 192 192 192 F F F
115772024-02-28T22:17:34.687ZMax gen: 107, Max flush: 194
115782024-02-28T22:17:34.687Z
115792024-02-28T22:17:34.688ZBegin loop 54
115802024-02-28T22:17:34.688ZDownstairs to restart: 1
115812024-02-28T22:17:34.695Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583326197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
115822024-02-28T22:17:34.695Z{"Crucible runtime is spawned
115832024-02-28T22:17:34.704Zmsg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583734116Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
115842024-02-28T22:17:34.704Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583779194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
115852024-02-28T22:17:34.704Z{"msg":"Crucible 0eef1262-f0b0-4350-8aad-f1d3fda520bc has session id: 1fe09c71-3419-45bb-82d0-c59d855a358c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583798434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115862024-02-28T22:17:34.704Z{"msg":"Upstairs opts: Upstairs UUID: 0eef1262-f0b0-4350-8aad-f1d3fda520bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583836383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115872024-02-28T22:17:34.704Z{"msg":"Crucible stats registered with UUID: 0eef1262-f0b0-4350-8aad-f1d3fda520bc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583855332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115882024-02-28T22:17:34.704Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583884162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
115892024-02-28T22:17:34.704Z{"msg":"0eef1262-f0b0-4350-8aad-f1d3fda520bc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.58392183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115902024-02-28T22:17:34.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583956569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115912024-02-28T22:17:34.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.583989748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115922024-02-28T22:17:34.704Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584038507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115932024-02-28T22:17:34.704Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584086206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115942024-02-28T22:17:34.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584123225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115952024-02-28T22:17:34.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584156424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115962024-02-28T22:17:34.705Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584183593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115972024-02-28T22:17:34.705Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584274601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115982024-02-28T22:17:34.705Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.58430993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
115992024-02-28T22:17:34.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584352378Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116002024-02-28T22:17:34.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584386098Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116012024-02-28T22:17:34.705Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584403727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116022024-02-28T22:17:34.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584951272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116032024-02-28T22:17:34.705Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584979651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116042024-02-28T22:17:34.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.584994951Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116052024-02-28T22:17:34.705Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.58501689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116062024-02-28T22:17:34.705Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.585092168Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116072024-02-28T22:17:34.706Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.585126717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116082024-02-28T22:17:34.706Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.585545626Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116092024-02-28T22:17:34.706Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.585573385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116102024-02-28T22:17:34.706Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.585769999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116112024-02-28T22:17:34.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586093401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116122024-02-28T22:17:34.706Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586130789Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116132024-02-28T22:17:34.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586146809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116142024-02-28T22:17:34.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586167368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116152024-02-28T22:17:34.706Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586190138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116162024-02-28T22:17:34.706Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586204398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116172024-02-28T22:17:34.706Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586222987Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116182024-02-28T22:17:34.706Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586245017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116192024-02-28T22:17:34.707Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 185, 184, 192, 188, 187, 175, 188, 176, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586286475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116202024-02-28T22:17:34.707Z{"msg":"[0]R generation[0..12]: [102, 104, 102, 101, 106, 104, 103, 96, 104, 97, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586321274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116212024-02-28T22:17:34.707Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586347554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116222024-02-28T22:17:34.707Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 185, 184, 192, 188, 187, 175, 188, 176, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586373493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116232024-02-28T22:17:34.707Z{"msg":"[1]R generation[0..12]: [102, 104, 102, 101, 106, 104, 103, 96, 104, 97, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586388342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116242024-02-28T22:17:34.707Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586410172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116252024-02-28T22:17:34.707Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 185, 184, 192, 188, 187, 175, 188, 176, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586424812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO
116262024-02-28T22:17:34.707Z1fe09c71-3419-45bb-82d0-c59d855a358c"}
116272024-02-28T22:17:34.707Z{"msg":"[2]R generation[0..12]: [102, 104, 102, 101, 106, 104, 103, 96, 104, 97, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586453811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116282024-02-28T22:17:34.707Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.58646811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116292024-02-28T22:17:34.707Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.58648927Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116302024-02-28T22:17:34.707Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586505999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116312024-02-28T22:17:34.707Z{"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586538768Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116322024-02-28T22:17:34.708Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586555428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"":"downstairs","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116332024-02-28T22:17:34.708Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586597257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116342024-02-28T22:17:34.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586616996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"0","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116352024-02-28T22:17:34.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586648055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"1","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116362024-02-28T22:17:34.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586664145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"client":"2","":"downstairs","session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116372024-02-28T22:17:34.708Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586682564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116382024-02-28T22:17:34.708Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586703684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116392024-02-28T22:17:34.708Z{"msg":"0eef1262-f0b0-4350-8aad-f1d3fda520bc is now active with session: 1fe09c71-3419-45bb-82d0-c59d855a358c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586723673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116402024-02-28T22:17:34.708Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586752823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567,"session_id":"1fe09c71-3419-45bb-82d0-c59d855a358c"}
116412024-02-28T22:17:34.708Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:33.586766962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
116422024-02-28T22:17:34.708ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116432024-02-28T22:17:34.708ZLoading write count information from file "/tmp/test_repair_verify.data"
116442024-02-28T22:17:34.709ZRun Repair workload
116452024-02-28T22:17:34.709Z01/30 Write block 273 len 12288 data: 4 3 3
116462024-02-28T22:17:34.725Z02/30 Read block 305 len 36864
116472024-02-28T22:17:34.725Z03/30 Write block 116 len 20480 data: 9 8 6 6 7
116482024-02-28T22:17:34.725Z04/30 Read block 566 len 20480
116492024-02-28T22:17:34.725Z05/30 Flush
116502024-02-28T22:17:34.725Z06/30 Write block 494 len 24576 data: 9 10 10 10 8 9
116512024-02-28T22:17:34.725Z07/30 Read block 36 len 4096
116522024-02-28T22:17:34.726Z08/30 Write block 519 len 28672 data: 9 9 8 6 5 5 7
116532024-02-28T22:17:34.726Z09/30 Write block 341 len 28672 data: 9 11 10 8 8 7 7
116542024-02-28T22:17:34.726Z10/30 Flush
116552024-02-28T22:17:34.726Z11/30 Write block 583 len 20480 data: 5 5 7 7 6
116562024-02-28T22:17:34.726Z12/30 Write block 543 len 12288 data: 9 9 10
116572024-02-28T22:17:34.726Z13/30 Write block 288 len 16384 data: 5 5 5 6
116582024-02-28T22:17:34.726Z14/30 Read block 390 len 28672
116592024-02-28T22:17:34.726Z15/30 Read block 144 len 24576
116602024-02-28T22:17:34.726Z16/30 Write block 465 len 32768 data: 9 7 6 8 7 8 9 11
116612024-02-28T22:17:34.727Z17/30 Flush
116622024-02-28T22:17:34.727Z18/30 Write block 130 len 36864 data: 12 13 12 13 12 11 11 11 11
116632024-02-28T22:17:34.727Z19/30 Read block 120 len 8192
116642024-02-28T22:17:34.727Z20/30 Write block 282 len 36864 data: 7 6 6 5 7 4 6 6 6
116652024-02-28T22:17:34.727Z21/30 Read block 234 len 12288
116662024-02-28T22:17:34.727Z22/30 Flush
116672024-02-28T22:17:34.732Z23/30 Write block 362 len 8192 data: 7 8
116682024-02-28T22:17:34.732Z24/30 Write block 460 len 8192 data: 7 7
116692024-02-28T22:17:34.732Z25/30 Write block 311 len 12288 data: 11 12 10
116702024-02-28T22:17:34.732Z26/30 Write block 406 len 16384 data: 10 10 9 8
116712024-02-28T22:17:34.732Z27/30 Write block 188 len 20480 data: 8 8 7 8 11
116722024-02-28T22:17:34.732Z28/30 Write block 197 len 8192 data: 9 9
116732024-02-28T22:17:34.732Z29/30 Read block 301 len 8192
116742024-02-28T22:17:34.733Z30/30 Write block 72 len 36864 data: 8 6 7 6 5 7 7 6 9
116752024-02-28T22:17:34.733Z----------------------------------------------------------------
116762024-02-28T22:17:34.733Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
116772024-02-28T22:17:34.733ZStates: Active Active Active
116782024-02-28T22:17:34.733ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116792024-02-28T22:17:34.733Z 1 Acked 1000 Write 3 Done Done Done false
116802024-02-28T22:17:34.733Z 2 Acked 1001 Read 9 Done Done Done false
116812024-02-28T22:17:34.733Z 3 Acked 1002 Write 5 Done Done Done false
116822024-02-28T22:17:34.733Z 4 Acked 1003 Read 5 Done Done Done false
116832024-02-28T22:17:34.733Z 5 Acked 1004 Flush 0 Done Sent Done false
116842024-02-28T22:17:34.733Z 6 Acked 1005 Write 6 Done Sent Done false
116852024-02-28T22:17:34.733Z 7 Acked 1006 Read 1 Done Sent Done false
116862024-02-28T22:17:34.733Z 8 Acked 1007 Write 7 Done Sent Done false
116872024-02-28T22:17:34.733Z 9 Acked 1008 Write 7 Done Sent Done false
116882024-02-28T22:17:34.734Z 10 Acked 1009 Flush 0 Done Sent Done false
116892024-02-28T22:17:34.734Z 11 Acked 1010 Write 5 Done Sent Done false
116902024-02-28T22:17:34.734Z 12 Acked 1011 Write 3 Done Sent Done false
116912024-02-28T22:17:34.734Z 13 Acked 1012 Write 4 Done Sent Done false
116922024-02-28T22:17:34.734Z 14 Acked 1013 Read 7 Done Sent Done false
116932024-02-28T22:17:34.734Z 15 Acked 1014 Read 6 Done Sent Done false
116942024-02-28T22:17:34.734Z 16 Acked 1015 Write 8 Done Sent Done false
116952024-02-28T22:17:34.734Z 17 Acked 1016 Flush 0 Done Sent Done false
116962024-02-28T22:17:34.734Z 18 Acked 1017 Write 9 Done Sent Done false
116972024-02-28T22:17:34.734Z 19 Acked 1018 Read 2 Done Sent Done false
116982024-02-28T22:17:34.734Z 20 Acked 1019 Write 9 Done Sent Done false
116992024-02-28T22:17:34.734Z 21 Acked 1020 Read 3 Done Sent Done false
117002024-02-28T22:17:34.734Z 22 Acked 1021 Flush 0 Done Sent Done false
117012024-02-28T22:17:34.734Z 23 Acked 1022 Write 2 Done Sent Done false
117022024-02-28T22:17:34.734Z 24 Acked 1023 Write 2 Done Sent Done false
117032024-02-28T22:17:34.734Z 25 Acked 1024 Write 3 Done Sent Done false
117042024-02-28T22:17:34.734Z 26 Acked 1025 Write 4 Done Sent Done false
117052024-02-28T22:17:34.734Z 27 Acked 1026 Write 5 Done Sent Done false
117062024-02-28T22:17:34.734Z 28 Acked 1027 Write 2 Done Sent Done false
117072024-02-28T22:17:34.734Z 29 Acked 1028 Read 2 Done Sent Done false
117082024-02-28T22:17:34.734Z 30 Acked 1029 Write 9 Sent Sent Sent false
117092024-02-28T22:17:34.734Z STATES DS:0 DS:1 DS:2 TOTAL
117102024-02-28T22:17:34.734Z New 0 0 0 0
117112024-02-28T22:17:34.734Z Sent 1 26 1 28
117122024-02-28T22:17:34.735Z Done 29 4 29 62
117132024-02-28T22:17:34.735Z Skipped 0 0 0 0
117142024-02-28T22:17:34.735Z Error 0 0 0 0
117152024-02-28T22:17:34.735ZLast Flush: 1021 0 1021
117162024-02-28T22:17:34.735ZDownstairs last five completed:
117172024-02-28T22:17:34.735ZUpstairs last five completed: 30 29 28 27 26
117182024-02-28T22:17:34.735Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:33.619268867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1567}
117192024-02-28T22:17:34.735ZWrote out file "/tmp/test_repair_verify.data"
117202024-02-28T22:17:34.742Z
117212024-02-28T22:17:34.742ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117222024-02-28T22:17:34.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117232024-02-28T22:17:34.756Z 0 000-019 102 102 102 185 185 185 F F F
117242024-02-28T22:17:34.756Z 1 020-039 104 104 104 188 188 188 F F F
117252024-02-28T22:17:34.756Z 2 040-059 102 102 102 185 185 185 F F F
117262024-02-28T22:17:34.756Z 3 060-079 101 101 101 184 184 184 T F T <---
117272024-02-28T22:17:34.756Z 4 080-099 106 106 106 192 192 192 T F T <---
117282024-02-28T22:17:34.756Z 5 100-119 108 104 108 195 188 195 F T F <---
117292024-02-28T22:17:34.756Z 6 120-139 108 103 108 198 187 198 F T F <---
117302024-02-28T22:17:34.757Z 7 140-159 96 96 96 175 175 175 F F F
117312024-02-28T22:17:34.757Z 8 160-179 104 104 104 188 188 188 F F F
117322024-02-28T22:17:34.775Z 9 180-199 97 97 97 176 176 176 T F T <---
117332024-02-28T22:17:34.776Z 10 200-219 105 105 105 189 189 189 F F F
117342024-02-28T22:17:34.776Z 11 220-239 106 106 106 190 190 190 F F F
117352024-02-28T22:17:34.776Z 12 240-259 102 102 102 186 186 186 F F F
117362024-02-28T22:17:34.776Z 13 260-279 108 106 108 195 191 195 F T F <---
117372024-02-28T22:17:34.776Z 14 280-299 108 104 108 198 188 198 F F F <---
117382024-02-28T22:17:34.776Z 15 300-319 105 105 105 189 189 189 T F T <---
117392024-02-28T22:17:34.776Z 16 320-339 104 104 104 188 188 188 F F F
117402024-02-28T22:17:34.776Z 17 340-359 108 106 108 196 190 196 F F F <---
117412024-02-28T22:17:34.776Z 18 360-379 107 107 107 194 194 194 T F T <---
117422024-02-28T22:17:34.776Z 19 380-399 104 104 104 188 188 188 F F F
117432024-02-28T22:17:34.776Z 20 400-419 104 104 104 188 188 188 T F T <---
117442024-02-28T22:17:34.776Z 21 420-439 107 107 107 194 194 194 F F F
117452024-02-28T22:17:34.777Z 22 440-459 105 105 105 189 189 189 F F F
117462024-02-28T22:17:34.777Z 23 460-479 108 105 108 197 189 197 T F T <---
117472024-02-28T22:17:34.777Z 24 480-499 108 104 108 196 188 196 F F F <---
117482024-02-28T22:17:34.777Z 25 500-519 108 107 108 196 194 196 F F F <---
117492024-02-28T22:17:34.777Z 26 520-539 108 106 108 196 191 196 F F F <---
117502024-02-28T22:17:34.777Z 27 540-559 108 106 108 197 193 197 F F F <---
117512024-02-28T22:17:34.777Z 28 560-579 101 101 101 184 184 184 F F F
117522024-02-28T22:17:34.777Z 29 580-599 108 106 108 197 192 197 F F F <---
117532024-02-28T22:17:34.777ZMax gen: 108, Max flush: 198
117542024-02-28T22:17:34.777ZError: Difference in extent metadata found!
117552024-02-28T22:17:34.777ZA Difference in extent metadata is expected here
117562024-02-28T22:17:34.777ZOn loop 54
117572024-02-28T22:17:34.777Z
117582024-02-28T22:17:34.778ZVerifying data now
117592024-02-28T22:17:40.602ZNo change after verify
117602024-02-28T22:17:40.602ZLoop: 54 Downstairs dump after verify (and repair):
117612024-02-28T22:17:40.616ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117622024-02-28T22:17:40.629Z 0 000-019 102 102 102 185 185 185 F F F
117632024-02-28T22:17:40.629Z 1 020-039 104 104 104 188 188 188 F F F
117642024-02-28T22:17:40.629Z 2 040-059 102 102 102 185 185 185 F F F
117652024-02-28T22:17:40.629Z 3 060-079 109 109 109 199 199 199 F F F
117662024-02-28T22:17:40.629Z 4 080-099 109 109 109 199 199 199 F F F
117672024-02-28T22:17:40.629Z 5 100-119 108 108 108 195 195 195 F F F
117682024-02-28T22:17:40.629Z 6 120-139 108 108 108 198 198 198 F F F
117692024-02-28T22:17:40.629Z 7 140-159 96 96 96 175 175 175 F F F
117702024-02-28T22:17:40.630Z 8 160-179 104 104 104 188 188 188 F F F
117712024-02-28T22:17:40.630Z 9 180-199 109 109 109 199 199 199 F F F
117722024-02-28T22:17:40.634Z 10 200-219 105 105 105 189 189 189 F F F
117732024-02-28T22:17:40.634Z 11 220-239 106 106 106 190 190 190 F F F
117742024-02-28T22:17:40.634Z 12 240-259 102 102 102 186 186 186 F F F
117752024-02-28T22:17:40.635Z 13 260-279 108 108 108 195 195 195 F F F
117762024-02-28T22:17:40.635Z 14 280-299 108 108 108 198 198 198 F F F
117772024-02-28T22:17:40.635Z 15 300-319 109 109 109 199 199 199 F F F
117782024-02-28T22:17:40.635Z 16 320-339 104 104 104 188 188 188 F F F
117792024-02-28T22:17:40.635Z 17 340-359 108 108 108 196 196 196 F F F
117802024-02-28T22:17:40.635Z 18 360-379 109 109 109 199 199 199 F F F
117812024-02-28T22:17:40.635Z 19 380-399 104 104 104 188 188 188 F F F
117822024-02-28T22:17:40.635Z 20 400-419 109 109 109 199 199 199 F F F
117832024-02-28T22:17:40.635Z 21 420-439 107 107 107 194 194 194 F F F
117842024-02-28T22:17:40.635Z 22 440-459 105 105 105 189 189 189 F F F
117852024-02-28T22:17:40.635Z 23 460-479 109 109 109 199 199 199 F F F
117862024-02-28T22:17:40.635Z 24 480-499 108 108 108 196 196 196 F F F
117872024-02-28T22:17:40.635Z 25 500-519 108 108 108 196 196 196 F F F
117882024-02-28T22:17:40.635Z 26 520-539 108 108 108 196 196 196 F F F
117892024-02-28T22:17:40.635Z 27 540-559 108 108 108 197 197 197 F F F
117902024-02-28T22:17:40.635Z 28 560-579 101 101 101 184 184 184 F F F
117912024-02-28T22:17:40.635Z 29 580-599 108 108 108 197 197 197 F F F
117922024-02-28T22:17:40.635ZMax gen: 109, Max flush: 199
117932024-02-28T22:17:40.635Z
117942024-02-28T22:17:40.635ZBegin loop 55
117952024-02-28T22:17:40.635ZDownstairs to restart: 0
117962024-02-28T22:17:40.641Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.524942392Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
117972024-02-28T22:17:40.642Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525183525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
117982024-02-28T22:17:40.642Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525224414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
117992024-02-28T22:17:40.642Z{"msg":"Crucible b8665cc9-41b0-4ae2-9ca8-4a43e1a31d30 has session id: 6db054d9-2106-449f-b06d-e65fb6e658c8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525245164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118002024-02-28T22:17:40.642ZCrucible runtime is spawned
118012024-02-28T22:17:40.642Z{"msg":"Upstairs opts: Upstairs UUID: b8665cc9-41b0-4ae2-9ca8-4a43e1a31d30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525291362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118022024-02-28T22:17:40.642Z{"msg":"Crucible stats registered with UUID: b8665cc9-41b0-4ae2-9ca8-4a43e1a31d30","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525308992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118032024-02-28T22:17:40.642Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525341461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
118042024-02-28T22:17:40.643Z{"msg":"b8665cc9-41b0-4ae2-9ca8-4a43e1a31d30 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525453478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118052024-02-28T22:17:40.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525505226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118062024-02-28T22:17:40.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525529086Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118072024-02-28T22:17:40.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525548115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118082024-02-28T22:17:40.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525580164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118092024-02-28T22:17:40.643Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525596554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118102024-02-28T22:17:40.643Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525616653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118112024-02-28T22:17:40.643Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525638053Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118122024-02-28T22:17:40.644Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525669072Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118132024-02-28T22:17:40.644Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525705951Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118142024-02-28T22:17:40.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525816518Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118152024-02-28T22:17:40.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525866796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118162024-02-28T22:17:40.644Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.525896966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118172024-02-28T22:17:40.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526577797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118182024-02-28T22:17:40.644Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526615756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118192024-02-28T22:17:40.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526632065Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118202024-02-28T22:17:40.644Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526661965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118212024-02-28T22:17:40.644Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526677884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118222024-02-28T22:17:40.645Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.526694514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118232024-02-28T22:17:40.645Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.527267948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118242024-02-28T22:17:40.645Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.527319586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118252024-02-28T22:17:40.645Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.527365165Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118262024-02-28T22:17:40.653Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.527892771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118272024-02-28T22:17:40.653Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.52793409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118282024-02-28T22:17:40.653Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.527967569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118292024-02-28T22:17:40.654Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528006888Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118302024-02-28T22:17:40.654Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528041647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118312024-02-28T22:17:40.654Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528066236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118322024-02-28T22:17:40.654Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528092415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118332024-02-28T22:17:40.654Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528116745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118342024-02-28T22:17:40.655Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 185, 199, 199, 195, 198, 175, 188, 199, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528145244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118352024-02-28T22:17:40.655Z{"msg":"[0]R generation[0..12]: [102, 104, 102, 109, 109, 108, 108, 96, 104, 109, 105, 106]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
118362024-02-28T22:17:40.655Ztime":"2024-02-28T22:17:39.528170463Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118372024-02-28T22:17:40.656Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528206792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118382024-02-28T22:17:40.656Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 185, 199, 199, 195, 198, 175, 188, 199, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528247211Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118392024-02-28T22:17:40.656Z{"msg":"[1]R generation[0..12]: [102, 104, 102, 109, 109, 108, 108, 96, 104, 109, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.52827282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118402024-02-28T22:17:40.656Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528303319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118412024-02-28T22:17:40.657Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 185, 199, 199, 195, 198, 175, 188, 199, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528330259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118422024-02-28T22:17:40.657Z{"msg":"[2]R generation[0..12]: [102, 104, 102, 109, 109, 108, 108, 96, 104, 109, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528362848Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118432024-02-28T22:17:40.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528388507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118442024-02-28T22:17:40.657Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528420396Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118452024-02-28T22:17:40.657Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528444595Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118462024-02-28T22:17:40.658Z{"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528471175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118472024-02-28T22:17:40.658Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528494554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"":"downstairs","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118482024-02-28T22:17:40.658Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528521673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118492024-02-28T22:17:40.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528543793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"0","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118502024-02-28T22:17:40.659Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528570922Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"1","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118512024-02-28T22:17:40.659Z{"msg":"Transition from WaitQuorum to Active"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118522024-02-28T22:17:40.659Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528598901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"client":"2","":"downstairs","session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118532024-02-28T22:17:40.660Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.52862242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118542024-02-28T22:17:40.660Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.52864398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118552024-02-28T22:17:40.660Z{"msg":"b8665cc9-41b0-4ae2-9ca8-4a43e1a31d30 is now active with session: 6db054d9-2106-449f-b06d-e65fb6e658c8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528666829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118562024-02-28T22:17:40.660Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528696398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575,"session_id":"6db054d9-2106-449f-b06d-e65fb6e658c8"}
118572024-02-28T22:17:40.660Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:39.528718848Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
118582024-02-28T22:17:40.660ZLoading write count information from file "/tmp/test_repair_verify.data"
118592024-02-28T22:17:40.661ZRun Repair workload
118602024-02-28T22:17:40.661Z01/30 Write block 284 len 12288 data: 7 6 8
118612024-02-28T22:17:40.661Z02/30 Write block 347 len 20480 data: 8 8 6 7 7
118622024-02-28T22:17:40.662Z03/30 Read block 165 len 16384
118632024-02-28T22:17:40.662Z04/30 Write block 466 len 8192 data: 8 7
118642024-02-28T22:17:40.662Z05/30 Write block 301 len 40960 data: 5 7 10 10 9 9 11 10 10 6
118652024-02-28T22:17:40.662Z06/30 Read block 192 len 24576
118662024-02-28T22:17:40.662Z07/30 Write block 243 len 24576 data: 9 9 9 5 10 13
118672024-02-28T22:17:40.662Z08/30 Write block 84 len 28672 data: 7 7 7 8 8 8 9
118682024-02-28T22:17:40.662Z09/30 Read block 274 len 16384
118692024-02-28T22:17:40.668Z10/30 Write block 588 len 32768 data: 6 6 8 8 9 10 9 9
118702024-02-28T22:17:40.668Z11/30 Read block 400 len 16384
118712024-02-28T22:17:40.668Z12/30 Write block 489 len 32768 data: 9 10 12 11 10 10 11 11
118722024-02-28T22:17:40.668Z13/30 Read block 514 len 20480
118732024-02-28T22:17:40.668Z14/30 Write block 498 len 20480 data: 9 10 11 12 12
118742024-02-28T22:17:40.668Z15/30 Flush
118752024-02-28T22:17:40.668Z16/30 Write block 183 len 4096 data: 8
118762024-02-28T22:17:40.668Z17/30 Read block 97 len 32768
118772024-02-28T22:17:40.668Z18/30 Write block 108 len 8192 data: 12 14
118782024-02-28T22:17:40.668Z19/30 Write block 520 len 28672 data: 10 9 7 6 6 8 7
118792024-02-28T22:17:40.668Z20/30 Read block 243 len 4096
118802024-02-28T22:17:40.668Z21/30 Read block 233 len 36864
118812024-02-28T22:17:40.668Z22/30 Flush
118822024-02-28T22:17:40.668Z23/30 Write block 43 len 36864 data: 7 4 6 5 7 8 9 7 8
118832024-02-28T22:17:40.668Z24/30 Read block 54 len 32768
118842024-02-28T22:17:40.672Z25/30 Read block 259 len 24576
118852024-02-28T22:17:40.672Z26/30 Write block 321 len 4096 data: 6
118862024-02-28T22:17:40.672Z27/30 Read block 119 len 16384
118872024-02-28T22:17:40.672Z28/30 Write block 442 len 32768 data: 9 6 9 10 12 12 11 12
118882024-02-28T22:17:40.672Z29/30 Write block 258 len 36864 data: 10 10 10 10 11 9 8 7 10
118892024-02-28T22:17:40.677Z30/30 Write block 49 len 40960 data: 10 8 9 5 5 5 7 9 9 8
118902024-02-28T22:17:40.677Z----------------------------------------------------------------
118912024-02-28T22:17:40.677Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
118922024-02-28T22:17:40.678ZStates: Active Active Active
118932024-02-28T22:17:40.678ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118942024-02-28T22:17:40.678Z 1 Acked 1000 Write 3 Done Done Done false
118952024-02-28T22:17:40.678Z 2 Acked 1001 Write 5 Sent Done Done false
118962024-02-28T22:17:40.678Z 3 Acked 1002 Read 4 Sent Done Done false
118972024-02-28T22:17:40.679Z 4 Acked 1003 Write 2 Sent Done Done false
118982024-02-28T22:17:40.679Z 5 Acked 1004 Write 10 Sent Done Done false
118992024-02-28T22:17:40.679Z 6 Acked 1005 Read 6 Sent Done Done false
119002024-02-28T22:17:40.679Z 7 Acked 1006 Write 6 Sent Done Done false
119012024-02-28T22:17:40.679Z 8 Acked 1007 Write 7 Sent Done Done false
119022024-02-28T22:17:40.679Z 9 Acked 1008 Read 4 Sent Done Done false
119032024-02-28T22:17:40.679Z 10 Acked 1009 Write 8 Sent Done Done false
119042024-02-28T22:17:40.679Z 11 Acked 1010 Read 4 Sent Done Done false
119052024-02-28T22:17:40.680Z 12 Acked 1011 Write 8 Sent Done Done false
119062024-02-28T22:17:40.680Z 13 Acked 1012 Read 5 Sent Done Done false
119072024-02-28T22:17:40.680Z 14 Acked 1013 Write 5 Sent Done Done false
119082024-02-28T22:17:40.680Z 15 Acked 1014 Flush 0 Sent Done Done false
119092024-02-28T22:17:40.680Z 16 Acked 1015 Write 1 Sent Done Done false
119102024-02-28T22:17:40.680Z 17 Acked 1016 Read 8 Sent Done Done false
119112024-02-28T22:17:40.680Z 18 Acked 1017 Write 2 Sent Done Done false
119122024-02-28T22:17:40.680Z 19 Acked 1018 Write 7 Sent Done Done false
119132024-02-28T22:17:40.680Z 20 Acked 1019 Read 1 Sent Done Done false
119142024-02-28T22:17:40.680Z 21 Acked 1020 Read 9 Sent Done Done false
119152024-02-28T22:17:40.680Z 22 Acked 1021 Flush 0 Sent Done Done false
119162024-02-28T22:17:40.680Z 23 Acked 1022 Write 9 Sent Done Done false
119172024-02-28T22:17:40.680Z 24 Acked 1023 Read 8 Sent Done Done false
119182024-02-28T22:17:40.680Z 25 Acked 1024 Read 6 Sent Done Done false
119192024-02-28T22:17:40.680Z 26 Acked 1025 Write 1 Sent Done Done false
119202024-02-28T22:17:40.680Z 27 Acked 1026 Read 4 Sent Done Done false
119212024-02-28T22:17:40.680Z 28 Acked 1027 Write 8 Sent Done Done false
119222024-02-28T22:17:40.680Z 29 Acked 1028 Write 9 Sent Done Done false
119232024-02-28T22:17:40.680Z 30 Acked 1029 Write 10 Sent Sent Sent false
119242024-02-28T22:17:40.680Z STATES DS:0 DS:1 DS:2 TOTAL
119252024-02-28T22:17:40.680Z New 0 0 0 0
119262024-02-28T22:17:40.680Z Sent 29 1 1 31
119272024-02-28T22:17:40.680Z Done 1 29 29 59
119282024-02-28T22:17:40.681Z Skipped 0 0 0 0
119292024-02-28T22:17:40.681Z Error 0 0 0 0
119302024-02-28T22:17:40.681ZLast Flush: 0 1021 1021
119312024-02-28T22:17:40.681ZDownstairs last five completed:
119322024-02-28T22:17:40.681ZUpstairs last five completed: 30 29 28 27 26
119332024-02-28T22:17:40.681Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:39.562354261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1575}
119342024-02-28T22:17:40.681ZWrote out file "/tmp/test_repair_verify.data"
119352024-02-28T22:17:40.685Z
119362024-02-28T22:17:40.685ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119372024-02-28T22:17:40.698ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119382024-02-28T22:17:40.699Z 0 000-019 102 102 102 185 185 185 F F F
119392024-02-28T22:17:40.699Z 1 020-039 104 104 104 188 188 188 F F F
119402024-02-28T22:17:40.699Z 2 040-059 102 102 102 185 185 185 F T T <---
119412024-02-28T22:17:40.699Z 3 060-079 109 109 109 199 199 199 F F F
119422024-02-28T22:17:40.703Z 4 080-099 109 110 110 199 200 200 F F F <---
119432024-02-28T22:17:40.703Z 5 100-119 108 110 110 195 201 201 F F F <---
119442024-02-28T22:17:40.703Z 6 120-139 108 108 108 198 198 198 F F F
119452024-02-28T22:17:40.703Z 7 140-159 96 96 96 175 175 175 F F F
119462024-02-28T22:17:40.703Z 8 160-179 104 104 104 188 188 188 F F F
119472024-02-28T22:17:40.704Z 9 180-199 109 110 110 199 201 201 F F F <---
119482024-02-28T22:17:40.704Z 10 200-219 105 105 105 189 189 189 F F F
119492024-02-28T22:17:40.704Z 11 220-239 106 106 106 190 190 190 F F F
119502024-02-28T22:17:40.704Z 12 240-259 102 110 110 186 200 200 F T T <---
119512024-02-28T22:17:40.704Z 13 260-279 108 108 108 195 195 195 F T T <---
119522024-02-28T22:17:40.704Z 14 280-299 108 110 110 198 200 200 T F F <---
119532024-02-28T22:17:40.704Z 15 300-319 109 110 110 199 200 200 F F F <---
119542024-02-28T22:17:40.704Z 16 320-339 104 104 104 188 188 188 F T T <---
119552024-02-28T22:17:40.705Z 17 340-359 108 110 110 196 200 200 F F F <---
119562024-02-28T22:17:40.705Z 18 360-379 109 109 109 199 199 199 F F F
119572024-02-28T22:17:40.705Z 19 380-399 104 104 104 188 188 188 F F F
119582024-02-28T22:17:40.705Z 20 400-419 109 109 109 199 199 199 F F F
119592024-02-28T22:17:40.705Z 21 420-439 107 107 107 194 194 194 F F F
119602024-02-28T22:17:40.705Z 22 440-459 105 105 105 189 189 189 F T T <---
119612024-02-28T22:17:40.705Z 23 460-479 109 110 110 199 200 200 F F F <---
119622024-02-28T22:17:40.705Z 24 480-499 108 110 110 196 200 200 F F F <---
119632024-02-28T22:17:40.705Z 25 500-519 108 110 110 196 200 200 F F F <---
119642024-02-28T22:17:40.705Z 26 520-539 108 110 110 196 201 201 F F F <---
119652024-02-28T22:17:40.706Z 27 540-559 108 108 108 197 197 197 F F F
119662024-02-28T22:17:40.706Z 28 560-579 101 101 101 184 184 184 F F F
119672024-02-28T22:17:40.706Z 29 580-599 108 110 110 197 200 200 F F F <---
119682024-02-28T22:17:40.706ZMax gen: 110, Max flush: 201
119692024-02-28T22:17:40.706ZError: Difference in extent metadata found!
119702024-02-28T22:17:40.706ZA Difference in extent metadata is expected here
119712024-02-28T22:17:40.706ZOn loop 55
119722024-02-28T22:17:40.706Z
119732024-02-28T22:17:40.711ZVerifying data now
119742024-02-28T22:17:45.479ZNo change after verify
119752024-02-28T22:17:45.479ZLoop: 55 Downstairs dump after verify (and repair):
119762024-02-28T22:17:45.495ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119772024-02-28T22:17:45.495Z 0 000-019 102 102 102 185 185 185 F F F
119782024-02-28T22:17:45.496Z 1 020-039 104 104 104 188 188 188 F F F
119792024-02-28T22:17:45.496Z 2 040-059 111 111 111 202 202 202 F F F
119802024-02-28T22:17:45.496Z 3 060-079 109 109 109 199 199 199 F F F
119812024-02-28T22:17:45.496Z 4 080-099 110 110 110 200 200 200 F F F
119822024-02-28T22:17:45.496Z 5 100-119 110 110 110 201 201 201 F F F
119832024-02-28T22:17:45.496Z 6 120-139 108 108 108 198 198 198 F F F
119842024-02-28T22:17:45.497Z 7 140-159 96 96 96 175 175 175 F F F
119852024-02-28T22:17:45.497Z 8 160-179 104 104 104 188 188 188 F F F
119862024-02-28T22:17:45.497Z 9 180-199 110 110 110 201 201 201 F F F
119872024-02-28T22:17:45.503Z 10 200-219 105 105 105 189 189 189 F F F
119882024-02-28T22:17:45.503Z 11 220-239 106 106 106 190 190 190 F F F
119892024-02-28T22:17:45.503Z 12 240-259 111 111 111 202 202 202 F F F
119902024-02-28T22:17:45.503Z 13 260-279 111 111 111 202 202 202 F F F
119912024-02-28T22:17:45.503Z 14 280-299 110 110 110 200 200 200 F F F
119922024-02-28T22:17:45.504Z 15 300-319 110 110 110 200 200 200 F F F
119932024-02-28T22:17:45.504Z 16 320-339 111 111 111 202 202 202 F F F
119942024-02-28T22:17:45.504Z 17 340-359 110 110 110 200 200 200 F F F
119952024-02-28T22:17:45.504Z 18 360-379 109 109 109 199 199 199 F F F
119962024-02-28T22:17:45.504Z 19 380-399 104 104 104 188 188 188 F F F
119972024-02-28T22:17:45.504Z 20 400-419 109 109 109 199 199 199 F F F
119982024-02-28T22:17:45.504Z 21 420-439 107 107 107 194 194 194 F F F
119992024-02-28T22:17:45.505Z 22 440-459 111 111 111 202 202 202 F F F
120002024-02-28T22:17:45.505Z 23 460-479 110 110 110 200 200 200 F F F
120012024-02-28T22:17:45.505Z 24 480-499 110 110 110 200 200 200 F F F
120022024-02-28T22:17:45.505Z 25 500-519 110 110 110 200 200 200 F F F
120032024-02-28T22:17:45.505Z 26 520-539 110 110 110 201 201 201 F F F
120042024-02-28T22:17:45.505Z 27 540-559 108 108 108 197 197 197 F F F
120052024-02-28T22:17:45.505Z 28 560-579 101 101 101 184 184 184 F F F
120062024-02-28T22:17:45.505Z 29 580-599 110 110 110 200 200 200 F F F
120072024-02-28T22:17:45.506ZMax gen: 111, Max flush: 202
120082024-02-28T22:17:45.506Z
120092024-02-28T22:17:45.506ZBegin loop 56
120102024-02-28T22:17:45.506ZDownstairs to restart: 2
120112024-02-28T22:17:45.516Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.40324388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
120122024-02-28T22:17:45.516Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\nCrucible runtime is spawned
120132024-02-28T22:17:45.526Z git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.40358066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
120142024-02-28T22:17:45.526Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403617499Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
120152024-02-28T22:17:45.526Z{"msg":"Crucible b10feb66-a0b4-4fbe-8e3e-f6d66a8bb15b has session id: 8b61afef-d606-4211-b8a6-b1240d9d17c5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403673568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120162024-02-28T22:17:45.527Z{"msg":"Upstairs opts: Upstairs UUID: b10feb66-a0b4-4fbe-8e3e-f6d66a8bb15b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403737476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120172024-02-28T22:17:45.527Z{"msg":"Crucible stats registered with UUID: b10feb66-a0b4-4fbe-8e3e-f6d66a8bb15b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403766375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120182024-02-28T22:17:45.527Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403783025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
120192024-02-28T22:17:45.527Z{"msg":"b10feb66-a0b4-4fbe-8e3e-f6d66a8bb15b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403807404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120202024-02-28T22:17:45.527Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403834703Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120212024-02-28T22:17:45.527Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403867252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120222024-02-28T22:17:45.528Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.403912531Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120232024-02-28T22:17:45.528Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404021968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120242024-02-28T22:17:45.528Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404058387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120252024-02-28T22:17:45.528Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404083636Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120262024-02-28T22:17:45.528Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404108546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120272024-02-28T22:17:45.528Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404156924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120282024-02-28T22:17:45.528Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404182983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120292024-02-28T22:17:45.528Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404207493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120302024-02-28T22:17:45.528Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404232282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120312024-02-28T22:17:45.529Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404275191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120322024-02-28T22:17:45.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404812336Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120332024-02-28T22:17:45.529Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404852205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120342024-02-28T22:17:45.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404880404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120352024-02-28T22:17:45.529Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404902854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120362024-02-28T22:17:45.529Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404922483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120372024-02-28T22:17:45.529Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.404945042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120382024-02-28T22:17:45.530Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.40540456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120392024-02-28T22:17:45.530Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405452388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120402024-02-28T22:17:45.530Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405679212Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120412024-02-28T22:17:45.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405865407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120422024-02-28T22:17:45.530Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405903386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120432024-02-28T22:17:45.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405928426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120442024-02-28T22:17:45.530Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405963645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120452024-02-28T22:17:45.530Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.405999104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120462024-02-28T22:17:45.530Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406033543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120472024-02-28T22:17:45.531Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406217347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120482024-02-28T22:17:45.531Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406264296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120492024-02-28T22:17:45.531Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 202, 199, 200, 201, 198, 175, 188, 201, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406292836Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120502024-02-28T22:17:45.531Z{"msg":"[0]R generation[0..12]: [102, 104, 111, 109, 110, 110, 108, 96, 104, 110, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406328955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120512024-02-28T22:17:45.531Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406363943Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120522024-02-28T22:17:45.531Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 202, 199, 200, 201, 198, 175, 188, 201, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406410362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120532024-02-28T22:17:45.532ZWait for a query_work_queue command to finish before sending IO
120542024-02-28T22:17:45.532Z{"msg":"[1]R generation[0..12]: [102, 104, 111, 109, 110, 110, 108, 96, 104, 110, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406454021Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120552024-02-28T22:17:45.532Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.40649497Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120562024-02-28T22:17:45.532Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 202, 199, 200, 201, 198, 175, 188, 201, 189, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406536059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120572024-02-28T22:17:45.532Z{"msg":"[2]R generation[0..12]: [102, 104, 111, 109, 110, 110, 108, 96, 104, 110, 105, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406576508Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120582024-02-28T22:17:45.532Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406615347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120592024-02-28T22:17:45.532Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406651075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120602024-02-28T22:17:45.532Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406688405Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120612024-02-28T22:17:45.533Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406723304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120622024-02-28T22:17:45.533Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406757193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"":"downstairs","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120632024-02-28T22:17:45.533Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406782302Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120642024-02-28T22:17:45.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406796171Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120652024-02-28T22:17:45.533Zip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"0","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120662024-02-28T22:17:45.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406814201Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"1","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120672024-02-28T22:17:45.533Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406829161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"client":"2","":"downstairs","session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120682024-02-28T22:17:45.533Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.4068427Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120692024-02-28T22:17:45.534Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.40686409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120702024-02-28T22:17:45.534Z{"msg":"b10feb66-a0b4-4fbe-8e3e-f6d66a8bb15b is now active with session: 8b61afef-d606-4211-b8a6-b1240d9d17c5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406878559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120712024-02-28T22:17:45.534Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406899059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583,"session_id":"8b61afef-d606-4211-b8a6-b1240d9d17c5"}
120722024-02-28T22:17:45.534Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:44.406911938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
120732024-02-28T22:17:45.534ZLoading write count information from file "/tmp/test_repair_verify.data"
120742024-02-28T22:17:45.534ZRun Repair workload
120752024-02-28T22:17:45.534Z01/30 Write block 66 len 36864 data: 11 12 11 8 10 8 9 7 8
120762024-02-28T22:17:45.551Z02/30 Write block 510 len 40960 data: 11 9 10 10 9 10 10 10 9 10
120772024-02-28T22:17:45.551Z03/30 Write block 104 len 40960 data: 7 6 8 9 13 15 13 11 11 8
120782024-02-28T22:17:45.551Z04/30 Read block 270 len 20480
120792024-02-28T22:17:45.551Z05/30 Read block 171 len 28672
120802024-02-28T22:17:45.551Z06/30 Read block 84 len 20480
120812024-02-28T22:17:45.551Z07/30 Read block 215 len 32768
120822024-02-28T22:17:45.551Z08/30 Write block 424 len 28672 data: 10 9 5 5 5 6 6
120832024-02-28T22:17:45.552Z09/30 Read block 150 len 24576
120842024-02-28T22:17:45.552Z10/30 Write block 257 len 36864 data: 11 11 11 11 11 12 10 9 8
120852024-02-28T22:17:45.552Z11/30 Write block 387 len 24576 data: 9 7 6 9 10 9
120862024-02-28T22:17:45.552Z12/30 Write block 68 len 24576 data: 12 9 11 9 10 8
120872024-02-28T22:17:45.552Z13/30 Read block 90 len 28672
120882024-02-28T22:17:45.552Z14/30 Read block 389 len 16384
120892024-02-28T22:17:45.552Z15/30 Read block 30 len 16384
120902024-02-28T22:17:45.552Z16/30 Write block 225 len 28672 data: 7 9 6 8 12 11 15
120912024-02-28T22:17:45.552Z17/30 Write block 93 len 20480 data: 8 8 6 6 6
120922024-02-28T22:17:45.552Z18/30 Write block 228 len 20480 data: 9 13 12 16 15
120932024-02-28T22:17:45.552Z19/30 Write block 170 len 24576 data: 14 13 12 13 13 11
120942024-02-28T22:17:45.552Z20/30 Read block 137 len 36864
120952024-02-28T22:17:45.553Z21/30 Flush
120962024-02-28T22:17:45.557Z22/30 Write block 98 len 40960 data: 7 8 9 8 8 8 8 7 9 10
120972024-02-28T22:17:45.558Z23/30 Write block 61 len 4096 data: 12
120982024-02-28T22:17:45.558Z24/30 Read block 448 len 24576
120992024-02-28T22:17:45.558Z25/30 Write block 514 len 36864 data: 10 11 11 11 10 11 11 10 8
121002024-02-28T22:17:45.558Z26/30 Read block 576 len 16384
121012024-02-28T22:17:45.558Z27/30 Write block 510 len 4096 data: 12
121022024-02-28T22:17:45.558Z28/30 Write block 42 len 40960 data: 7 8 5 7 6 8 9 11 9 10
121032024-02-28T22:17:45.558Z29/30 Write block 584 len 12288 data: 6 8 8
121042024-02-28T22:17:45.558Z30/30 Read block 364 len 40960
121052024-02-28T22:17:45.559Z----------------------------------------------------------------
121062024-02-28T22:17:45.559Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121072024-02-28T22:17:45.559ZStates: Active Active Active
121082024-02-28T22:17:45.559ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121092024-02-28T22:17:45.559Z 1 Acked 1000 Write 9 Done Done Sent false
121102024-02-28T22:17:45.559Z 2 Acked 1001 Write 10 Done Done Sent false
121112024-02-28T22:17:45.560Z 3 Acked 1002 Write 10 Done Done Sent false
121122024-02-28T22:17:45.560Z 4 Acked 1003 Read 5 Done Done Sent false
121132024-02-28T22:17:45.560Z 5 Acked 1004 Read 7 Done Done Sent false
121142024-02-28T22:17:45.560Z 6 Acked 1005 Read 5 Done Done Sent false
121152024-02-28T22:17:45.560Z 7 Acked 1006 Read 8 Done Done Sent false
121162024-02-28T22:17:45.560Z 8 Acked 1007 Write 7 Done Done Sent false
121172024-02-28T22:17:45.560Z 9 Acked 1008 Read 6 Done Done Sent false
121182024-02-28T22:17:45.560Z 10 Acked 1009 Write 9 Done Done Sent false
121192024-02-28T22:17:45.560Z 11 Acked 1010 Write 6 Done Done Sent false
121202024-02-28T22:17:45.561Z 12 Acked 1011 Write 6 Done Done Sent false
121212024-02-28T22:17:45.561Z 13 Acked 1012 Read 7 Done Done Sent false
121222024-02-28T22:17:45.561Z 14 Acked 1013 Read 4 Done Done Sent false
121232024-02-28T22:17:45.561Z 15 Acked 1014 Read 4 Done Done Sent false
121242024-02-28T22:17:45.561Z 16 Acked 1015 Write 7 Done Done Sent false
121252024-02-28T22:17:45.562Z 17 Acked 1016 Write 5 Done Done Sent false
121262024-02-28T22:17:45.562Z 18 Acked 1017 Write 5 Done Done Sent false
121272024-02-28T22:17:45.562Z 19 Acked 1018 Write 6 Done Done Sent false
121282024-02-28T22:17:45.562Z 20 Acked 1019 Read 9 Done Done Sent false
121292024-02-28T22:17:45.562Z 21 Acked 1020 Flush 0 Done Done Sent false
121302024-02-28T22:17:45.562Z 22 Acked 1021 Write 10 Done Done Sent false
121312024-02-28T22:17:45.562Z 23 Acked 1022 Write 1 Done Done Sent false
121322024-02-28T22:17:45.562Z 24 Acked 1023 Read 6 Done Done Sent false
121332024-02-28T22:17:45.563Z 25 Acked 1024 Write 9 Done Done Sent false
121342024-02-28T22:17:45.563Z 26 Acked 1025 Read 4 Done Done Sent false
121352024-02-28T22:17:45.563Z 27 Acked 1026 Write 1 Done Done Sent false
121362024-02-28T22:17:45.563Z 28 Acked 1027 Write 10 Done Done Sent false
121372024-02-28T22:17:45.563Z 29 Acked 1028 Write 3 Done Done Sent false
121382024-02-28T22:17:45.563Z 30 Acked 1029 Read 10 Done Done Sent false
121392024-02-28T22:17:45.563Z STATES DS:0 DS:1 DS:2 TOTAL
121402024-02-28T22:17:45.563Z New 0 0 0 0
121412024-02-28T22:17:45.564Z Sent 0 0 30 30
121422024-02-28T22:17:45.564Z Done 30 30 0 60
121432024-02-28T22:17:45.564Z Skipped 0 0 0 0
121442024-02-28T22:17:45.564Z Error 0 0 0 0
121452024-02-28T22:17:45.564ZLast Flush: 1020 1020 0
121462024-02-28T22:17:45.564ZDownstairs last five completed:
121472024-02-28T22:17:45.564ZUpstairs last five completed: 30 29 28 27 26
121482024-02-28T22:17:45.565Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:44.439107392Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1583}
121492024-02-28T22:17:45.565ZWrote out file "/tmp/test_repair_verify.data"
121502024-02-28T22:17:45.565Z
121512024-02-28T22:17:45.570ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121522024-02-28T22:17:45.575ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121532024-02-28T22:17:45.575Z 0 000-019 102 102 102 185 185 185 F F F
121542024-02-28T22:17:45.576Z 1 020-039 104 104 104 188 188 188 F F F
121552024-02-28T22:17:45.576Z 2 040-059 111 111 111 202 202 202 T T F <---
121562024-02-28T22:17:45.576Z 3 060-079 112 112 109 203 203 199 T T F <---
121572024-02-28T22:17:45.576Z 4 080-099 112 112 110 203 203 200 T T F <---
121582024-02-28T22:17:45.576Z 5 100-119 112 112 110 203 203 201 T T F <---
121592024-02-28T22:17:45.576Z 6 120-139 108 108 108 198 198 198 F F F
121602024-02-28T22:17:45.576Z 7 140-159 96 96 96 175 175 175 F F F
121612024-02-28T22:17:45.576Z 8 160-179 112 112 104 203 203 188 F F F <---
121622024-02-28T22:17:45.576Z 9 180-199 110 110 110 201 201 201 F F F
121632024-02-28T22:17:45.576Z 10 200-219 105 105 105 189 189 189 F F F
121642024-02-28T22:17:45.580Z 11 220-239 112 112 106 203 203 190 F F F <---
121652024-02-28T22:17:45.581Z 12 240-259 112 112 111 203 203 202 F F F <---
121662024-02-28T22:17:45.581Z 13 260-279 112 112 111 203 203 202 F F F <---
121672024-02-28T22:17:45.581Z 14 280-299 110 110 110 200 200 200 F F F
121682024-02-28T22:17:45.581Z 15 300-319 110 110 110 200 200 200 F F F
121692024-02-28T22:17:45.581Z 16 320-339 111 111 111 202 202 202 F F F
121702024-02-28T22:17:45.581Z 17 340-359 110 110 110 200 200 200 F F F
121712024-02-28T22:17:45.581Z 18 360-379 109 109 109 199 199 199 F F F
121722024-02-28T22:17:45.581Z 19 380-399 112 112 104 203 203 188 F F F <---
121732024-02-28T22:17:45.581Z 20 400-419 109 109 109 199 199 199 F F F
121742024-02-28T22:17:45.581Z 21 420-439 112 112 107 203 203 194 F F F <---
121752024-02-28T22:17:45.582Z 22 440-459 111 111 111 202 202 202 F F F
121762024-02-28T22:17:45.582Z 23 460-479 110 110 110 200 200 200 F F F
121772024-02-28T22:17:45.582Z 24 480-499 110 110 110 200 200 200 F F F
121782024-02-28T22:17:45.582Z 25 500-519 112 112 110 203 203 200 T T F <---
121792024-02-28T22:17:45.582Z 26 520-539 110 110 110 201 201 201 T T F <---
121802024-02-28T22:17:45.582Z 27 540-559 108 108 108 197 197 197 F F F
121812024-02-28T22:17:45.582Z 28 560-579 101 101 101 184 184 184 F F F
121822024-02-28T22:17:45.582Z 29 580-599 110 110 110 200 200 200 T T F <---
121832024-02-28T22:17:45.582ZMax gen: 112, Max flush: 203
121842024-02-28T22:17:45.583ZError: Difference in extent metadata found!
121852024-02-28T22:17:45.583ZA Difference in extent metadata is expected here
121862024-02-28T22:17:45.583ZOn loop 56
121872024-02-28T22:17:45.583Z
121882024-02-28T22:17:45.587ZVerifying data now
121892024-02-28T22:17:50.239ZNo change after verify
121902024-02-28T22:17:50.239ZLoop: 56 Downstairs dump after verify (and repair):
121912024-02-28T22:17:50.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121922024-02-28T22:17:50.253Z 0 000-019 102 102 102 185 185 185 F F F
121932024-02-28T22:17:50.253Z 1 020-039 104 104 104 188 188 188 F F F
121942024-02-28T22:17:50.253Z 2 040-059 113 113 113 204 204 204 F F F
121952024-02-28T22:17:50.253Z 3 060-079 113 113 113 204 204 204 F F F
121962024-02-28T22:17:50.253Z 4 080-099 113 113 113 204 204 204 F F F
121972024-02-28T22:17:50.254Z 5 100-119 113 113 113 204 204 204 F F F
121982024-02-28T22:17:50.254Z 6 120-139 108 108 108 198 198 198 F F F
121992024-02-28T22:17:50.254Z 7 140-159 96 96 96 175 175 175 F F F
122002024-02-28T22:17:50.254Z 8 160-179 112 112 112 203 203 203 F F F
122012024-02-28T22:17:50.254Z 9 180-199 110 110 110 201 201 201 F F F
122022024-02-28T22:17:50.254Z 10 200-219 105 105 105 189 189 189 F F F
122032024-02-28T22:17:50.254Z 11 220-239 112 112 112 203 203 203 F F F
122042024-02-28T22:17:50.254Z 12 240-259 112 112 112 203 203 203 F F F
122052024-02-28T22:17:50.260Z 13 260-279 112 112 112 203 203 203 F F F
122062024-02-28T22:17:50.260Z 14 280-299 110 110 110 200 200 200 F F F
122072024-02-28T22:17:50.260Z 15 300-319 110 110 110 200 200 200 F F F
122082024-02-28T22:17:50.260Z 16 320-339 111 111 111 202 202 202 F F F
122092024-02-28T22:17:50.260Z 17 340-359 110 110 110 200 200 200 F F F
122102024-02-28T22:17:50.260Z 18 360-379 109 109 109 199 199 199 F F F
122112024-02-28T22:17:50.261Z 19 380-399 112 112 112 203 203 203 F F F
122122024-02-28T22:17:50.261Z 20 400-419 109 109 109 199 199 199 F F F
122132024-02-28T22:17:50.261Z 21 420-439 112 112 112 203 203 203 F F F
122142024-02-28T22:17:50.261Z 22 440-459 111 111 111 202 202 202 F F F
122152024-02-28T22:17:50.261Z 23 460-479 110 110 110 200 200 200 F F F
122162024-02-28T22:17:50.261Z 24 480-499 110 110 110 200 200 200 F F F
122172024-02-28T22:17:50.261Z 25 500-519 113 113 113 204 204 204 F F F
122182024-02-28T22:17:50.262Z 26 520-539 113 113 113 204 204 204 F F F
122192024-02-28T22:17:50.262Z 27 540-559 108 108 108 197 197 197 F F F
122202024-02-28T22:17:50.262Z 28 560-579 101 101 101 184 184 184 F F F
122212024-02-28T22:17:50.262Z 29 580-599 113 113 113 204 204 204 F F F
122222024-02-28T22:17:50.262ZMax gen: 113, Max flush: 204
122232024-02-28T22:17:50.262Z
122242024-02-28T22:17:50.262ZBegin loop 57
122252024-02-28T22:17:50.262ZDownstairs to restart: 2
122262024-02-28T22:17:50.274Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.161892424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
122272024-02-28T22:17:50.274Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162170106Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
122282024-02-28T22:17:50.274Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162195755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
122292024-02-28T22:17:50.281Z{"msg":"Crucible 83505cf8-e40a-4dce-9b6c-94c75717aba6 has session id: 0349d86d-40e3-4bc9-a448-32d3ebfae6a2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162231164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122302024-02-28T22:17:50.281Z{"msg":"Crucible runtime is spawned
122312024-02-28T22:17:50.281ZUpstairs opts: Upstairs UUID: 83505cf8-e40a-4dce-9b6c-94c75717aba6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162286433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122322024-02-28T22:17:50.282Z{"msg":"Crucible stats registered with UUID: 83505cf8-e40a-4dce-9b6c-94c75717aba6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162324612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122332024-02-28T22:17:50.282Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162347681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
122342024-02-28T22:17:50.282Z{"msg":"83505cf8-e40a-4dce-9b6c-94c75717aba6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162529176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122352024-02-28T22:17:50.282Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162575225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122362024-02-28T22:17:50.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162605854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122372024-02-28T22:17:50.282Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162639563Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122382024-02-28T22:17:50.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162672012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122392024-02-28T22:17:50.282Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162696502Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122402024-02-28T22:17:50.282Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.1627401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122412024-02-28T22:17:50.282Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162781309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122422024-02-28T22:17:50.282Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162826238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122432024-02-28T22:17:50.282Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162864817Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122442024-02-28T22:17:50.283Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162901486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122452024-02-28T22:17:50.283Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.162929665Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122462024-02-28T22:17:50.283Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163047182Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122472024-02-28T22:17:50.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163499899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122482024-02-28T22:17:50.283Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163537838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122492024-02-28T22:17:50.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163678784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122502024-02-28T22:17:50.283Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163729333Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122512024-02-28T22:17:50.283Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163776092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122522024-02-28T22:17:50.283Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.163816481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122532024-02-28T22:17:50.283Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164143052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122542024-02-28T22:17:50.284Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164412654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122552024-02-28T22:17:50.284Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164453713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122562024-02-28T22:17:50.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164645978Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122572024-02-28T22:17:50.284Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164712836Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122582024-02-28T22:17:50.284Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164756585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122592024-02-28T22:17:50.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.164985468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122602024-02-28T22:17:50.284Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165029347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122612024-02-28T22:17:50.284Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165056486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122622024-02-28T22:17:50.284Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165142304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122632024-02-28T22:17:50.284Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165170633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122642024-02-28T22:17:50.284Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 204, 204, 204, 204, 198, 175, 203, 201, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165187203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122652024-02-28T22:17:50.285Z{"msg":"[0]R generation[0..12]: [102, 104, 113, 113, 113, 113, 108, 96, 112, 110, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165209082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122662024-02-28T22:17:50.285Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165224602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122672024-02-28T22:17:50.285Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 204, 204, 204, 204, 198, 175, 203, 201, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165246481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122682024-02-28T22:17:50.285Z{"msg":"[1]R generation[0..12]: [102, 104, 113, 113, 113, 113, 108, 96, 112, 110, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165261001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122692024-02-28T22:17:50.285Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.16528268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122702024-02-28T22:17:50.285Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 204, 204, 204, 204, 198, 175, 203, 201, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.1652977Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122712024-02-28T22:17:50.285Z{"msg":"[2]R generation[0..12]: [102, 104, 113, 113, 113, 113, 108, 96, 112, 110, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165324429Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122722024-02-28T22:17:50.285Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165339119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122732024-02-28T22:17:50.285Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
122742024-02-28T22:17:50.285Z,"time":"2024-02-28T22:17:49.165353138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122752024-02-28T22:17:50.285Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165378957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122762024-02-28T22:17:50.285Z{"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165392057Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122772024-02-28T22:17:50.285Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165414417Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"":"downstairs","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122782024-02-28T22:17:50.286Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165428396Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122792024-02-28T22:17:50.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165453035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"0","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122802024-02-28T22:17:50.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165467615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"1","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122812024-02-28T22:17:50.286Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165480955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"client":"2","":"downstairs","session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122822024-02-28T22:17:50.286Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165501804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122832024-02-28T22:17:50.286Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165515854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122842024-02-28T22:17:50.286Z{"msg":"83505cf8-e40a-4dce-9b6c-94c75717aba6 is now active with session: 0349d86d-40e3-4bc9-a448-32d3ebfae6a2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165529113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122852024-02-28T22:17:50.286Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165548553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591,"session_id":"0349d86d-40e3-4bc9-a448-32d3ebfae6a2"}
122862024-02-28T22:17:50.286Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:49.165562023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
122872024-02-28T22:17:50.286ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
122882024-02-28T22:17:50.286ZLoading write count information from file "/tmp/test_repair_verify.data"
122892024-02-28T22:17:50.287ZRun Repair workload
122902024-02-28T22:17:50.294Z01/30 Write block 280 len 8192 data: 7 6
122912024-02-28T22:17:50.294Z02/30 Flush
122922024-02-28T22:17:50.294Z03/30 Write block 586 len 40960 data: 9 7 7 7 9 9 10 11 10 10
122932024-02-28T22:17:50.294Z04/30 Read block 288 len 40960
122942024-02-28T22:17:50.295Z05/30 Read block 280 len 16384
122952024-02-28T22:17:50.295Z06/30 Write block 51 len 40960 data: 11 6 6 6 8 10 10 9 12 13
122962024-02-28T22:17:50.295Z07/30 Write block 574 len 12288 data: 7 8 8
122972024-02-28T22:17:50.295Z08/30 Read block 118 len 36864
122982024-02-28T22:17:50.295Z09/30 Read block 137 len 24576
122992024-02-28T22:17:50.295Z10/30 Read block 123 len 16384
123002024-02-28T22:17:50.295Z11/30 Write block 250 len 8192 data: 7 8
123012024-02-28T22:17:50.300Z12/30 Write block 72 len 32768 data: 11 9 9 7 6 8 8 7
123022024-02-28T22:17:50.300Z13/30 Write block 323 len 36864 data: 7 9 9 10 11 14 14 15 12
123032024-02-28T22:17:50.300Z14/30 Write block 533 len 28672 data: 10 12 9 11 10 9 9
123042024-02-28T22:17:50.301Z15/30 Write block 182 len 16384 data: 7 9 10 10
123052024-02-28T22:17:50.301Z16/30 Write block 250 len 12288 data: 8 9 8
123062024-02-28T22:17:50.301Z17/30 Write block 515 len 8192 data: 12 12
123072024-02-28T22:17:50.301Z18/30 Write block 591 len 4096 data: 10
123082024-02-28T22:17:50.301Z19/30 Read block 103 len 40960
123092024-02-28T22:17:50.301Z20/30 Write block 136 len 36864 data: 12 12 12 10 9 8 9 11 11
123102024-02-28T22:17:50.301Z21/30 Write block 562 len 28672 data: 9 10 10 11 13 15 14
123112024-02-28T22:17:50.301Z22/30 Read block 308 len 40960
123122024-02-28T22:17:50.305Z23/30 Write block 567 len 4096 data: 16
123132024-02-28T22:17:50.305Z24/30 Read block 34 len 4096
123142024-02-28T22:17:50.305Z25/30 Write block 284 len 16384 data: 8 7 9 5
123152024-02-28T22:17:50.306Z26/30 Flush
123162024-02-28T22:17:50.309Z27/30 Write block 82 len 16384 data: 12 10 8 8
123172024-02-28T22:17:50.309Z28/30 Write block 317 len 40960 data: 5 6 6 6 7 8 8 10 10 11
123182024-02-28T22:17:50.309Z29/30 Read block 539 len 20480
123192024-02-28T22:17:50.313Z30/30 Write block 287 len 20480 data: 6 7 7 7 7
123202024-02-28T22:17:50.313Z----------------------------------------------------------------
123212024-02-28T22:17:50.314Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123222024-02-28T22:17:50.314ZStates: Active Active Active
123232024-02-28T22:17:50.314ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123242024-02-28T22:17:50.314Z 1 Acked 1000 Write 2 Done Done Sent false
123252024-02-28T22:17:50.314Z 2 Acked 1001 Flush 0 Done Done Sent false
123262024-02-28T22:17:50.314Z 3 Acked 1002 Write 10 Done Done Sent false
123272024-02-28T22:17:50.314Z 4 Acked 1003 Read 10 Done Done Sent false
123282024-02-28T22:17:50.314Z 5 Acked 1004 Read 4 Done Done Sent false
123292024-02-28T22:17:50.314Z 6 Acked 1005 Write 10 Done Done Sent false
123302024-02-28T22:17:50.315Z 7 Acked 1006 Write 3 Done Done Sent false
123312024-02-28T22:17:50.315Z 8 Acked 1007 Read 9 Done Done Sent false
123322024-02-28T22:17:50.315Z 9 Acked 1008 Read 6 Done Done Sent false
123332024-02-28T22:17:50.315Z 10 Acked 1009 Read 4 Done Done Sent false
123342024-02-28T22:17:50.315Z 11 Acked 1010 Write 2 Done Done Sent false
123352024-02-28T22:17:50.315Z 12 Acked 1011 Write 8 Done Done Sent false
123362024-02-28T22:17:50.315Z 13 Acked 1012 Write 9 Done Done Sent false
123372024-02-28T22:17:50.316Z 14 Acked 1013 Write 7 Done Done Sent false
123382024-02-28T22:17:50.316Z 15 Acked 1014 Write 4 Done Done Sent false
123392024-02-28T22:17:50.316Z 16 Acked 1015 Write 3 Done Done Sent false
123402024-02-28T22:17:50.316Z 17 Acked 1016 Write 2 Done Done Sent false
123412024-02-28T22:17:50.316Z 18 Acked 1017 Write 1 Done Done Sent false
123422024-02-28T22:17:50.316Z 19 Acked 1018 Read 10 Done Done Sent false
123432024-02-28T22:17:50.316Z 20 Acked 1019 Write 9 Done Done Sent false
123442024-02-28T22:17:50.316Z 21 Acked 1020 Write 7 Done Done Sent false
123452024-02-28T22:17:50.316Z 22 Acked 1021 Read 10 Done Done Sent false
123462024-02-28T22:17:50.317Z 23 Acked 1022 Write 1 Done Done Sent false
123472024-02-28T22:17:50.317Z 24 Acked 1023 Read 1 Done Done Sent false
123482024-02-28T22:17:50.317Z 25 Acked 1024 Write 4 Done Done Sent false
123492024-02-28T22:17:50.317Z 26 Acked 1025 Flush 0 Done Done Sent false
123502024-02-28T22:17:50.317Z 27 Acked 1026 Write 4 Done Done Sent false
123512024-02-28T22:17:50.317Z 28 Acked 1027 Write 10 Done Done Sent false
123522024-02-28T22:17:50.317Z 29 Acked 1028 Read 5 Done Done Sent false
123532024-02-28T22:17:50.317Z 30 Acked 1029 Write 5 Sent Sent Sent false
123542024-02-28T22:17:50.318Z STATES DS:0 DS:1 DS:2 TOTAL
123552024-02-28T22:17:50.318Z New 0 0 0 0
123562024-02-28T22:17:50.318Z Sent 1 1 30 32
123572024-02-28T22:17:50.318Z Done 29 29 0 58
123582024-02-28T22:17:50.318Z Skipped 0 0 0 0
123592024-02-28T22:17:50.318Z Error 0 0 0 0
123602024-02-28T22:17:50.318ZLast Flush: 1025 1025 0
123612024-02-28T22:17:50.318ZDownstairs last five completed:
123622024-02-28T22:17:50.319ZUpstairs last five completed: 30 29 28 27 26
123632024-02-28T22:17:50.319Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:49.1994133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1591}
123642024-02-28T22:17:50.319ZWrote out file "/tmp/test_repair_verify.data"
123652024-02-28T22:17:50.323Z
123662024-02-28T22:17:50.324ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123672024-02-28T22:17:50.335ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123682024-02-28T22:17:50.335Z 0 000-019 102 102 102 185 185 185 F F F
123692024-02-28T22:17:50.335Z 1 020-039 104 104 104 188 188 188 F F F
123702024-02-28T22:17:50.335Z 2 040-059 114 114 113 206 206 204 F F F <---
123712024-02-28T22:17:50.335Z 3 060-079 114 114 113 206 206 204 F F F <---
123722024-02-28T22:17:50.336Z 4 080-099 113 113 113 204 204 204 T T F <---
123732024-02-28T22:17:50.336Z 5 100-119 113 113 113 204 204 204 F F F
123742024-02-28T22:17:50.336Z 6 120-139 114 114 108 206 206 198 F F F <---
123752024-02-28T22:17:50.340Z 7 140-159 114 114 96 206 206 175 F F F <---
123762024-02-28T22:17:50.340Z 8 160-179 112 112 112 203 203 203 F F F
123772024-02-28T22:17:50.340Z 9 180-199 114 114 110 206 206 201 F F F <---
123782024-02-28T22:17:50.340Z 10 200-219 105 105 105 189 189 189 F F F
123792024-02-28T22:17:50.341Z 11 220-239 112 112 112 203 203 203 F F F
123802024-02-28T22:17:50.341Z 12 240-259 114 114 112 206 206 203 F F F <---
123812024-02-28T22:17:50.341Z 13 260-279 112 112 112 203 203 203 F F F
123822024-02-28T22:17:50.341Z 14 280-299 114 114 110 206 206 200 T T F <---
123832024-02-28T22:17:50.341Z 15 300-319 110 110 110 200 200 200 T T F <---
123842024-02-28T22:17:50.341Z 16 320-339 114 114 111 206 206 202 T T F <---
123852024-02-28T22:17:50.341Z 17 340-359 110 110 110 200 200 200 F F F
123862024-02-28T22:17:50.341Z 18 360-379 109 109 109 199 199 199 F F F
123872024-02-28T22:17:50.341Z 19 380-399 112 112 112 203 203 203 F F F
123882024-02-28T22:17:50.342Z 20 400-419 109 109 109 199 199 199 F F F
123892024-02-28T22:17:50.342Z 21 420-439 112 112 112 203 203 203 F F F
123902024-02-28T22:17:50.342Z 22 440-459 111 111 111 202 202 202 F F F
123912024-02-28T22:17:50.342Z 23 460-479 110 110 110 200 200 200 F F F
123922024-02-28T22:17:50.342Z 24 480-499 110 110 110 200 200 200 F F F
123932024-02-28T22:17:50.342Z 25 500-519 114 114 113 206 206 204 F F F <---
123942024-02-28T22:17:50.342Z 26 520-539 114 114 113 206 206 204 F F F <---
123952024-02-28T22:17:50.343Z 27 540-559 108 108 108 197 197 197 F F F
123962024-02-28T22:17:50.343Z 28 560-579 114 114 101 206 206 184 F F F <---
123972024-02-28T22:17:50.343Z 29 580-599 114 114 113 206 206 204 F F F <---
123982024-02-28T22:17:50.343ZMax gen: 114, Max flush: 206
123992024-02-28T22:17:50.343ZError: Difference in extent metadata found!
124002024-02-28T22:17:50.343ZA Difference in extent metadata is expected here
124012024-02-28T22:17:50.343ZOn loop 57
124022024-02-28T22:17:50.343Z
124032024-02-28T22:17:50.348ZVerifying data now
124042024-02-28T22:17:55.025ZNo change after verify
124052024-02-28T22:17:55.026ZLoop: 57 Downstairs dump after verify (and repair):
124062024-02-28T22:17:55.043ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124072024-02-28T22:17:55.043Z 0 000-019 102 102 102 185 185 185 F F F
124082024-02-28T22:17:55.043Z 1 020-039 104 104 104 188 188 188 F F F
124092024-02-28T22:17:55.043Z 2 040-059 114 114 114 206 206 206 F F F
124102024-02-28T22:17:55.043Z 3 060-079 114 114 114 206 206 206 F F F
124112024-02-28T22:17:55.044Z 4 080-099 115 115 115 207 207 207 F F F
124122024-02-28T22:17:55.044Z 5 100-119 113 113 113 204 204 204 F F F
124132024-02-28T22:17:55.044Z 6 120-139 114 114 114 206 206 206 F F F
124142024-02-28T22:17:55.044Z 7 140-159 114 114 114 206 206 206 F F F
124152024-02-28T22:17:55.044Z 8 160-179 112 112 112 203 203 203 F F F
124162024-02-28T22:17:55.044Z 9 180-199 114 114 114 206 206 206 F F F
124172024-02-28T22:17:55.044Z 10 200-219 105 105 105 189 189 189 F F F
124182024-02-28T22:17:55.045Z 11 220-239 112 112 112 203 203 203 F F F
124192024-02-28T22:17:55.045Z 12 240-259 114 114 114 206 206 206 F F F
124202024-02-28T22:17:55.045Z 13 260-279 112 112 112 203 203 203 F F F
124212024-02-28T22:17:55.045Z 14 280-299 115 115 115 207 207 207 F F F
124222024-02-28T22:17:55.045Z 15 300-319 115 115 115 207 207 207 F F F
124232024-02-28T22:17:55.045Z 16 320-339 115 115 115 207 207 207 F F F
124242024-02-28T22:17:55.045Z 17 340-359 110 110 110 200 200 200 F F F
124252024-02-28T22:17:55.045Z 18 360-379 109 109 109 199 199 199 F F F
124262024-02-28T22:17:55.046Z 19 380-399 112 112 112 203 203 203 F F F
124272024-02-28T22:17:55.046Z 20 400-419 109 109 109 199 199 199 F F F
124282024-02-28T22:17:55.046Z 21 420-439 112 112 112 203 203 203 F F F
124292024-02-28T22:17:55.046Z 22 440-459 111 111 111 202 202 202 F F F
124302024-02-28T22:17:55.046Z 23 460-479 110 110 110 200 200 200 F F F
124312024-02-28T22:17:55.046Z 24 480-499 110 110 110 200 200 200 F F F
124322024-02-28T22:17:55.046Z 25 500-519 114 114 114 206 206 206 F F F
124332024-02-28T22:17:55.046Z 26 520-539 114 114 114 206 206 206 F F F
124342024-02-28T22:17:55.046Z 27 540-559 108 108 108 197 197 197 F F F
124352024-02-28T22:17:55.046Z 28 560-579 114 114 114 206 206 206 F F F
124362024-02-28T22:17:55.046Z 29 580-599 114 114 114 206 206 206 F F F
124372024-02-28T22:17:55.046ZMax gen: 115, Max flush: 207
124382024-02-28T22:17:55.046Z
124392024-02-28T22:17:55.047ZBegin loop 58
124402024-02-28T22:17:55.047ZDownstairs to restart: 1
124412024-02-28T22:17:55.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948182556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
124422024-02-28T22:17:55.062Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948483858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
124432024-02-28T22:17:55.062ZCrucible runtime is spawned
124442024-02-28T22:17:55.089Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948539076Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
124452024-02-28T22:17:55.089Z{"msg":"Crucible 5810d6b5-4989-454b-91fb-1138780fc702 has session id: 92e16066-3056-4d45-899a-11659bae84d6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948586585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124462024-02-28T22:17:55.090Z{"msg":"Upstairs opts: Upstairs UUID: 5810d6b5-4989-454b-91fb-1138780fc702, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948618844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124472024-02-28T22:17:55.090Z{"msg":"Crucible stats registered with UUID: 5810d6b5-4989-454b-91fb-1138780fc702","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948663302Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124482024-02-28T22:17:55.090Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948693602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
124492024-02-28T22:17:55.090Z{"msg":"5810d6b5-4989-454b-91fb-1138780fc702 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.94874706Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124502024-02-28T22:17:55.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948787619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124512024-02-28T22:17:55.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948816138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124522024-02-28T22:17:55.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948844428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124532024-02-28T22:17:55.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948865107Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124542024-02-28T22:17:55.090Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948888516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124552024-02-28T22:17:55.090Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948925965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124562024-02-28T22:17:55.090Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948944085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124572024-02-28T22:17:55.090Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.948994644Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124582024-02-28T22:17:55.091Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.949029032Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124592024-02-28T22:17:55.091Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.94912153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124602024-02-28T22:17:55.091Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.949265176Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124612024-02-28T22:17:55.091Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.949315655Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124622024-02-28T22:17:55.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.949962757Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124632024-02-28T22:17:55.091Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950004356Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124642024-02-28T22:17:55.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950030525Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124652024-02-28T22:17:55.091Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950063514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124662024-02-28T22:17:55.091Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950095883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124672024-02-28T22:17:55.091Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950129172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124682024-02-28T22:17:55.091Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950614059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124692024-02-28T22:17:55.091Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950666497Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124702024-02-28T22:17:55.092Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.950712856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124712024-02-28T22:17:55.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951145304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124722024-02-28T22:17:55.092Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951184703Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124732024-02-28T22:17:55.092Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951210792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124742024-02-28T22:17:55.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951266281Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124752024-02-28T22:17:55.092Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.95130511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124762024-02-28T22:17:55.092Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951327779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124772024-02-28T22:17:55.092Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951356698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124782024-02-28T22:17:55.092Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951377338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124792024-02-28T22:17:55.092Z{"msg":"[0]R flush_numbers[0..12]: [185, 188, 206, 206, 207, 204, 206, 206, 203, 206, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951407217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124802024-02-28T22:17:55.092Z{"msg":"[0]R generation[0..12]: [102, 104, 114, 114, 115, 113, 114, 114, 112, 114, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951445286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124812024-02-28T22:17:55.092Z{"msg":"Wait for a query_work_queue command to finish before sending IO
124822024-02-28T22:17:55.093Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951476075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124832024-02-28T22:17:55.093Z{"msg":"[1]R flush_numbers[0..12]: [185, 188, 206, 206, 207, 204, 206, 206, 203, 206, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951507524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124842024-02-28T22:17:55.093Z{"msg":"[1]R generation[0..12]: [102, 104, 114, 114, 115, 113, 114, 114, 112, 114, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951533063Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124852024-02-28T22:17:55.093Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951562443Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124862024-02-28T22:17:55.093Z{"msg":"[2]R flush_numbers[0..12]: [185, 188, 206, 206, 207, 204, 206, 206, 203, 206, 189, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951589042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124872024-02-28T22:17:55.093Z{"msg":"[2]R generation[0..12]: [102, 104, 114, 114, 115, 113, 114, 114, 112, 114, 105, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951619541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124882024-02-28T22:17:55.093Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.95164605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124892024-02-28T22:17:55.093Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951676219Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124902024-02-28T22:17:55.093Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951700779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124912024-02-28T22:17:55.093Z{"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951727538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124922024-02-28T22:17:55.093Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951750548Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"":"downstairs","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124932024-02-28T22:17:55.093Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951778437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124942024-02-28T22:17:55.093Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951799746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"0","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124952024-02-28T22:17:55.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951828755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"1","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124962024-02-28T22:17:55.094Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951852535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"client":"2","":"downstairs","session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124972024-02-28T22:17:55.094Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951879264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124982024-02-28T22:17:55.094Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951901483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
124992024-02-28T22:17:55.094Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125002024-02-28T22:17:55.094Z5810d6b5-4989-454b-91fb-1138780fc702 is now active with session: 92e16066-3056-4d45-899a-11659bae84d6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951933883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
125012024-02-28T22:17:55.094Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951956092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599,"session_id":"92e16066-3056-4d45-899a-11659bae84d6"}
125022024-02-28T22:17:55.094Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:53.951985101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
125032024-02-28T22:17:55.094ZLoading write count information from file "/tmp/test_repair_verify.data"
125042024-02-28T22:17:55.094ZRun Repair workload
125052024-02-28T22:17:55.094Z01/30 Write block 89 len 12288 data: 9 10 9
125062024-02-28T22:17:55.094Z02/30 Flush
125072024-02-28T22:17:55.101Z03/30 Write block 243 len 32768 data: 10 10 10 6 11 14 9 9
125082024-02-28T22:17:55.101Z04/30 Read block 386 len 12288
125092024-02-28T22:17:55.102Z05/30 Write block 204 len 8192 data: 7 9
125102024-02-28T22:17:55.102Z06/30 Read block 179 len 12288
125112024-02-28T22:17:55.102Z07/30 Read block 284 len 20480
125122024-02-28T22:17:55.102Z08/30 Read block 345 len 36864
125132024-02-28T22:17:55.102Z09/30 Read block 236 len 36864
125142024-02-28T22:17:55.102Z10/30 Read block 439 len 36864
125152024-02-28T22:17:55.102Z11/30 Read block 249 len 32768
125162024-02-28T22:17:55.102Z12/30 Write block 563 len 16384 data: 11 11 12 14
125172024-02-28T22:17:55.103Z13/30 Read block 119 len 4096
125182024-02-28T22:17:55.103Z14/30 Read block 143 len 4096
125192024-02-28T22:17:55.103Z15/30 Write block 489 len 32768 data: 10 11 13 12 11 11 12 12
125202024-02-28T22:17:55.103Z16/30 Read block 0 len 24576
125212024-02-28T22:17:55.103Z17/30 Read block 420 len 24576
125222024-02-28T22:17:55.103Z18/30 Read block 24 len 28672
125232024-02-28T22:17:55.103Z19/30 Flush
125242024-02-28T22:17:55.103Z20/30 Write block 200 len 4096 data: 5
125252024-02-28T22:17:55.103Z21/30 Read block 189 len 40960
125262024-02-28T22:17:55.103Z22/30 Read block 285 len 20480
125272024-02-28T22:17:55.103Z23/30 Read block 215 len 20480
125282024-02-28T22:17:55.103Z24/30 Read block 109 len 40960
125292024-02-28T22:17:55.103Z25/30 Flush
125302024-02-28T22:17:55.103Z26/30 Write block 14 len 16384 data: 9 9 10 10
125312024-02-28T22:17:55.103Z27/30 Read block 237 len 4096
125322024-02-28T22:17:55.104Z28/30 Read block 347 len 40960
125332024-02-28T22:17:55.104Z29/30 Read block 350 len 16384
125342024-02-28T22:17:55.104Z30/30 Read block 150 len 40960
125352024-02-28T22:17:55.104Z----------------------------------------------------------------
125362024-02-28T22:17:55.104Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:28
125372024-02-28T22:17:55.104ZStates: Active Active Active
125382024-02-28T22:17:55.104ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125392024-02-28T22:17:55.104Z 3 Acked 1002 Write 8 Done Done Done false
125402024-02-28T22:17:55.104Z 4 Acked 1003 Read 3 Done Done Done false
125412024-02-28T22:17:55.104Z 5 Acked 1004 Write 2 Done Done Done false
125422024-02-28T22:17:55.104Z 6 Acked 1005 Read 3 Done Done Done false
125432024-02-28T22:17:55.104Z 7 Acked 1006 Read 5 Done Done Done false
125442024-02-28T22:17:55.104Z 8 Acked 1007 Read 9 Done Done Done false
125452024-02-28T22:17:55.104Z 9 Acked 1008 Read 9 Done Done Done false
125462024-02-28T22:17:55.105Z 10 Acked 1009 Read 9 Done Sent Done false
125472024-02-28T22:17:55.105Z 11 Acked 1010 Read 8 Done Sent Done false
125482024-02-28T22:17:55.105Z 12 Acked 1011 Write 4 Done Sent Done false
125492024-02-28T22:17:55.105Z 13 Acked 1012 Read 1 Done Sent Done false
125502024-02-28T22:17:55.105Z 14 Acked 1013 Read 1 Done Sent Done false
125512024-02-28T22:17:55.105Z 15 Acked 1014 Write 8 Done Sent Done false
125522024-02-28T22:17:55.105Z 16 Acked 1015 Read 6 Done Sent Done false
125532024-02-28T22:17:55.105Z 17 Acked 1016 Read 6 Done Sent Done false
125542024-02-28T22:17:55.105Z 18 Acked 1017 Read 7 Done Sent Done false
125552024-02-28T22:17:55.106Z 19 Acked 1018 Flush 0 Done Sent Done false
125562024-02-28T22:17:55.106Z 20 Acked 1019 Write 1 Done Sent Done false
125572024-02-28T22:17:55.106Z 21 Acked 1020 Read 10 Done Sent Done false
125582024-02-28T22:17:55.106Z 22 Acked 1021 Read 5 Done Sent Done false
125592024-02-28T22:17:55.106Z 23 Acked 1022 Read 5 Done Sent Done false
125602024-02-28T22:17:55.106Z 24 Acked 1023 Read 10 Done Sent Done false
125612024-02-28T22:17:55.106Z 25 Acked 1024 Flush 0 Done Sent Done false
125622024-02-28T22:17:55.106Z 26 Acked 1025 Write 4 Done Sent Done false
125632024-02-28T22:17:55.106Z 27 Acked 1026 Read 1 Done Sent Done false
125642024-02-28T22:17:55.106Z 28 Acked 1027 Read 10 Done Sent Done false
125652024-02-28T22:17:55.106Z 29 Acked 1028 Read 4 Done Sent Done false
125662024-02-28T22:17:55.106Z 30 Acked 1029 Read 10 Done Sent Done false
125672024-02-28T22:17:55.106Z STATES DS:0 DS:1 DS:2 TOTAL
125682024-02-28T22:17:55.106Z New 0 0 0 0
125692024-02-28T22:17:55.106Z Sent 0 21 0 21
125702024-02-28T22:17:55.107Z Done 28 7 28 63
125712024-02-28T22:17:55.107Z Skipped 0 0 0 0
125722024-02-28T22:17:55.107Z Error 0 0 0 0
125732024-02-28T22:17:55.107ZLast Flush: 1024 1001 1024
125742024-02-28T22:17:55.107ZDownstairs last five completed: 1001 1000
125752024-02-28T22:17:55.107ZUpstairs last five completed: 30 29 28 27 26
125762024-02-28T22:17:55.107Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:53.982717844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1599}
125772024-02-28T22:17:55.107ZWrote out file "/tmp/test_repair_verify.data"
125782024-02-28T22:17:55.111Z
125792024-02-28T22:17:55.112ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125802024-02-28T22:17:55.118ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125812024-02-28T22:17:55.119Z 0 000-019 102 102 102 185 185 185 T F T <---
125822024-02-28T22:17:55.119Z 1 020-039 104 104 104 188 188 188 F F F
125832024-02-28T22:17:55.119Z 2 040-059 114 114 114 206 206 206 F F F
125842024-02-28T22:17:55.119Z 3 060-079 114 114 114 206 206 206 F F F
125852024-02-28T22:17:55.119Z 4 080-099 116 116 116 208 208 208 F F F
125862024-02-28T22:17:55.119Z 5 100-119 113 113 113 204 204 204 F F F
125872024-02-28T22:17:55.119Z 6 120-139 114 114 114 206 206 206 F F F
125882024-02-28T22:17:55.120Z 7 140-159 114 114 114 206 206 206 F F F
125892024-02-28T22:17:55.120Z 8 160-179 112 112 112 203 203 203 F F F
125902024-02-28T22:17:55.120Z 9 180-199 114 114 114 206 206 206 F F F
125912024-02-28T22:17:55.120Z 10 200-219 116 105 116 210 189 210 F T F <---
125922024-02-28T22:17:55.156Z 11 220-239 112 112 112 203 203 203 F F F
125932024-02-28T22:17:55.157Z 12 240-259 116 114 116 209 206 209 F T F <---
125942024-02-28T22:17:55.157Z 13 260-279 112 112 112 203 203 203 F F F
125952024-02-28T22:17:55.157Z 14 280-299 115 115 115 207 207 207 F F F
125962024-02-28T22:17:55.157Z 15 300-319 115 115 115 207 207 207 F F F
125972024-02-28T22:17:55.157Z 16 320-339 115 115 115 207 207 207 F F F
125982024-02-28T22:17:55.157Z 17 340-359 110 110 110 200 200 200 F F F
125992024-02-28T22:17:55.157Z 18 360-379 109 109 109 199 199 199 F F F
126002024-02-28T22:17:55.157Z 19 380-399 112 112 112 203 203 203 F F F
126012024-02-28T22:17:55.158Z 20 400-419 109 109 109 199 199 199 F F F
126022024-02-28T22:17:55.158Z 21 420-439 112 112 112 203 203 203 F F F
126032024-02-28T22:17:55.158Z 22 440-459 111 111 111 202 202 202 F F F
126042024-02-28T22:17:55.158Z 23 460-479 110 110 110 200 200 200 F F F
126052024-02-28T22:17:55.158Z 24 480-499 116 110 116 209 200 209 F F F <---
126062024-02-28T22:17:55.158Z 25 500-519 114 114 114 206 206 206 F F F
126072024-02-28T22:17:55.158Z 26 520-539 114 114 114 206 206 206 F F F
126082024-02-28T22:17:55.158Z 27 540-559 108 108 108 197 197 197 F F F
126092024-02-28T22:17:55.158Z 28 560-579 116 114 116 209 206 209 F F F <---
126102024-02-28T22:17:55.158Z 29 580-599 114 114 114 206 206 206 F F F
126112024-02-28T22:17:55.158ZMax gen: 116, Max flush: 210
126122024-02-28T22:17:55.159ZError: Difference in extent metadata found!
126132024-02-28T22:17:55.159ZA Difference in extent metadata is expected here
126142024-02-28T22:17:55.159ZOn loop 58
126152024-02-28T22:17:55.159Z
126162024-02-28T22:17:55.159ZVerifying data now
126172024-02-28T22:17:59.439ZNo change after verify
126182024-02-28T22:17:59.439ZLoop: 58 Downstairs dump after verify (and repair):
126192024-02-28T22:17:59.453ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126202024-02-28T22:17:59.453Z 0 000-019 117 117 117 211 211 211 F F F
126212024-02-28T22:17:59.453Z 1 020-039 104 104 104 188 188 188 F F F
126222024-02-28T22:17:59.454Z 2 040-059 114 114 114 206 206 206 F F F
126232024-02-28T22:17:59.454Z 3 060-079 114 114 114 206 206 206 F F F
126242024-02-28T22:17:59.454Z 4 080-099 116 116 116 208 208 208 F F F
126252024-02-28T22:17:59.454Z 5 100-119 113 113 113 204 204 204 F F F
126262024-02-28T22:17:59.454Z 6 120-139 114 114 114 206 206 206 F F F
126272024-02-28T22:17:59.454Z 7 140-159 114 114 114 206 206 206 F F F
126282024-02-28T22:17:59.454Z 8 160-179 112 112 112 203 203 203 F F F
126292024-02-28T22:17:59.454Z 9 180-199 114 114 114 206 206 206 F F F
126302024-02-28T22:17:59.455Z 10 200-219 116 116 116 210 210 210 F F F
126312024-02-28T22:17:59.455Z 11 220-239 112 112 112 203 203 203 F F F
126322024-02-28T22:17:59.455Z 12 240-259 116 116 116 209 209 209 F F F
126332024-02-28T22:17:59.455Z 13 260-279 112 112 112 203 203 203 F F F
126342024-02-28T22:17:59.461Z 14 280-299 115 115 115 207 207 207 F F F
126352024-02-28T22:17:59.461Z 15 300-319 115 115 115 207 207 207 F F F
126362024-02-28T22:17:59.461Z 16 320-339 115 115 115 207 207 207 F F F
126372024-02-28T22:17:59.461Z 17 340-359 110 110 110 200 200 200 F F F
126382024-02-28T22:17:59.461Z 18 360-379 109 109 109 199 199 199 F F F
126392024-02-28T22:17:59.461Z 19 380-399 112 112 112 203 203 203 F F F
126402024-02-28T22:17:59.461Z 20 400-419 109 109 109 199 199 199 F F F
126412024-02-28T22:17:59.461Z 21 420-439 112 112 112 203 203 203 F F F
126422024-02-28T22:17:59.461Z 22 440-459 111 111 111 202 202 202 F F F
126432024-02-28T22:17:59.461Z 23 460-479 110 110 110 200 200 200 F F F
126442024-02-28T22:17:59.461Z 24 480-499 116 116 116 209 209 209 F F F
126452024-02-28T22:17:59.461Z 25 500-519 114 114 114 206 206 206 F F F
126462024-02-28T22:17:59.461Z 26 520-539 114 114 114 206 206 206 F F F
126472024-02-28T22:17:59.461Z 27 540-559 108 108 108 197 197 197 F F F
126482024-02-28T22:17:59.461Z 28 560-579 116 116 116 209 209 209 F F F
126492024-02-28T22:17:59.461Z 29 580-599 114 114 114 206 206 206 F F F
126502024-02-28T22:17:59.461ZMax gen: 117, Max flush: 211
126512024-02-28T22:17:59.462Z
126522024-02-28T22:17:59.462ZBegin loop 59
126532024-02-28T22:17:59.462ZDownstairs to restart: 0
126542024-02-28T22:17:59.475Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362161926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
126552024-02-28T22:17:59.475Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362421258Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
126562024-02-28T22:17:59.476Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362439538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
126572024-02-28T22:17:59.483Z{"msg":"Crucible 292d6147-c508-459d-99da-d8c13e204a2b has session id: 9cb50baf-0e2a-4dce-a772-3f709368ddbf","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362473167Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126582024-02-28T22:17:59.483ZCrucible runtime is spawned
126592024-02-28T22:17:59.484Z{"msg":"Upstairs opts: Upstairs UUID: 292d6147-c508-459d-99da-d8c13e204a2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362532545Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126602024-02-28T22:17:59.484Z{"msg":"Crucible stats registered with UUID: 292d6147-c508-459d-99da-d8c13e204a2b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362552455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126612024-02-28T22:17:59.484Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362587624Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
126622024-02-28T22:17:59.484Z{"msg":"292d6147-c508-459d-99da-d8c13e204a2b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362750209Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126632024-02-28T22:17:59.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362809658Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126642024-02-28T22:17:59.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362828017Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126652024-02-28T22:17:59.484Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362845377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126662024-02-28T22:17:59.484Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362881316Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126672024-02-28T22:17:59.485Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362903195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126682024-02-28T22:17:59.485Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362935424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126692024-02-28T22:17:59.485Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362951314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126702024-02-28T22:17:59.485Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.362990903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126712024-02-28T22:17:59.485Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363071001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126722024-02-28T22:17:59.485Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363123079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126732024-02-28T22:17:59.486Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363162158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126742024-02-28T22:17:59.486Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363284164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126752024-02-28T22:17:59.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363902118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126762024-02-28T22:17:59.486Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363946296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126772024-02-28T22:17:59.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363965366Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126782024-02-28T22:17:59.486Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.363989925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126792024-02-28T22:17:59.486Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.364028524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126802024-02-28T22:17:59.486Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.364057923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126812024-02-28T22:17:59.487Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.364514661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126822024-02-28T22:17:59.487Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.364569589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126832024-02-28T22:17:59.487Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.364622478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126842024-02-28T22:17:59.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365063235Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126852024-02-28T22:17:59.487Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365108524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126862024-02-28T22:17:59.487Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365130494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126872024-02-28T22:17:59.487Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365167303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126882024-02-28T22:17:59.488Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365183932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126892024-02-28T22:17:59.488Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365215091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126902024-02-28T22:17:59.488Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365281949Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126912024-02-28T22:17:59.488Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365310659Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126922024-02-28T22:17:59.488Z{"msg":"[0]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 206, 206, 203, 206, 210, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365326488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126932024-02-28T22:17:59.488Z{"msg":"[0]R generation[0..12]: [117, 104, 114, 114, 116, 113, 114, 114, 112, 114, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365359477Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126942024-02-28T22:17:59.488Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365392517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126952024-02-28T22:17:59.488Z{"msg":"[1]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 206, 206, 203, 206, 210, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365425865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126962024-02-28T22:17:59.488ZWait for a query_work_queue command to finish before sending IO
126972024-02-28T22:17:59.489Z{"msg":"[1]R generation[0..12]: [117, 104, 114, 114, 116, 113, 114, 114, 112, 114, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365460455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126982024-02-28T22:17:59.489Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365495374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
126992024-02-28T22:17:59.489Z{"msg":"[2]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 206, 206, 203, 206, 210, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365513823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127002024-02-28T22:17:59.489Z{"msg":"[2]R generation[0..12]: [117, 104, 114, 114, 116, 113, 114, 114, 112, 114, 116, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365545932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127012024-02-28T22:17:59.489Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365566152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127022024-02-28T22:17:59.489Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365594941Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127032024-02-28T22:17:59.489Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.36561098Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127042024-02-28T22:17:59.489Z{"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365658939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127052024-02-28T22:17:59.489Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365678069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"":"downstairs","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127062024-02-28T22:17:59.490Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365707758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127072024-02-28T22:17:59.490Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365724027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"0","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127082024-02-28T22:17:59.490Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365753707Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"1","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127092024-02-28T22:17:59.490Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365774286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"client":"2","":"downstairs","session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127102024-02-28T22:17:59.490Z{"msg":"All required reconciliation work is completed","v":0,"name"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127112024-02-28T22:17:59.490Z:"crucible","level":30,"time":"2024-02-28T22:17:58.365807065Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127122024-02-28T22:17:59.490Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365836174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127132024-02-28T22:17:59.490Z{"msg":"292d6147-c508-459d-99da-d8c13e204a2b is now active with session: 9cb50baf-0e2a-4dce-a772-3f709368ddbf","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365868273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127142024-02-28T22:17:59.491Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365897463Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607,"session_id":"9cb50baf-0e2a-4dce-a772-3f709368ddbf"}
127152024-02-28T22:17:59.491Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:17:58.365928942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
127162024-02-28T22:17:59.491ZLoading write count information from file "/tmp/test_repair_verify.data"
127172024-02-28T22:17:59.491ZRun Repair workload
127182024-02-28T22:17:59.491Z01/30 Flush
127192024-02-28T22:17:59.496Z02/30 Write block 201 len 32768 data: 7 7 6 8 10 7 6 7
127202024-02-28T22:17:59.496Z03/30 Read block 419 len 8192
127212024-02-28T22:17:59.496Z04/30 Read block 545 len 8192
127222024-02-28T22:17:59.496Z05/30 Flush
127232024-02-28T22:17:59.497Z06/30 Flush
127242024-02-28T22:17:59.497Z07/30 Write block 126 len 16384 data: 10 14 15 12
127252024-02-28T22:17:59.497Z08/30 Read block 54 len 20480
127262024-02-28T22:17:59.497Z09/30 Flush
127272024-02-28T22:17:59.497Z10/30 Write block 331 len 40960 data: 13 9 10 11 11 11 9 10 10 9
127282024-02-28T22:17:59.498Z11/30 Read block 505 len 20480
127292024-02-28T22:17:59.498Z12/30 Flush
127302024-02-28T22:17:59.498Z13/30 Write block 429 len 32768 data: 7 7 6 8 9 10 10 11
127312024-02-28T22:17:59.498Z14/30 Write block 471 len 12288 data: 10 12 9
127322024-02-28T22:17:59.504Z15/30 Read block 201 len 4096
127332024-02-28T22:17:59.504Z16/30 Read block 131 len 4096
127342024-02-28T22:17:59.504Z17/30 Write block 340 len 8192 data: 10 10
127352024-02-28T22:17:59.505Z18/30 Read block 417 len 32768
127362024-02-28T22:17:59.505Z19/30 Write block 152 len 20480 data: 6 4 5 7 5
127372024-02-28T22:17:59.505Z20/30 Read block 61 len 4096
127382024-02-28T22:17:59.505Z21/30 Read block 237 len 8192
127392024-02-28T22:17:59.505Z22/30 Write block 429 len 36864 data: 8 8 7 9 10 11 11 12 11
127402024-02-28T22:17:59.505Z23/30 Read block 411 len 40960
127412024-02-28T22:17:59.509Z24/30 Write block 184 len 28672 data: 11 11 8 9 9 9 8
127422024-02-28T22:17:59.510Z25/30 Read block 510 len 4096
127432024-02-28T22:17:59.510Z26/30 Write block 440 len 4096 data: 9
127442024-02-28T22:17:59.510Z27/30 Write block 557 len 20480 data: 7 7 7 9 11
127452024-02-28T22:17:59.510Z28/30 Read block 290 len 28672
127462024-02-28T22:17:59.510Z29/30 Write block 482 len 32768 data: 7 7 8 11 8 8 6 11
127472024-02-28T22:17:59.510Z30/30 Read block 350 len 12288
127482024-02-28T22:17:59.515Z----------------------------------------------------------------
127492024-02-28T22:17:59.516Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:30
127502024-02-28T22:17:59.516ZStates: Active Active Active
127512024-02-28T22:17:59.516ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127522024-02-28T22:17:59.516Z 1 Acked 1000 Flush 0 Sent Done Done false
127532024-02-28T22:17:59.516Z 2 Acked 1001 Write 8 Sent Done Done false
127542024-02-28T22:17:59.516Z 3 Acked 1002 Read 2 Sent Done Done false
127552024-02-28T22:17:59.517Z 4 Acked 1003 Read 2 Sent Done Done false
127562024-02-28T22:17:59.517Z 5 Acked 1004 Flush 0 Sent Done Done false
127572024-02-28T22:17:59.517Z 6 Acked 1005 Flush 0 Sent Done Done false
127582024-02-28T22:17:59.517Z 7 Acked 1006 Write 4 Sent Done Done false
127592024-02-28T22:17:59.517Z 8 Acked 1007 Read 5 Sent Done Done false
127602024-02-28T22:17:59.518Z 9 Acked 1008 Flush 0 Sent Done Done false
127612024-02-28T22:17:59.518Z 10 Acked 1009 Write 10 Sent Done Done false
127622024-02-28T22:17:59.518Z 11 Acked 1010 Read 5 Sent Done Done false
127632024-02-28T22:17:59.518Z 12 Acked 1011 Flush 0 Sent Done Done false
127642024-02-28T22:17:59.518Z 13 Acked 1012 Write 8 Sent Done Done false
127652024-02-28T22:17:59.518Z 14 Acked 1013 Write 3 Sent Done Done false
127662024-02-28T22:17:59.519Z 15 Acked 1014 Read 1 Sent Done Done false
127672024-02-28T22:17:59.519Z 16 Acked 1015 Read 1 Sent Done Done false
127682024-02-28T22:17:59.519Z 17 Acked 1016 Write 2 Sent Done Done false
127692024-02-28T22:17:59.519Z 18 Acked 1017 Read 8 Sent Done Done false
127702024-02-28T22:17:59.519Z 19 Acked 1018 Write 5 Sent Done Done false
127712024-02-28T22:17:59.519Z 20 Acked 1019 Read 1 Sent Done Done false
127722024-02-28T22:17:59.520Z 21 Acked 1020 Read 2 Sent Done Done false
127732024-02-28T22:17:59.520Z 22 Acked 1021 Write 9 Sent Done Done false
127742024-02-28T22:17:59.520Z 23 Acked 1022 Read 10 Sent Done Done false
127752024-02-28T22:17:59.520Z 24 Acked 1023 Write 7 Sent Done Done false
127762024-02-28T22:17:59.520Z 25 Acked 1024 Read 1 Sent Done Done false
127772024-02-28T22:17:59.520Z 26 Acked 1025 Write 1 Sent Done Done false
127782024-02-28T22:17:59.521Z 27 Acked 1026 Write 5 Sent Done Done false
127792024-02-28T22:17:59.521Z 28 Acked 1027 Read 7 Sent Done Done false
127802024-02-28T22:17:59.521Z 29 Acked 1028 Write 8 Sent Done Done false
127812024-02-28T22:17:59.521Z 30 Acked 1029 Read 3 Sent Done Done false
127822024-02-28T22:17:59.521Z STATES DS:0 DS:1 DS:2 TOTAL
127832024-02-28T22:17:59.522Z New 0 0 0 0
127842024-02-28T22:17:59.522Z Sent 30 0 0 30
127852024-02-28T22:17:59.522Z Done 0 30 30 60
127862024-02-28T22:17:59.522Z Skipped 0 0 0 0
127872024-02-28T22:17:59.522Z Error 0 0 0 0
127882024-02-28T22:17:59.522ZLast Flush: 0 1011 1011
127892024-02-28T22:17:59.523ZDownstairs last five completed:
127902024-02-28T22:17:59.523ZUpstairs last five completed: 30 29 28 27 26
127912024-02-28T22:17:59.523Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:17:58.397432414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1607}
127922024-02-28T22:17:59.523ZWrote out file "/tmp/test_repair_verify.data"
127932024-02-28T22:17:59.523Z
127942024-02-28T22:17:59.529ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127952024-02-28T22:17:59.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127962024-02-28T22:17:59.534Z 0 000-019 117 117 117 211 211 211 F F F
127972024-02-28T22:17:59.535Z 1 020-039 104 104 104 188 188 188 F F F
127982024-02-28T22:17:59.535Z 2 040-059 114 114 114 206 206 206 F F F
127992024-02-28T22:17:59.535Z 3 060-079 114 114 114 206 206 206 F F F
128002024-02-28T22:17:59.540Z 4 080-099 116 116 116 208 208 208 F F F
128012024-02-28T22:17:59.540Z 5 100-119 113 113 113 204 204 204 F F F
128022024-02-28T22:17:59.540Z 6 120-139 114 118 118 206 215 215 F F F <---
128032024-02-28T22:17:59.541Z 7 140-159 114 114 114 206 206 206 F T T <---
128042024-02-28T22:17:59.541Z 8 160-179 112 112 112 203 203 203 F F F
128052024-02-28T22:17:59.541Z 9 180-199 114 114 114 206 206 206 F T T <---
128062024-02-28T22:17:59.541Z 10 200-219 116 118 118 210 213 213 F F F <---
128072024-02-28T22:17:59.541Z 11 220-239 112 112 112 203 203 203 F F F
128082024-02-28T22:17:59.541Z 12 240-259 116 116 116 209 209 209 F F F
128092024-02-28T22:17:59.542Z 13 260-279 112 112 112 203 203 203 F F F
128102024-02-28T22:17:59.542Z 14 280-299 115 115 115 207 207 207 F F F
128112024-02-28T22:17:59.542Z 15 300-319 115 115 115 207 207 207 F F F
128122024-02-28T22:17:59.542Z 16 320-339 115 118 118 207 216 216 F F F <---
128132024-02-28T22:17:59.542Z 17 340-359 110 118 118 200 216 216 F T T <---
128142024-02-28T22:17:59.542Z 18 360-379 109 109 109 199 199 199 F F F
128152024-02-28T22:17:59.542Z 19 380-399 112 112 112 203 203 203 F F F
128162024-02-28T22:17:59.543Z 20 400-419 109 109 109 199 199 199 F F F
128172024-02-28T22:17:59.543Z 21 420-439 112 112 112 203 203 203 F T T <---
128182024-02-28T22:17:59.543Z 22 440-459 111 111 111 202 202 202 F T T <---
128192024-02-28T22:17:59.543Z 23 460-479 110 110 110 200 200 200 F T T <---
128202024-02-28T22:17:59.543Z 24 480-499 116 116 116 209 209 209 F T T <---
128212024-02-28T22:17:59.543Z 25 500-519 114 114 114 206 206 206 F F F
128222024-02-28T22:17:59.543Z 26 520-539 114 114 114 206 206 206 F F F
128232024-02-28T22:17:59.544Z 27 540-559 108 108 108 197 197 197 F T T <---
128242024-02-28T22:17:59.544Z 28 560-579 116 116 116 209 209 209 F T T <---
128252024-02-28T22:17:59.544Z 29 580-599 114 114 114 206 206 206 F F F
128262024-02-28T22:17:59.544ZMax gen: 118, Max flush: 216
128272024-02-28T22:17:59.544ZError: Difference in extent metadata found!
128282024-02-28T22:17:59.544ZA Difference in extent metadata is expected here
128292024-02-28T22:17:59.544ZOn loop 59
128302024-02-28T22:17:59.544Z
128312024-02-28T22:17:59.549ZVerifying data now
128322024-02-28T22:18:04.160ZNo change after verify
128332024-02-28T22:18:04.160ZLoop: 59 Downstairs dump after verify (and repair):
128342024-02-28T22:18:04.174ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128352024-02-28T22:18:04.174Z 0 000-019 117 117 117 211 211 211 F F F
128362024-02-28T22:18:04.174Z 1 020-039 104 104 104 188 188 188 F F F
128372024-02-28T22:18:04.175Z 2 040-059 114 114 114 206 206 206 F F F
128382024-02-28T22:18:04.175Z 3 060-079 114 114 114 206 206 206 F F F
128392024-02-28T22:18:04.175Z 4 080-099 116 116 116 208 208 208 F F F
128402024-02-28T22:18:04.175Z 5 100-119 113 113 113 204 204 204 F F F
128412024-02-28T22:18:04.175Z 6 120-139 118 118 118 215 215 215 F F F
128422024-02-28T22:18:04.175Z 7 140-159 119 119 119 217 217 217 F F F
128432024-02-28T22:18:04.175Z 8 160-179 112 112 112 203 203 203 F F F
128442024-02-28T22:18:04.175Z 9 180-199 119 119 119 217 217 217 F F F
128452024-02-28T22:18:04.175Z 10 200-219 118 118 118 213 213 213 F F F
128462024-02-28T22:18:04.175Z 11 220-239 112 112 112 203 203 203 F F F
128472024-02-28T22:18:04.176Z 12 240-259 116 116 116 209 209 209 F F F
128482024-02-28T22:18:04.176Z 13 260-279 112 112 112 203 203 203 F F F
128492024-02-28T22:18:04.176Z 14 280-299 115 115 115 207 207 207 F F F
128502024-02-28T22:18:04.180Z 15 300-319 115 115 115 207 207 207 F F F
128512024-02-28T22:18:04.180Z 16 320-339 118 118 118 216 216 216 F F F
128522024-02-28T22:18:04.180Z 17 340-359 119 119 119 217 217 217 F F F
128532024-02-28T22:18:04.180Z 18 360-379 109 109 109 199 199 199 F F F
128542024-02-28T22:18:04.180Z 19 380-399 112 112 112 203 203 203 F F F
128552024-02-28T22:18:04.180Z 20 400-419 109 109 109 199 199 199 F F F
128562024-02-28T22:18:04.180Z 21 420-439 119 119 119 217 217 217 F F F
128572024-02-28T22:18:04.180Z 22 440-459 119 119 119 217 217 217 F F F
128582024-02-28T22:18:04.181Z 23 460-479 119 119 119 217 217 217 F F F
128592024-02-28T22:18:04.181Z 24 480-499 119 119 119 217 217 217 F F F
128602024-02-28T22:18:04.181Z 25 500-519 114 114 114 206 206 206 F F F
128612024-02-28T22:18:04.181Z 26 520-539 114 114 114 206 206 206 F F F
128622024-02-28T22:18:04.181Z 27 540-559 119 119 119 217 217 217 F F F
128632024-02-28T22:18:04.181Z 28 560-579 119 119 119 217 217 217 F F F
128642024-02-28T22:18:04.181Z 29 580-599 114 114 114 206 206 206 F F F
128652024-02-28T22:18:04.181ZMax gen: 119, Max flush: 217
128662024-02-28T22:18:04.181Z
128672024-02-28T22:18:04.181ZBegin loop 60
128682024-02-28T22:18:04.181ZDownstairs to restart: 2
128692024-02-28T22:18:04.196Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08231053Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
128702024-02-28T22:18:04.196Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082553273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
128712024-02-28T22:18:04.203Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082579043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
128722024-02-28T22:18:04.204Z{"msg":"Crucible 0e708b10-6803-4c0b-b382-7e5a72d2e909 has session id: 910c68fc-d121-4c75-8d72-7c567f77de93","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082610722Z","hostname"Crucible runtime is spawned
128732024-02-28T22:18:04.204Z:"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128742024-02-28T22:18:04.204Z{"msg":"Upstairs opts: Upstairs UUID: 0e708b10-6803-4c0b-b382-7e5a72d2e909, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08266434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128752024-02-28T22:18:04.204Z{"msg":"Crucible stats registered with UUID: 0e708b10-6803-4c0b-b382-7e5a72d2e909","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082698689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128762024-02-28T22:18:04.205Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082723139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
128772024-02-28T22:18:04.205Z{"msg":"0e708b10-6803-4c0b-b382-7e5a72d2e909 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082878504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128782024-02-28T22:18:04.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082923083Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128792024-02-28T22:18:04.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082947642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128802024-02-28T22:18:04.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.082981341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128812024-02-28T22:18:04.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083014551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128822024-02-28T22:18:04.205Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08303775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128832024-02-28T22:18:04.206Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083062329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128842024-02-28T22:18:04.206Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083094058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128852024-02-28T22:18:04.206Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083138377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128862024-02-28T22:18:04.206Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083174566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128872024-02-28T22:18:04.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083263064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128882024-02-28T22:18:04.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083303012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128892024-02-28T22:18:04.206Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.083333582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128902024-02-28T22:18:04.206Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084039022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128912024-02-28T22:18:04.206Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084079771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128922024-02-28T22:18:04.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084182538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128932024-02-28T22:18:04.207Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084221277Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128942024-02-28T22:18:04.207Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084238587Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128952024-02-28T22:18:04.207Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084276736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128962024-02-28T22:18:04.207Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084549048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128972024-02-28T22:18:04.207Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084630966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128982024-02-28T22:18:04.207Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.084901839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
128992024-02-28T22:18:04.207Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085078794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129002024-02-28T22:18:04.207Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085117913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129012024-02-28T22:18:04.207Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085153601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129022024-02-28T22:18:04.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085170101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129032024-02-28T22:18:04.208Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085193471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129042024-02-28T22:18:04.208Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08520806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129052024-02-28T22:18:04.208Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085465863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129062024-02-28T22:18:04.208Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085492972Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129072024-02-28T22:18:04.208Z{"msg":"[0]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 215, 217, 203, 217, 213, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085508422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129082024-02-28T22:18:04.208Z{"msg":"[0]R generation[0..12]: [117, 104, 114, 114, 116, 113, 118, 119, 112, 119, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085523631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129092024-02-28T22:18:04.208Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085546591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129102024-02-28T22:18:04.208Z{"msg":"[1]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 215, 217, 203, 217, 213, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08556121Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129112024-02-28T22:18:04.208Z{"msg":"[1]R generation[0..12]: [117, 104, 114, 114, 116, 113, 118, 119, 112, 119, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.08557577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129122024-02-28T22:18:04.208Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085596789Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129132024-02-28T22:18:04.209Z{"msg":"[2]R flush_numbers[0..12]: [211, 188, 206, 206, 208, 204, 215, 217, 203, 217, 213, 203]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085611419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129142024-02-28T22:18:04.209Z{"msg":"[2]R generation[0..12]: [117, 104, 114, 114, 116, 113, 118, 119, 112, 119, 118, 112]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085632199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129152024-02-28T22:18:04.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085646748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129162024-02-28T22:18:04.209Z{"msg":"Wait for a query_work_queue command to finish before sending IO
129172024-02-28T22:18:04.209ZMax found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085670877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129182024-02-28T22:18:04.209Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085686547Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129192024-02-28T22:18:04.209Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085706996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129202024-02-28T22:18:04.209Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085721326Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"":"downstairs","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129212024-02-28T22:18:04.209Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085741625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129222024-02-28T22:18:04.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085763355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"0","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129232024-02-28T22:18:04.209Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085777434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"1","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129242024-02-28T22:18:04.210Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085791594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"client":"2","":"downstairs","session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129252024-02-28T22:18:04.210Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085820753Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129262024-02-28T22:18:04.210Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085837633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129272024-02-28T22:18:04.210Z{"msg":"0e708b10-6803-4c0b-b382-7e5a72d2e909 is now active with session: 910c68fc-d121-4c75-8d72-7c567f77de93","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085862032Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129282024-02-28T22:18:04.210Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085877322Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615,"session_id":"910c68fc-d121-4c75-8d72-7c567f77de93"}
129292024-02-28T22:18:04.210Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:03.085890081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
129302024-02-28T22:18:04.210ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129312024-02-28T22:18:04.210ZLoading write count information from file "/tmp/test_repair_verify.data"
129322024-02-28T22:18:04.210ZRun Repair workload
129332024-02-28T22:18:04.211Z01/30 Write block 128 len 8192 data: 16 13
129342024-02-28T22:18:04.218Z02/30 Read block 101 len 24576
129352024-02-28T22:18:04.218Z03/30 Flush
129362024-02-28T22:18:04.218Z04/30 Write block 513 len 24576 data: 11 11 13 13 12 11
129372024-02-28T22:18:04.218Z05/30 Read block 359 len 32768
129382024-02-28T22:18:04.218Z06/30 Write block 245 len 12288 data: 11 7 12
129392024-02-28T22:18:04.218Z07/30 Read block 43 len 12288
129402024-02-28T22:18:04.218Z08/30 Read block 418 len 36864
129412024-02-28T22:18:04.218Z09/30 Flush
129422024-02-28T22:18:04.218Z10/30 Write block 280 len 36864 data: 8 7 8 7 9 8 10 7 8
129432024-02-28T22:18:04.218Z11/30 Write block 227 len 4096 data: 7
129442024-02-28T22:18:04.218Z12/30 Read block 488 len 32768
129452024-02-28T22:18:04.218Z13/30 Write block 81 len 24576 data: 10 13 11 9 9 8
129462024-02-28T22:18:04.219Z14/30 Read block 307 len 36864
129472024-02-28T22:18:04.219Z15/30 Read block 444 len 40960
129482024-02-28T22:18:04.222Z16/30 Read block 188 len 12288
129492024-02-28T22:18:04.223Z17/30 Flush
129502024-02-28T22:18:04.223Z18/30 Write block 138 len 20480 data: 13 11 10 9 10
129512024-02-28T22:18:04.223Z19/30 Read block 360 len 20480
129522024-02-28T22:18:04.223Z20/30 Flush
129532024-02-28T22:18:04.228Z21/30 Write block 2 len 12288 data: 6 8 10
129542024-02-28T22:18:04.228Z22/30 Flush
129552024-02-28T22:18:04.228Z23/30 Flush
129562024-02-28T22:18:04.228Z24/30 Write block 36 len 16384 data: 10 11 12 11
129572024-02-28T22:18:04.228Z25/30 Read block 92 len 24576
129582024-02-28T22:18:04.232Z26/30 Write block 109 len 32768 data: 16 14 12 12 9 10 10 10
129592024-02-28T22:18:04.232Z27/30 Read block 471 len 36864
129602024-02-28T22:18:04.232Z28/30 Write block 162 len 28672 data: 6 6 7 8 9 13 12
129612024-02-28T22:18:04.232Z29/30 Write block 157 len 36864 data: 6 6 7 5 6 7 7 8 9
129622024-02-28T22:18:04.233Z30/30 Read block 212 len 4096
129632024-02-28T22:18:04.238Z----------------------------------------------------------------
129642024-02-28T22:18:04.238Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129652024-02-28T22:18:04.238ZStates: Active Active Active
129662024-02-28T22:18:04.239ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129672024-02-28T22:18:04.239Z 1 Acked 1000 Write 2 Done Done Done false
129682024-02-28T22:18:04.239Z 2 Acked 1001 Read 6 Done Done Sent false
129692024-02-28T22:18:04.239Z 3 Acked 1002 Flush 0 Done Done Sent false
129702024-02-28T22:18:04.239Z 4 Acked 1003 Write 6 Done Done Sent false
129712024-02-28T22:18:04.239Z 5 Acked 1004 Read 8 Done Done Sent false
129722024-02-28T22:18:04.239Z 6 Acked 1005 Write 3 Done Done Sent false
129732024-02-28T22:18:04.239Z 7 Acked 1006 Read 3 Done Done Sent false
129742024-02-28T22:18:04.239Z 8 Acked 1007 Read 9 Done Done Sent false
129752024-02-28T22:18:04.240Z 9 Acked 1008 Flush 0 Done Done Sent false
129762024-02-28T22:18:04.240Z 10 Acked 1009 Write 9 Done Done Sent false
129772024-02-28T22:18:04.240Z 11 Acked 1010 Write 1 Done Done Sent false
129782024-02-28T22:18:04.240Z 12 Acked 1011 Read 8 Done Done Sent false
129792024-02-28T22:18:04.240Z 13 Acked 1012 Write 6 Done Done Sent false
129802024-02-28T22:18:04.241Z 14 Acked 1013 Read 9 Done Done Sent false
129812024-02-28T22:18:04.241Z 15 Acked 1014 Read 10 Done Done Sent false
129822024-02-28T22:18:04.241Z 16 Acked 1015 Read 3 Done Done Sent false
129832024-02-28T22:18:04.241Z 17 Acked 1016 Flush 0 Done Done Sent false
129842024-02-28T22:18:04.241Z 18 Acked 1017 Write 5 Done Done Sent false
129852024-02-28T22:18:04.241Z 19 Acked 1018 Read 5 Done Done Sent false
129862024-02-28T22:18:04.241Z 20 Acked 1019 Flush 0 Done Done Sent false
129872024-02-28T22:18:04.241Z 21 Acked 1020 Write 3 Done Done Sent false
129882024-02-28T22:18:04.241Z 22 Acked 1021 Flush 0 Done Done Sent false
129892024-02-28T22:18:04.241Z 23 Acked 1022 Flush 0 Done Done Sent false
129902024-02-28T22:18:04.242Z 24 Acked 1023 Write 4 Done Done Sent false
129912024-02-28T22:18:04.242Z 25 Acked 1024 Read 6 Done Done Sent false
129922024-02-28T22:18:04.242Z 26 Acked 1025 Write 8 Done Done Sent false
129932024-02-28T22:18:04.242Z 27 Acked 1026 Read 9 Done Done Sent false
129942024-02-28T22:18:04.242Z 28 Acked 1027 Write 7 Done Done Sent false
129952024-02-28T22:18:04.242Z 29 Acked 1028 Write 9 Done Done Sent false
129962024-02-28T22:18:04.242Z 30 Acked 1029 Read 1 Done Done Sent false
129972024-02-28T22:18:04.242Z STATES DS:0 DS:1 DS:2 TOTAL
129982024-02-28T22:18:04.242Z New 0 0 0 0
129992024-02-28T22:18:04.243Z Sent 0 0 29 29
130002024-02-28T22:18:04.243Z Done 30 30 1 61
130012024-02-28T22:18:04.243Z Skipped 0 0 0 0
130022024-02-28T22:18:04.243Z Error 0 0 0 0
130032024-02-28T22:18:04.243ZLast Flush: 1022 1022 0
130042024-02-28T22:18:04.243ZDownstairs last five completed:
130052024-02-28T22:18:04.243ZUpstairs last five completed: 30 29 28 27 26
130062024-02-28T22:18:04.243Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:03.120973975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1615}
130072024-02-28T22:18:04.243ZWrote out file "/tmp/test_repair_verify.data"
130082024-02-28T22:18:04.243Z
130092024-02-28T22:18:04.249ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130102024-02-28T22:18:04.257ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130112024-02-28T22:18:04.258Z 0 000-019 120 120 117 222 222 211 F F F <---
130122024-02-28T22:18:04.258Z 1 020-039 104 104 104 188 188 188 T T F <---
130132024-02-28T22:18:04.258Z 2 040-059 114 114 114 206 206 206 F F F
130142024-02-28T22:18:04.258Z 3 060-079 114 114 114 206 206 206 F F F
130152024-02-28T22:18:04.258Z 4 080-099 120 120 116 220 220 208 F F F <---
130162024-02-28T22:18:04.258Z 5 100-119 113 113 113 204 204 204 T T F <---
130172024-02-28T22:18:04.258Z 6 120-139 120 120 118 221 221 215 F F T <---
130182024-02-28T22:18:04.258Z 7 140-159 120 120 119 221 221 217 T T F <---
130192024-02-28T22:18:04.258Z 8 160-179 112 112 112 203 203 203 T T F <---
130202024-02-28T22:18:04.258Z 9 180-199 119 119 119 217 217 217 F F F
130212024-02-28T22:18:04.259Z 10 200-219 118 118 118 213 213 213 F F F
130222024-02-28T22:18:04.263Z 11 220-239 120 120 112 220 220 203 F F F <---
130232024-02-28T22:18:04.263Z 12 240-259 120 120 116 219 219 209 F F F <---
130242024-02-28T22:18:04.263Z 13 260-279 112 112 112 203 203 203 F F F
130252024-02-28T22:18:04.263Z 14 280-299 120 120 115 220 220 207 F F F <---
130262024-02-28T22:18:04.263Z 15 300-319 115 115 115 207 207 207 F F F
130272024-02-28T22:18:04.263Z 16 320-339 118 118 118 216 216 216 F F F
130282024-02-28T22:18:04.264Z 17 340-359 119 119 119 217 217 217 F F F
130292024-02-28T22:18:04.264Z 18 360-379 109 109 109 199 199 199 F F F
130302024-02-28T22:18:04.264Z 19 380-399 112 112 112 203 203 203 F F F
130312024-02-28T22:18:04.264Z 20 400-419 109 109 109 199 199 199 F F F
130322024-02-28T22:18:04.264Z 21 420-439 119 119 119 217 217 217 F F F
130332024-02-28T22:18:04.264Z 22 440-459 119 119 119 217 217 217 F F F
130342024-02-28T22:18:04.264Z 23 460-479 119 119 119 217 217 217 F F F
130352024-02-28T22:18:04.264Z 24 480-499 119 119 119 217 217 217 F F F
130362024-02-28T22:18:04.264Z 25 500-519 120 120 114 219 219 206 F F F <---
130372024-02-28T22:18:04.264Z 26 520-539 114 114 114 206 206 206 F F F
130382024-02-28T22:18:04.264Z 27 540-559 119 119 119 217 217 217 F F F
130392024-02-28T22:18:04.265Z 28 560-579 119 119 119 217 217 217 F F F
130402024-02-28T22:18:04.265Z 29 580-599 114 114 114 206 206 206 F F F
130412024-02-28T22:18:04.265ZMax gen: 120, Max flush: 222
130422024-02-28T22:18:04.265ZError: Difference in extent metadata found!
130432024-02-28T22:18:04.265ZA Difference in extent metadata is expected here
130442024-02-28T22:18:04.265ZOn loop 60
130452024-02-28T22:18:04.265Z
130462024-02-28T22:18:04.269ZVerifying data now
130472024-02-28T22:18:08.835ZNo change after verify
130482024-02-28T22:18:08.835ZLoop: 60 Downstairs dump after verify (and repair):
130492024-02-28T22:18:08.850ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130502024-02-28T22:18:08.850Z 0 000-019 120 120 120 222 222 222 F F F
130512024-02-28T22:18:08.850Z 1 020-039 121 121 121 223 223 223 F F F
130522024-02-28T22:18:08.851Z 2 040-059 114 114 114 206 206 206 F F F
130532024-02-28T22:18:08.851Z 3 060-079 114 114 114 206 206 206 F F F
130542024-02-28T22:18:08.851Z 4 080-099 120 120 120 220 220 220 F F F
130552024-02-28T22:18:08.851Z 5 100-119 121 121 121 223 223 223 F F F
130562024-02-28T22:18:08.851Z 6 120-139 120 120 120 221 221 221 F F F
130572024-02-28T22:18:08.851Z 7 140-159 121 121 121 223 223 223 F F F
130582024-02-28T22:18:08.851Z 8 160-179 121 121 121 223 223 223 F F F
130592024-02-28T22:18:08.851Z 9 180-199 119 119 119 217 217 217 F F F
130602024-02-28T22:18:08.851Z 10 200-219 118 118 118 213 213 213 F F F
130612024-02-28T22:18:08.855Z 11 220-239 120 120 120 220 220 220 F F F
130622024-02-28T22:18:08.856Z 12 240-259 120 120 120 219 219 219 F F F
130632024-02-28T22:18:08.856Z 13 260-279 112 112 112 203 203 203 F F F
130642024-02-28T22:18:08.856Z 14 280-299 120 120 120 220 220 220 F F F
130652024-02-28T22:18:08.856Z 15 300-319 115 115 115 207 207 207 F F F
130662024-02-28T22:18:08.856Z 16 320-339 118 118 118 216 216 216 F F F
130672024-02-28T22:18:08.856Z 17 340-359 119 119 119 217 217 217 F F F
130682024-02-28T22:18:08.857Z 18 360-379 109 109 109 199 199 199 F F F
130692024-02-28T22:18:08.857Z 19 380-399 112 112 112 203 203 203 F F F
130702024-02-28T22:18:08.857Z 20 400-419 109 109 109 199 199 199 F F F
130712024-02-28T22:18:08.857Z 21 420-439 119 119 119 217 217 217 F F F
130722024-02-28T22:18:08.857Z 22 440-459 119 119 119 217 217 217 F F F
130732024-02-28T22:18:08.857Z 23 460-479 119 119 119 217 217 217 F F F
130742024-02-28T22:18:08.857Z 24 480-499 119 119 119 217 217 217 F F F
130752024-02-28T22:18:08.857Z 25 500-519 120 120 120 219 219 219 F F F
130762024-02-28T22:18:08.857Z 26 520-539 114 114 114 206 206 206 F F F
130772024-02-28T22:18:08.857Z 27 540-559 119 119 119 217 217 217 F F F
130782024-02-28T22:18:08.857Z 28 560-579 119 119 119 217 217 217 F F F
130792024-02-28T22:18:08.858Z 29 580-599 114 114 114 206 206 206 F F F
130802024-02-28T22:18:08.858ZMax gen: 121, Max flush: 223
130812024-02-28T22:18:08.858Z
130822024-02-28T22:18:08.858ZBegin loop 61
130832024-02-28T22:18:08.858ZDownstairs to restart: 0
130842024-02-28T22:18:08.873Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759356722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
130852024-02-28T22:18:08.873ZCrucible runtime is spawned
130862024-02-28T22:18:08.882Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759746971Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
130872024-02-28T22:18:08.882Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.75978856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
130882024-02-28T22:18:08.882Z{"msg":"Crucible ef6aa6f9-8339-4146-83db-c94495b221c1 has session id: 1ab74140-c03c-4fae-a9f1-fe05f509e0a5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759818369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130892024-02-28T22:18:08.882Z{"msg":"Upstairs opts: Upstairs UUID: ef6aa6f9-8339-4146-83db-c94495b221c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759855078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130902024-02-28T22:18:08.882Z{"msg":"Crucible stats registered with UUID: ef6aa6f9-8339-4146-83db-c94495b221c1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759880217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130912024-02-28T22:18:08.883Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759914076Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
130922024-02-28T22:18:08.883Z{"msg":"ef6aa6f9-8339-4146-83db-c94495b221c1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759935316Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130932024-02-28T22:18:08.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.759969175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130942024-02-28T22:18:08.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760006784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130952024-02-28T22:18:08.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760044803Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130962024-02-28T22:18:08.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760084722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130972024-02-28T22:18:08.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760120441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130982024-02-28T22:18:08.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.76015399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
130992024-02-28T22:18:08.883Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760178899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131002024-02-28T22:18:08.883Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760258407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131012024-02-28T22:18:08.883Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760308886Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131022024-02-28T22:18:08.884Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760353095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131032024-02-28T22:18:08.884Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760384374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131042024-02-28T22:18:08.884Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.760424853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131052024-02-28T22:18:08.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761140783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131062024-02-28T22:18:08.884Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761178422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131072024-02-28T22:18:08.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761194391Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131082024-02-28T22:18:08.884Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761208311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131092024-02-28T22:18:08.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761262549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131102024-02-28T22:18:08.884Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761278719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131112024-02-28T22:18:08.884Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761771655Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131122024-02-28T22:18:08.885Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.761809534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131132024-02-28T22:18:08.885Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762035928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131142024-02-28T22:18:08.885Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.76232272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131152024-02-28T22:18:08.885Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762363849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131162024-02-28T22:18:08.885Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762385679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131172024-02-28T22:18:08.885Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762449427Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131182024-02-28T22:18:08.885Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762518855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131192024-02-28T22:18:08.885Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762536814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131202024-02-28T22:18:08.885Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762561694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131212024-02-28T22:18:08.891Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762587943Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131222024-02-28T22:18:08.891Z{"msg":"[0]R flush_numbers[0..12]: [222, 223, 206, 206, 220, 223, 221, 223, 223, 217, 213, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762626052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131232024-02-28T22:18:08.891Z{"msg":"[0]R generation[0..12]: [120, 121, 114, 114, 120, 121, 120, 121, 121, 119, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762645771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131242024-02-28T22:18:08.891Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762660801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131252024-02-28T22:18:08.891Z{"msg":"[1]R flush_numbers[0..12]: [222, 223, 206, 206, 220, 223, 221, 223, 223, 217, 213, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.76268506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131262024-02-28T22:18:08.891Z{"msg":"[1]R generation[0..12]: [120, 121, 114, 114, 120, 121, 120, 121, 121, 119, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.76270068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131272024-02-28T22:18:08.892Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762715529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131282024-02-28T22:18:08.892Z{"msg":"[2]R flush_numbers[0..12]: [222, 223, 206, 206, 220, 223, 221, 223, 223, 217, 213, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762751528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131292024-02-28T22:18:08.892Z{"msg":"[2]R generation[0..12]: [120, 121, 114, 114, 120, 121, 120, 121, 121, 119, 118, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762766588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131302024-02-28T22:18:08.892Z{"msg":"Wait for a query_work_queue command to finish before sending IO
131312024-02-28T22:18:08.892Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762792927Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131322024-02-28T22:18:08.892Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762806957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131332024-02-28T22:18:08.892Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762828046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131342024-02-28T22:18:08.892Z{"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762846816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131352024-02-28T22:18:08.893Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762878555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"":"downstairs","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131362024-02-28T22:18:08.893Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762895354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131372024-02-28T22:18:08.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762915474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"0","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131382024-02-28T22:18:08.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762930273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"1","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131392024-02-28T22:18:08.893Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762951523Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"client":"2","":"downstairs","session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131402024-02-28T22:18:08.893Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762975982Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131412024-02-28T22:18:08.893Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.762993802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131422024-02-28T22:18:08.893Z{"msg":"ef6aa6f9-8339-4146-83db-c94495b221c1 is now active with session: 1ab74140-c03c-4fae-a9f1-fe05f509e0a5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.763017381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131432024-02-28T22:18:08.894Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.763030261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623,"session_id":"1ab74140-c03c-4fae-a9f1-fe05f509e0a5"}
131442024-02-28T22:18:08.894Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:07.7630505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
131452024-02-28T22:18:08.894ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131462024-02-28T22:18:08.894ZLoading write count information from file "/tmp/test_repair_verify.data"
131472024-02-28T22:18:08.894ZRun Repair workload
131482024-02-28T22:18:08.894Z01/30 Write block 344 len 20480 data: 9 9 8 9 9
131492024-02-28T22:18:08.899Z02/30 Read block 205 len 24576
131502024-02-28T22:18:08.900Z03/30 Read block 357 len 32768
131512024-02-28T22:18:08.900Z04/30 Write block 129 len 28672 data: 14 13 14 13 14 13 12
131522024-02-28T22:18:08.900Z05/30 Write block 414 len 12288 data: 12 13 12
131532024-02-28T22:18:08.900Z06/30 Write block 426 len 20480 data: 6 6 6 9 9
131542024-02-28T22:18:08.900Z07/30 Write block 135 len 24576 data: 13 13 13 14 12 11
131552024-02-28T22:18:08.900Z08/30 Flush
131562024-02-28T22:18:08.901Z09/30 Write block 201 len 24576 data: 8 8 7 9 11 8
131572024-02-28T22:18:08.901Z10/30 Read block 350 len 40960
131582024-02-28T22:18:08.901Z11/30 Write block 210 len 20480 data: 8 9 10 9 8
131592024-02-28T22:18:08.901Z12/30 Read block 123 len 16384
131602024-02-28T22:18:08.901Z13/30 Read block 96 len 24576
131612024-02-28T22:18:08.901Z14/30 Write block 283 len 20480 data: 8 10 9 11 8
131622024-02-28T22:18:08.901Z15/30 Write block 326 len 24576 data: 12 12 15 15 16 14
131632024-02-28T22:18:08.901Z16/30 Read block 207 len 12288
131642024-02-28T22:18:08.902Z17/30 Write block 29 len 40960 data: 8 8 8 8 8 7 8 11 12 13
131652024-02-28T22:18:08.902Z18/30 Write block 371 len 32768 data: 11 14 11 11 10 10 10 10
131662024-02-28T22:18:08.902Z19/30 Read block 64 len 20480
131672024-02-28T22:18:08.902Z20/30 Flush
131682024-02-28T22:18:08.906Z21/30 Write block 491 len 28672 data: 14 13 12 12 13 13 11
131692024-02-28T22:18:08.906Z22/30 Write block 481 len 28672 data: 5 8 8 9 12 9 9
131702024-02-28T22:18:08.906Z23/30 Write block 469 len 32768 data: 8 9 11 13 10 11 10 9
131712024-02-28T22:18:08.906Z24/30 Read block 124 len 4096
131722024-02-28T22:18:08.906Z25/30 Flush
131732024-02-28T22:18:08.912Z26/30 Write block 464 len 8192 data: 8 10
131742024-02-28T22:18:08.912Z27/30 Write block 3 len 20480 data: 9 11 10 12 15
131752024-02-28T22:18:08.912Z28/30 Write block 482 len 36864 data: 9 9 10 13 10 10 7 12 12
131762024-02-28T22:18:08.913Z29/30 Read block 410 len 28672
131772024-02-28T22:18:08.913Z30/30 Write block 81 len 36864 data: 11 14 12 10 10 9 9 9 10
131782024-02-28T22:18:08.917Z----------------------------------------------------------------
131792024-02-28T22:18:08.917Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
131802024-02-28T22:18:08.917ZStates: Active Active Active
131812024-02-28T22:18:08.917ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
131822024-02-28T22:18:08.917Z 1 Acked 1000 Write 5 Sent Done Done false
131832024-02-28T22:18:08.917Z 2 Acked 1001 Read 6 Sent Done Done false
131842024-02-28T22:18:08.918Z 3 Acked 1002 Read 8 Sent Done Done false
131852024-02-28T22:18:08.918Z 4 Acked 1003 Write 7 Sent Done Done false
131862024-02-28T22:18:08.918Z 5 Acked 1004 Write 3 Sent Done Done false
131872024-02-28T22:18:08.918Z 6 Acked 1005 Write 5 Sent Done Done false
131882024-02-28T22:18:08.918Z 7 Acked 1006 Write 6 Sent Done Done false
131892024-02-28T22:18:08.918Z 8 Acked 1007 Flush 0 Sent Done Done false
131902024-02-28T22:18:08.918Z 9 Acked 1008 Write 6 Sent Done Done false
131912024-02-28T22:18:08.918Z 10 Acked 1009 Read 10 Sent Done Done false
131922024-02-28T22:18:08.918Z 11 Acked 1010 Write 5 Sent Done Done false
131932024-02-28T22:18:08.918Z 12 Acked 1011 Read 4 Sent Done Done false
131942024-02-28T22:18:08.919Z 13 Acked 1012 Read 6 Sent Done Done false
131952024-02-28T22:18:08.919Z 14 Acked 1013 Write 5 Sent Done Done false
131962024-02-28T22:18:08.919Z 15 Acked 1014 Write 6 Sent Done Done false
131972024-02-28T22:18:08.919Z 16 Acked 1015 Read 3 Sent Done Done false
131982024-02-28T22:18:08.919Z 17 Acked 1016 Write 10 Sent Done Done false
131992024-02-28T22:18:08.919Z 18 Acked 1017 Write 8 Sent Done Done false
132002024-02-28T22:18:08.919Z 19 Acked 1018 Read 5 Sent Done Done false
132012024-02-28T22:18:08.919Z 20 Acked 1019 Flush 0 Sent Done Done false
132022024-02-28T22:18:08.919Z 21 Acked 1020 Write 7 Sent Done Done false
132032024-02-28T22:18:08.919Z 22 Acked 1021 Write 7 Sent Done Done false
132042024-02-28T22:18:08.919Z 23 Acked 1022 Write 8 Sent Done Done false
132052024-02-28T22:18:08.919Z 24 Acked 1023 Read 1 Sent Done Done false
132062024-02-28T22:18:08.920Z 25 Acked 1024 Flush 0 Sent Done Done false
132072024-02-28T22:18:08.920Z 26 Acked 1025 Write 2 Sent Done Done false
132082024-02-28T22:18:08.920Z 27 Acked 1026 Write 5 Sent Done Done false
132092024-02-28T22:18:08.920Z 28 Acked 1027 Write 9 Sent Done Done false
132102024-02-28T22:18:08.920Z 29 Acked 1028 Read 7 Sent Done Done false
132112024-02-28T22:18:08.920Z 30 Acked 1029 Write 9 Sent Sent Sent false
132122024-02-28T22:18:08.920Z STATES DS:0 DS:1 DS:2 TOTAL
132132024-02-28T22:18:08.920Z New 0 0 0 0
132142024-02-28T22:18:08.920Z Sent 30 1 1 32
132152024-02-28T22:18:08.920Z Done 0 29 29 58
132162024-02-28T22:18:08.920Z Skipped 0 0 0 0
132172024-02-28T22:18:08.920Z Error 0 0 0 0
132182024-02-28T22:18:08.921ZLast Flush: 0 1024 1024
132192024-02-28T22:18:08.921ZDownstairs last five completed:
132202024-02-28T22:18:08.921ZUpstairs last five completed: 30 29 28 27 26
132212024-02-28T22:18:08.921Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:07.798744667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1623}
132222024-02-28T22:18:08.921ZWrote out file "/tmp/test_repair_verify.data"
132232024-02-28T22:18:08.921Z
132242024-02-28T22:18:08.925ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132252024-02-28T22:18:08.935ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132262024-02-28T22:18:08.935Z 0 000-019 120 120 120 222 222 222 F T T <---
132272024-02-28T22:18:08.936Z 1 020-039 121 122 122 223 225 225 F F F <---
132282024-02-28T22:18:08.936Z 2 040-059 114 114 114 206 206 206 F F F
132292024-02-28T22:18:08.936Z 3 060-079 114 114 114 206 206 206 F F F
132302024-02-28T22:18:08.936Z 4 080-099 120 120 120 220 220 220 F T T <---
132312024-02-28T22:18:08.940Z 5 100-119 121 121 121 223 223 223 F F F
132322024-02-28T22:18:08.940Z 6 120-139 120 122 122 221 224 224 F F F <---
132332024-02-28T22:18:08.940Z 7 140-159 121 122 122 223 224 224 F F F <---
132342024-02-28T22:18:08.940Z 8 160-179 121 121 121 223 223 223 F F F
132352024-02-28T22:18:08.940Z 9 180-199 119 119 119 217 217 217 F F F
132362024-02-28T22:18:08.940Z 10 200-219 118 122 122 213 225 225 F F F <---
132372024-02-28T22:18:08.940Z 11 220-239 120 120 120 220 220 220 F F F
132382024-02-28T22:18:08.940Z 12 240-259 120 120 120 219 219 219 F F F
132392024-02-28T22:18:08.940Z 13 260-279 112 112 112 203 203 203 F F F
132402024-02-28T22:18:08.940Z 14 280-299 120 122 122 220 225 225 F F F <---
132412024-02-28T22:18:08.941Z 15 300-319 115 115 115 207 207 207 F F F
132422024-02-28T22:18:08.941Z 16 320-339 118 122 122 216 225 225 F F F <---
132432024-02-28T22:18:08.941Z 17 340-359 119 122 122 217 224 224 F F F <---
132442024-02-28T22:18:08.941Z 18 360-379 109 122 122 199 225 225 F F F <---
132452024-02-28T22:18:08.941Z 19 380-399 112 112 112 203 203 203 F F F
132462024-02-28T22:18:08.941Z 20 400-419 109 122 122 199 224 224 F F F <---
132472024-02-28T22:18:08.941Z 21 420-439 119 122 122 217 224 224 F F F <---
132482024-02-28T22:18:08.942Z 22 440-459 119 119 119 217 217 217 F F F
132492024-02-28T22:18:08.942Z 23 460-479 119 122 122 217 226 226 F T T <---
132502024-02-28T22:18:08.942Z 24 480-499 119 122 122 217 226 226 F T T <---
132512024-02-28T22:18:08.942Z 25 500-519 120 120 120 219 219 219 F F F
132522024-02-28T22:18:08.942Z 26 520-539 114 114 114 206 206 206 F F F
132532024-02-28T22:18:08.942Z 27 540-559 119 119 119 217 217 217 F F F
132542024-02-28T22:18:08.942Z 28 560-579 119 119 119 217 217 217 F F F
132552024-02-28T22:18:08.943Z 29 580-599 114 114 114 206 206 206 F F F
132562024-02-28T22:18:08.943ZMax gen: 122, Max flush: 226
132572024-02-28T22:18:08.943ZError: Difference in extent metadata found!
132582024-02-28T22:18:08.943ZA Difference in extent metadata is expected here
132592024-02-28T22:18:08.943ZOn loop 61
132602024-02-28T22:18:08.943Z
132612024-02-28T22:18:08.947ZVerifying data now
132622024-02-28T22:18:13.628ZNo change after verify
132632024-02-28T22:18:13.629ZLoop: 61 Downstairs dump after verify (and repair):
132642024-02-28T22:18:13.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132652024-02-28T22:18:13.642Z 0 000-019 123 123 123 227 227 227 F F F
132662024-02-28T22:18:13.643Z 1 020-039 122 122 122 225 225 225 F F F
132672024-02-28T22:18:13.643Z 2 040-059 114 114 114 206 206 206 F F F
132682024-02-28T22:18:13.643Z 3 060-079 114 114 114 206 206 206 F F F
132692024-02-28T22:18:13.643Z 4 080-099 123 123 123 227 227 227 F F F
132702024-02-28T22:18:13.643Z 5 100-119 121 121 121 223 223 223 F F F
132712024-02-28T22:18:13.643Z 6 120-139 122 122 122 224 224 224 F F F
132722024-02-28T22:18:13.643Z 7 140-159 122 122 122 224 224 224 F F F
132732024-02-28T22:18:13.643Z 8 160-179 121 121 121 223 223 223 F F F
132742024-02-28T22:18:13.643Z 9 180-199 119 119 119 217 217 217 F F F
132752024-02-28T22:18:13.643Z 10 200-219 122 122 122 225 225 225 F F F
132762024-02-28T22:18:13.643Z 11 220-239 120 120 120 220 220 220 F F F
132772024-02-28T22:18:13.643Z 12 240-259 120 120 120 219 219 219 F F F
132782024-02-28T22:18:13.643Z 13 260-279 112 112 112 203 203 203 F F F
132792024-02-28T22:18:13.643Z 14 280-299 122 122 122 225 225 225 F F F
132802024-02-28T22:18:13.647Z 15 300-319 115 115 115 207 207 207 F F F
132812024-02-28T22:18:13.647Z 16 320-339 122 122 122 225 225 225 F F F
132822024-02-28T22:18:13.647Z 17 340-359 122 122 122 224 224 224 F F F
132832024-02-28T22:18:13.647Z 18 360-379 122 122 122 225 225 225 F F F
132842024-02-28T22:18:13.647Z 19 380-399 112 112 112 203 203 203 F F F
132852024-02-28T22:18:13.647Z 20 400-419 122 122 122 224 224 224 F F F
132862024-02-28T22:18:13.647Z 21 420-439 122 122 122 224 224 224 F F F
132872024-02-28T22:18:13.647Z 22 440-459 119 119 119 217 217 217 F F F
132882024-02-28T22:18:13.647Z 23 460-479 123 123 123 227 227 227 F F F
132892024-02-28T22:18:13.647Z 24 480-499 123 123 123 227 227 227 F F F
132902024-02-28T22:18:13.647Z 25 500-519 120 120 120 219 219 219 F F F
132912024-02-28T22:18:13.647Z 26 520-539 114 114 114 206 206 206 F F F
132922024-02-28T22:18:13.647Z 27 540-559 119 119 119 217 217 217 F F F
132932024-02-28T22:18:13.647Z 28 560-579 119 119 119 217 217 217 F F F
132942024-02-28T22:18:13.647Z 29 580-599 114 114 114 206 206 206 F F F
132952024-02-28T22:18:13.647ZMax gen: 123, Max flush: 227
132962024-02-28T22:18:13.647Z
132972024-02-28T22:18:13.647ZBegin loop 62
132982024-02-28T22:18:13.647ZDownstairs to restart: 0
132992024-02-28T22:18:13.664Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.55127419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
133002024-02-28T22:18:13.672Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551571362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
133012024-02-28T22:18:13.672Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
133022024-02-28T22:18:13.672Z2024-02-28T22:18:12.55160925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
133032024-02-28T22:18:13.672Z{"msg":"Crucible 50665114-b598-4d55-8bd1-58f0a95373bf has session id: aa60f35a-e2e7-4cf8-bac8-156b7c982cb5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551678188Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133042024-02-28T22:18:13.672Z{"msg":"Upstairs opts: Upstairs UUID: 50665114-b598-4d55-8bd1-58f0a95373bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551731827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133052024-02-28T22:18:13.672Z{"msg":"Crucible stats registered with UUID: 50665114-b598-4d55-8bd1-58f0a95373bf","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551768946Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133062024-02-28T22:18:13.672Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551802345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
133072024-02-28T22:18:13.672Z{"msg":"50665114-b598-4d55-8bd1-58f0a95373bf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551892173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133082024-02-28T22:18:13.672Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551925822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133092024-02-28T22:18:13.672Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.551953591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133102024-02-28T22:18:13.672Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.55199162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133112024-02-28T22:18:13.672Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552020999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133122024-02-28T22:18:13.672Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552046729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133132024-02-28T22:18:13.672Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552073778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133142024-02-28T22:18:13.672Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552097537Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133152024-02-28T22:18:13.672Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552172065Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133162024-02-28T22:18:13.672Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552220444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133172024-02-28T22:18:13.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552265712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133182024-02-28T22:18:13.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.552294772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133192024-02-28T22:18:13.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.55233341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133202024-02-28T22:18:13.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553025611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133212024-02-28T22:18:13.673Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553063141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133222024-02-28T22:18:13.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.5530789Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133232024-02-28T22:18:13.673Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553093949Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133242024-02-28T22:18:13.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553109059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133252024-02-28T22:18:13.673Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553129169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133262024-02-28T22:18:13.673Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553640314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133272024-02-28T22:18:13.673Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553677793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133282024-02-28T22:18:13.673Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.553695453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133292024-02-28T22:18:13.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554248158Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133302024-02-28T22:18:13.673Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554285647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133312024-02-28T22:18:13.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554301706Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133322024-02-28T22:18:13.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554322536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133332024-02-28T22:18:13.673Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554336725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133342024-02-28T22:18:13.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554350275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133352024-02-28T22:18:13.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554369345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133362024-02-28T22:18:13.673Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554383394Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133372024-02-28T22:18:13.673Z{"msg":"[0]R flush_numbers[0..12]: [227, 225, 206, 206, 227, 223, 224, 224, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554404004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133382024-02-28T22:18:13.673Z{"msg":"[0]R generation[0..12]: [123, 122, 114, 114, 123, 121, 122, 122, 121, 119, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554418613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133392024-02-28T22:18:13.673Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554440623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133402024-02-28T22:18:13.674Z{"msg":"[1]R flush_numbers[0..12]: [227, 225, 206, 206, 227, 223, 224, 224, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554455192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133412024-02-28T22:18:13.674ZWait for a query_work_queue command to finish before sending IO
133422024-02-28T22:18:13.674Z{"msg":"[1]R generation[0..12]: [123, 122, 114, 114, 123, 121, 122, 122, 121, 119, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554482101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133432024-02-28T22:18:13.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554509091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133442024-02-28T22:18:13.674Z{"msg":"[2]R flush_numbers[0..12]: [227, 225, 206, 206, 227, 223, 224, 224, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.55452363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133452024-02-28T22:18:13.674Z{"msg":"[2]R generation[0..12]: [123, 122, 114, 114, 123, 121, 122, 122, 121, 119, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.55453932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133462024-02-28T22:18:13.674Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554553459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133472024-02-28T22:18:13.674Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554566719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133482024-02-28T22:18:13.674Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554580579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133492024-02-28T22:18:13.674Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554601338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133502024-02-28T22:18:13.674Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554615228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"":"downstairs","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133512024-02-28T22:18:13.674Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554635647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133522024-02-28T22:18:13.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554648337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"0","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133532024-02-28T22:18:13.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554669766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"1","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133542024-02-28T22:18:13.674Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554683476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"client":"2","":"downstairs","session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133552024-02-28T22:18:13.674Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554703765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133562024-02-28T22:18:13.674Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554716515Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133572024-02-28T22:18:13.674Z{"msg":"50665114-b598-4d55-8bd1-58f0a95373bf is now active with session: aa60f35a-e2e7-4cf8-bac8-156b7c982cb5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554730004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133582024-02-28T22:18:13.674Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554749964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631,"session_id":"aa60f35a-e2e7-4cf8-bac8-156b7c982cb5"}
133592024-02-28T22:18:13.674Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:12.554762694Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
133602024-02-28T22:18:13.674ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133612024-02-28T22:18:13.678ZLoading write count information from file "/tmp/test_repair_verify.data"
133622024-02-28T22:18:13.678ZRun Repair workload
133632024-02-28T22:18:13.678Z01/30 Write block 85 len 36864 data: 11 10 10 10 11 11 10 11 9
133642024-02-28T22:18:13.679Z02/30 Read block 198 len 32768
133652024-02-28T22:18:13.679Z03/30 Write block 505 len 24576 data: 14 15 13 12 14 13
133662024-02-28T22:18:13.679Z04/30 Write block 72 len 36864 data: 12 10 10 8 7 9 9 8 10
133672024-02-28T22:18:13.679Z05/30 Flush
133682024-02-28T22:18:13.679Z06/30 Write block 374 len 28672 data: 12 11 11 11 11 10 11
133692024-02-28T22:18:13.679Z07/30 Write block 513 len 32768 data: 12 12 14 14 13 12 12 12
133702024-02-28T22:18:13.679Z08/30 Read block 317 len 20480
133712024-02-28T22:18:13.682Z09/30 Write block 362 len 40960 data: 8 9 6 6 5 7 9 11 12 12
133722024-02-28T22:18:13.682Z10/30 Write block 394 len 8192 data: 7 10
133732024-02-28T22:18:13.682Z11/30 Read block 445 len 28672
133742024-02-28T22:18:13.683Z12/30 Read block 282 len 12288
133752024-02-28T22:18:13.683Z13/30 Read block 581 len 24576
133762024-02-28T22:18:13.686Z14/30 Read block 256 len 32768
133772024-02-28T22:18:13.686Z15/30 Read block 145 len 40960
133782024-02-28T22:18:13.686Z16/30 Write block 247 len 20480 data: 13 15 10 10 10
133792024-02-28T22:18:13.692Z17/30 Write block 484 len 4096 data: 11
133802024-02-28T22:18:13.692Z18/30 Read block 158 len 28672
133812024-02-28T22:18:13.692Z19/30 Read block 31 len 12288
133822024-02-28T22:18:13.692Z20/30 Read block 185 len 28672
133832024-02-28T22:18:13.693Z21/30 Read block 363 len 8192
133842024-02-28T22:18:13.696Z22/30 Flush
133852024-02-28T22:18:13.696Z23/30 Write block 143 len 4096 data: 12
133862024-02-28T22:18:13.696Z24/30 Flush
133872024-02-28T22:18:13.696Z25/30 Write block 255 len 16384 data: 8 11 12 12
133882024-02-28T22:18:13.699Z26/30 Write block 109 len 24576 data: 17 15 13 13 10 11
133892024-02-28T22:18:13.699Z27/30 Write block 95 len 16384 data: 7 7 7 8
133902024-02-28T22:18:13.699Z28/30 Write block 274 len 32768 data: 4 4 5 5 6 7 9 8
133912024-02-28T22:18:13.699Z29/30 Write block 562 len 24576 data: 10 12 12 13 15 17
133922024-02-28T22:18:13.705Z30/30 Write block 385 len 36864 data: 13 13 10 8 7 10 11 10 8
133932024-02-28T22:18:13.705Z----------------------------------------------------------------
133942024-02-28T22:18:13.705Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
133952024-02-28T22:18:13.705ZStates: Active Active Active
133962024-02-28T22:18:13.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133972024-02-28T22:18:13.705Z 1 Acked 1000 Write 9 Sent Done Done false
133982024-02-28T22:18:13.705Z 2 Acked 1001 Read 8 Sent Done Done false
133992024-02-28T22:18:13.705Z 3 Acked 1002 Write 6 Sent Done Done false
134002024-02-28T22:18:13.706Z 4 Acked 1003 Write 9 Sent Done Done false
134012024-02-28T22:18:13.706Z 5 Acked 1004 Flush 0 Sent Done Done false
134022024-02-28T22:18:13.706Z 6 Acked 1005 Write 7 Sent Done Done false
134032024-02-28T22:18:13.706Z 7 Acked 1006 Write 8 Sent Done Done false
134042024-02-28T22:18:13.706Z 8 Acked 1007 Read 5 Sent Done Done false
134052024-02-28T22:18:13.706Z 9 Acked 1008 Write 10 Sent Done Done false
134062024-02-28T22:18:13.706Z 10 Acked 1009 Write 2 Sent Done Done false
134072024-02-28T22:18:13.706Z 11 Acked 1010 Read 7 Sent Done Done false
134082024-02-28T22:18:13.706Z 12 Acked 1011 Read 3 Sent Done Done false
134092024-02-28T22:18:13.706Z 13 Acked 1012 Read 6 Sent Done Done false
134102024-02-28T22:18:13.706Z 14 Acked 1013 Read 8 Sent Done Done false
134112024-02-28T22:18:13.706Z 15 Acked 1014 Read 10 Sent Done Done false
134122024-02-28T22:18:13.706Z 16 Acked 1015 Write 5 Sent Done Done false
134132024-02-28T22:18:13.706Z 17 Acked 1016 Write 1 Sent Done Done false
134142024-02-28T22:18:13.706Z 18 Acked 1017 Read 7 Sent Done Done false
134152024-02-28T22:18:13.706Z 19 Acked 1018 Read 3 Sent Done Done false
134162024-02-28T22:18:13.706Z 20 Acked 1019 Read 7 Sent Done Done false
134172024-02-28T22:18:13.706Z 21 Acked 1020 Read 2 Sent Done Done false
134182024-02-28T22:18:13.706Z 22 Acked 1021 Flush 0 Sent Done Done false
134192024-02-28T22:18:13.706Z 23 Acked 1022 Write 1 Sent Done Done false
134202024-02-28T22:18:13.706Z 24 Acked 1023 Flush 0 Sent Done Done false
134212024-02-28T22:18:13.706Z 25 Acked 1024 Write 4 Sent Done Done false
134222024-02-28T22:18:13.706Z 26 Acked 1025 Write 6 Sent Done Done false
134232024-02-28T22:18:13.706Z 27 Acked 1026 Write 4 Sent Done Done false
134242024-02-28T22:18:13.707Z 28 Acked 1027 Write 8 Sent Done Done false
134252024-02-28T22:18:13.707Z 29 Acked 1028 Write 6 Sent Done Done false
134262024-02-28T22:18:13.707Z 30 Acked 1029 Write 9 Sent Sent Sent false
134272024-02-28T22:18:13.707Z STATES DS:0 DS:1 DS:2 TOTAL
134282024-02-28T22:18:13.707Z New 0 0 0 0
134292024-02-28T22:18:13.707Z Sent 30 1 1 32
134302024-02-28T22:18:13.707Z Done 0 29 29 58
134312024-02-28T22:18:13.707Z Skipped 0 0 0 0
134322024-02-28T22:18:13.707Z Error 0 0 0 0
134332024-02-28T22:18:13.707ZLast Flush: 0 1023 1023
134342024-02-28T22:18:13.707ZDownstairs last five completed:
134352024-02-28T22:18:13.707ZUpstairs last five completed: 30 29 28 27 26
134362024-02-28T22:18:13.707Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:12.587569359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1631}
134372024-02-28T22:18:13.707ZWrote out file "/tmp/test_repair_verify.data"
134382024-02-28T22:18:13.711Z
134392024-02-28T22:18:13.711ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134402024-02-28T22:18:13.726ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134412024-02-28T22:18:13.726Z 0 000-019 123 123 123 227 227 227 F F F
134422024-02-28T22:18:13.726Z 1 020-039 122 122 122 225 225 225 F F F
134432024-02-28T22:18:13.726Z 2 040-059 114 114 114 206 206 206 F F F
134442024-02-28T22:18:13.726Z 3 060-079 114 124 124 206 228 228 F F F <---
134452024-02-28T22:18:13.726Z 4 080-099 123 124 124 227 228 228 F T T <---
134462024-02-28T22:18:13.726Z 5 100-119 121 121 121 223 223 223 F T T <---
134472024-02-28T22:18:13.726Z 6 120-139 122 122 122 224 224 224 F F F
134482024-02-28T22:18:13.726Z 7 140-159 122 124 124 224 230 230 F F F <---
134492024-02-28T22:18:13.726Z 8 160-179 121 121 121 223 223 223 F F F
134502024-02-28T22:18:13.727Z 9 180-199 119 119 119 217 217 217 F F F
134512024-02-28T22:18:13.731Z 10 200-219 122 122 122 225 225 225 F F F
134522024-02-28T22:18:13.731Z 11 220-239 120 120 120 220 220 220 F F F
134532024-02-28T22:18:13.731Z 12 240-259 120 124 124 219 229 229 F T T <---
134542024-02-28T22:18:13.731Z 13 260-279 112 112 112 203 203 203 F T T <---
134552024-02-28T22:18:13.731Z 14 280-299 122 122 122 225 225 225 F T T <---
134562024-02-28T22:18:13.731Z 15 300-319 115 115 115 207 207 207 F F F
134572024-02-28T22:18:13.731Z 16 320-339 122 122 122 225 225 225 F F F
134582024-02-28T22:18:13.731Z 17 340-359 122 122 122 224 224 224 F F F
134592024-02-28T22:18:13.731Z 18 360-379 122 124 124 225 229 229 F F F <---
134602024-02-28T22:18:13.732Z 19 380-399 112 124 124 203 229 229 F T T <---
134612024-02-28T22:18:13.732Z 20 400-419 122 122 122 224 224 224 F F F
134622024-02-28T22:18:13.732Z 21 420-439 122 122 122 224 224 224 F F F
134632024-02-28T22:18:13.732Z 22 440-459 119 119 119 217 217 217 F F F
134642024-02-28T22:18:13.732Z 23 460-479 123 123 123 227 227 227 F F F
134652024-02-28T22:18:13.732Z 24 480-499 123 124 124 227 229 229 F F F <---
134662024-02-28T22:18:13.732Z 25 500-519 120 124 124 219 229 229 F F F <---
134672024-02-28T22:18:13.732Z 26 520-539 114 124 124 206 229 229 F F F <---
134682024-02-28T22:18:13.732Z 27 540-559 119 119 119 217 217 217 F F F
134692024-02-28T22:18:13.732Z 28 560-579 119 119 119 217 217 217 F T T <---
134702024-02-28T22:18:13.733Z 29 580-599 114 114 114 206 206 206 F F F
134712024-02-28T22:18:13.733ZMax gen: 124, Max flush: 230
134722024-02-28T22:18:13.733ZError: Difference in extent metadata found!
134732024-02-28T22:18:13.733ZA Difference in extent metadata is expected here
134742024-02-28T22:18:13.733ZOn loop 62
134752024-02-28T22:18:13.733Z
134762024-02-28T22:18:13.738ZVerifying data now
134772024-02-28T22:18:18.391ZNo change after verify
134782024-02-28T22:18:18.391ZLoop: 62 Downstairs dump after verify (and repair):
134792024-02-28T22:18:18.404ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134802024-02-28T22:18:18.405Z 0 000-019 123 123 123 227 227 227 F F F
134812024-02-28T22:18:18.405Z 1 020-039 122 122 122 225 225 225 F F F
134822024-02-28T22:18:18.405Z 2 040-059 114 114 114 206 206 206 F F F
134832024-02-28T22:18:18.405Z 3 060-079 124 124 124 228 228 228 F F F
134842024-02-28T22:18:18.405Z 4 080-099 125 125 125 231 231 231 F F F
134852024-02-28T22:18:18.405Z 5 100-119 125 125 125 231 231 231 F F F
134862024-02-28T22:18:18.405Z 6 120-139 122 122 122 224 224 224 F F F
134872024-02-28T22:18:18.405Z 7 140-159 124 124 124 230 230 230 F F F
134882024-02-28T22:18:18.405Z 8 160-179 121 121 121 223 223 223 F F F
134892024-02-28T22:18:18.405Z 9 180-199 119 119 119 217 217 217 F F F
134902024-02-28T22:18:18.405Z 10 200-219 122 122 122 225 225 225 F F F
134912024-02-28T22:18:18.405Z 11 220-239 120 120 120 220 220 220 F F F
134922024-02-28T22:18:18.411Z 12 240-259 125 125 125 231 231 231 F F F
134932024-02-28T22:18:18.411Z 13 260-279 125 125 125 231 231 231 F F F
134942024-02-28T22:18:18.411Z 14 280-299 125 125 125 231 231 231 F F F
134952024-02-28T22:18:18.411Z 15 300-319 115 115 115 207 207 207 F F F
134962024-02-28T22:18:18.411Z 16 320-339 122 122 122 225 225 225 F F F
134972024-02-28T22:18:18.412Z 17 340-359 122 122 122 224 224 224 F F F
134982024-02-28T22:18:18.412Z 18 360-379 124 124 124 229 229 229 F F F
134992024-02-28T22:18:18.412Z 19 380-399 125 125 125 231 231 231 F F F
135002024-02-28T22:18:18.412Z 20 400-419 122 122 122 224 224 224 F F F
135012024-02-28T22:18:18.412Z 21 420-439 122 122 122 224 224 224 F F F
135022024-02-28T22:18:18.412Z 22 440-459 119 119 119 217 217 217 F F F
135032024-02-28T22:18:18.413Z 23 460-479 123 123 123 227 227 227 F F F
135042024-02-28T22:18:18.413Z 24 480-499 124 124 124 229 229 229 F F F
135052024-02-28T22:18:18.413Z 25 500-519 124 124 124 229 229 229 F F F
135062024-02-28T22:18:18.413Z 26 520-539 124 124 124 229 229 229 F F F
135072024-02-28T22:18:18.413Z 27 540-559 119 119 119 217 217 217 F F F
135082024-02-28T22:18:18.413Z 28 560-579 125 125 125 231 231 231 F F F
135092024-02-28T22:18:18.414Z 29 580-599 114 114 114 206 206 206 F F F
135102024-02-28T22:18:18.414ZMax gen: 125, Max flush: 231
135112024-02-28T22:18:18.414Z
135122024-02-28T22:18:18.414ZBegin loop 63
135132024-02-28T22:18:18.414ZDownstairs to restart: 2
135142024-02-28T22:18:18.435Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.312538861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
135152024-02-28T22:18:18.436Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"nameCrucible runtime is spawned
135162024-02-28T22:18:18.436Z":"crucible","level":30,"time":"2024-02-28T22:18:17.312866472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
135172024-02-28T22:18:18.436Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.312909231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
135182024-02-28T22:18:18.436Z{"msg":"Crucible 8306b40b-43a1-4283-a1db-67d9fe9a652f has session id: 136832b4-ee27-4a8f-84fd-73304cfc4577","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.31293818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135192024-02-28T22:18:18.436Z{"msg":"Upstairs opts: Upstairs UUID: 8306b40b-43a1-4283-a1db-67d9fe9a652f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.312990459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135202024-02-28T22:18:18.436Z{"msg":"Crucible stats registered with UUID: 8306b40b-43a1-4283-a1db-67d9fe9a652f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313021928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135212024-02-28T22:18:18.436Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313069177Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
135222024-02-28T22:18:18.437Z{"msg":"8306b40b-43a1-4283-a1db-67d9fe9a652f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313173444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135232024-02-28T22:18:18.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313223003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135242024-02-28T22:18:18.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313270271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135252024-02-28T22:18:18.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.31330388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135262024-02-28T22:18:18.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313351719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135272024-02-28T22:18:18.437Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313384768Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135282024-02-28T22:18:18.437Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313416237Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135292024-02-28T22:18:18.438Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313526624Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135302024-02-28T22:18:18.438Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313585522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135312024-02-28T22:18:18.438Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313604852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135322024-02-28T22:18:18.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313629231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135332024-02-28T22:18:18.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.31366779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135342024-02-28T22:18:18.438Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.313706579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135352024-02-28T22:18:18.438Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314255824Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135362024-02-28T22:18:18.438Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314302993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135372024-02-28T22:18:18.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314345601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135382024-02-28T22:18:18.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314383751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135392024-02-28T22:18:18.439Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.31441296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135402024-02-28T22:18:18.439Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314497747Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135412024-02-28T22:18:18.439Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314946875Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135422024-02-28T22:18:18.439Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.314984324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135432024-02-28T22:18:18.439Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315007253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135442024-02-28T22:18:18.439Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.31548722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135452024-02-28T22:18:18.439Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315517359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135462024-02-28T22:18:18.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315533739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135472024-02-28T22:18:18.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315562028Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135482024-02-28T22:18:18.440Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315585827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135492024-02-28T22:18:18.440Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315599887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135502024-02-28T22:18:18.440Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315618797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135512024-02-28T22:18:18.440Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315633186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135522024-02-28T22:18:18.440Z{"msg":"[0]R flush_numbers[0..12]: [227, 225, 206, 228, 231, 231, 224, 230, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315654056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135532024-02-28T22:18:18.440Z{"msg":"[0]R generation[0..12]: [123, 122, 114, 124, 125, 125, 122, 124, 121, 119, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315668475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135542024-02-28T22:18:18.441Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315689224Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135552024-02-28T22:18:18.441Z{"msg":"[1]R flush_numbers[0..12]: [227, 225, 206, 228, 231, 231, 224, 230, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315703954Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135562024-02-28T22:18:18.441Z{"msg":"[1]R generation[0..12]: [123, 122, 114, 124, 125, 125, 122, 124, 121, 119, 122, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315724603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135572024-02-28T22:18:18.441Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315739593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135582024-02-28T22:18:18.441Z{"msg":"[2]R flush_numbers[0..12]: [227, 225, 206, 228, 231, 231, 224, 230, 223, 217, 225, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315765862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135592024-02-28T22:18:18.441Z{"msg":"[2]R generation[0..12]: [123, 122, 114, 124, 125, 125, 122, 124, 121, 119, 122, 120]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
135602024-02-28T22:18:18.441Z,"time":"2024-02-28T22:18:17.315781262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135612024-02-28T22:18:18.442Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315809041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135622024-02-28T22:18:18.442Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315832241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135632024-02-28T22:18:18.442Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.3158466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135642024-02-28T22:18:18.442Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315876419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135652024-02-28T22:18:18.442Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315900569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"":"downstairs","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135662024-02-28T22:18:18.442Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315936718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135672024-02-28T22:18:18.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315953447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"0","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135682024-02-28T22:18:18.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.315988136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"1","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135692024-02-28T22:18:18.442Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316014955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"client":"2","":"downstairs","session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135702024-02-28T22:18:18.443Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316066044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135712024-02-28T22:18:18.443Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316101863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135722024-02-28T22:18:18.443Z{"msg":"8306b40b-43a1-4283-a1db-67d9fe9a652f is now active with session: 136832b4-ee27-4a8f-84fd-73304cfc4577","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316132582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135732024-02-28T22:18:18.443Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316146252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639,"session_id":"136832b4-ee27-4a8f-84fd-73304cfc4577"}
135742024-02-28T22:18:18.443Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:17.316172691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
135752024-02-28T22:18:18.443ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135762024-02-28T22:18:18.449ZLoading write count information from file "/tmp/test_repair_verify.data"
135772024-02-28T22:18:18.449ZRun Repair workload
135782024-02-28T22:18:18.449Z01/30 Write block 270 len 8192 data: 5 5
135792024-02-28T22:18:18.449Z02/30 Read block 492 len 8192
135802024-02-28T22:18:18.449Z03/30 Read block 344 len 16384
135812024-02-28T22:18:18.449Z04/30 Read block 385 len 24576
135822024-02-28T22:18:18.449Z05/30 Write block 166 len 16384 data: 10 14 13 13
135832024-02-28T22:18:18.450Z06/30 Write block 586 len 16384 data: 10 8 8 8
135842024-02-28T22:18:18.450Z07/30 Write block 93 len 12288 data: 10 9 8
135852024-02-28T22:18:18.450Z08/30 Write block 38 len 12288 data: 14 12 9
135862024-02-28T22:18:18.450Z09/30 Write block 467 len 32768 data: 8 9 9 10 12 14 11 12
135872024-02-28T22:18:18.450Z10/30 Read block 45 len 28672
135882024-02-28T22:18:18.450Z11/30 Flush
135892024-02-28T22:18:18.450Z12/30 Write block 283 len 20480 data: 9 11 10 12 9
135902024-02-28T22:18:18.451Z13/30 Write block 63 len 24576 data: 12 12 12 12 13 13
135912024-02-28T22:18:18.451Z14/30 Read block 152 len 32768
135922024-02-28T22:18:18.451Z15/30 Read block 525 len 4096
135932024-02-28T22:18:18.451Z16/30 Write block 91 len 36864 data: 11 12 11 10 9 8 8 9 9
135942024-02-28T22:18:18.452Z17/30 Read block 475 len 28672
135952024-02-28T22:18:18.452Z18/30 Flush
135962024-02-28T22:18:18.455Z19/30 Write block 497 len 32768 data: 12 10 11 12 13 13 14 15
135972024-02-28T22:18:18.455Z20/30 Read block 57 len 20480
135982024-02-28T22:18:18.456Z21/30 Read block 105 len 40960
135992024-02-28T22:18:18.456Z22/30 Flush
136002024-02-28T22:18:18.456Z23/30 Write block 307 len 40960 data: 12 11 11 7 12 13 11 10 7 7
136012024-02-28T22:18:18.460Z24/30 Write block 516 len 28672 data: 15 14 13 13 13 11 9
136022024-02-28T22:18:18.460Z25/30 Write block 405 len 16384 data: 10 11 11 10
136032024-02-28T22:18:18.460Z26/30 Read block 417 len 20480
136042024-02-28T22:18:18.460Z27/30 Read block 67 len 32768
136052024-02-28T22:18:18.460Z28/30 Write block 194 len 28672 data: 9 9 9 10 10 7 6
136062024-02-28T22:18:18.460Z29/30 Read block 410 len 36864
136072024-02-28T22:18:18.465Z30/30 Read block 479 len 20480
136082024-02-28T22:18:18.465Z----------------------------------------------------------------
136092024-02-28T22:18:18.465Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136102024-02-28T22:18:18.466ZStates: Active Active Active
136112024-02-28T22:18:18.466ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136122024-02-28T22:18:18.466Z 1 Acked 1000 Write 2 Done Done Done false
136132024-02-28T22:18:18.466Z 2 Acked 1001 Read 2 Done Done Done false
136142024-02-28T22:18:18.466Z 3 Acked 1002 Read 4 Done Done Done false
136152024-02-28T22:18:18.466Z 4 Acked 1003 Read 6 Done Done Done false
136162024-02-28T22:18:18.466Z 5 Acked 1004 Write 4 Done Done Sent false
136172024-02-28T22:18:18.466Z 6 Acked 1005 Write 4 Done Done Sent false
136182024-02-28T22:18:18.466Z 7 Acked 1006 Write 3 Done Done Sent false
136192024-02-28T22:18:18.466Z 8 Acked 1007 Write 3 Done Done Sent false
136202024-02-28T22:18:18.467Z 9 Acked 1008 Write 8 Done Done Sent false
136212024-02-28T22:18:18.467Z 10 Acked 1009 Read 7 Done Done Sent false
136222024-02-28T22:18:18.467Z 11 Acked 1010 Flush 0 Done Done Sent false
136232024-02-28T22:18:18.467Z 12 Acked 1011 Write 5 Done Done Sent false
136242024-02-28T22:18:18.467Z 13 Acked 1012 Write 6 Done Done Sent false
136252024-02-28T22:18:18.467Z 14 Acked 1013 Read 8 Done Done Sent false
136262024-02-28T22:18:18.467Z 15 Acked 1014 Read 1 Done Done Sent false
136272024-02-28T22:18:18.467Z 16 Acked 1015 Write 9 Done Done Sent false
136282024-02-28T22:18:18.467Z 17 Acked 1016 Read 7 Done Done Sent false
136292024-02-28T22:18:18.467Z 18 Acked 1017 Flush 0 Done Done Sent false
136302024-02-28T22:18:18.468Z 19 Acked 1018 Write 8 Done Done Sent false
136312024-02-28T22:18:18.468Z 20 Acked 1019 Read 5 Done Done Sent false
136322024-02-28T22:18:18.468Z 21 Acked 1020 Read 10 Done Done Sent false
136332024-02-28T22:18:18.468Z 22 Acked 1021 Flush 0 Done Done Sent false
136342024-02-28T22:18:18.468Z 23 Acked 1022 Write 10 Done Done Sent false
136352024-02-28T22:18:18.468Z 24 Acked 1023 Write 7 Done Done Sent false
136362024-02-28T22:18:18.468Z 25 Acked 1024 Write 4 Done Done Sent false
136372024-02-28T22:18:18.468Z 26 Acked 1025 Read 5 Done Done Sent false
136382024-02-28T22:18:18.468Z 27 Acked 1026 Read 8 Done Done Sent false
136392024-02-28T22:18:18.468Z 28 Acked 1027 Write 7 Done Done Sent false
136402024-02-28T22:18:18.469Z 29 Acked 1028 Read 9 Done Done Sent false
136412024-02-28T22:18:18.469Z 30 Acked 1029 Read 5 Done Done Sent false
136422024-02-28T22:18:18.469Z STATES DS:0 DS:1 DS:2 TOTAL
136432024-02-28T22:18:18.469Z New 0 0 0 0
136442024-02-28T22:18:18.469Z Sent 0 0 26 26
136452024-02-28T22:18:18.469Z Done 30 30 4 64
136462024-02-28T22:18:18.469Z Skipped 0 0 0 0
136472024-02-28T22:18:18.469Z Error 0 0 0 0
136482024-02-28T22:18:18.469ZLast Flush: 1021 1021 0
136492024-02-28T22:18:18.469ZDownstairs last five completed:
136502024-02-28T22:18:18.469ZUpstairs last five completed: 30 29 28 27 26
136512024-02-28T22:18:18.470Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:17.349448614Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1639}
136522024-02-28T22:18:18.470ZWrote out file "/tmp/test_repair_verify.data"
136532024-02-28T22:18:18.474Z
136542024-02-28T22:18:18.474ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136552024-02-28T22:18:18.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136562024-02-28T22:18:18.487Z 0 000-019 123 123 123 227 227 227 F F F
136572024-02-28T22:18:18.487Z 1 020-039 126 126 122 232 232 225 F F F <---
136582024-02-28T22:18:18.487Z 2 040-059 126 126 114 232 232 206 F F F <---
136592024-02-28T22:18:18.487Z 3 060-079 126 126 124 233 233 228 F F F <---
136602024-02-28T22:18:18.487Z 4 080-099 126 126 125 233 233 231 F F F <---
136612024-02-28T22:18:18.487Z 5 100-119 125 125 125 231 231 231 F F F
136622024-02-28T22:18:18.488Z 6 120-139 122 122 122 224 224 224 F F F
136632024-02-28T22:18:18.488Z 7 140-159 124 124 124 230 230 230 F F F
136642024-02-28T22:18:18.488Z 8 160-179 126 126 121 232 232 223 F F F <---
136652024-02-28T22:18:18.488Z 9 180-199 119 119 119 217 217 217 T T F <---
136662024-02-28T22:18:18.488Z 10 200-219 122 122 122 225 225 225 T T F <---
136672024-02-28T22:18:18.493Z 11 220-239 120 120 120 220 220 220 F F F
136682024-02-28T22:18:18.493Z 12 240-259 125 125 125 231 231 231 F F F
136692024-02-28T22:18:18.493Z 13 260-279 126 126 125 232 232 231 F F T <---
136702024-02-28T22:18:18.493Z 14 280-299 126 126 125 233 233 231 F F F <---
136712024-02-28T22:18:18.493Z 15 300-319 115 115 115 207 207 207 T T F <---
136722024-02-28T22:18:18.493Z 16 320-339 122 122 122 225 225 225 F F F
136732024-02-28T22:18:18.493Z 17 340-359 122 122 122 224 224 224 F F F
136742024-02-28T22:18:18.494Z 18 360-379 124 124 124 229 229 229 F F F
136752024-02-28T22:18:18.494Z 19 380-399 125 125 125 231 231 231 F F F
136762024-02-28T22:18:18.494Z 20 400-419 122 122 122 224 224 224 T T F <---
136772024-02-28T22:18:18.494Z 21 420-439 122 122 122 224 224 224 F F F
136782024-02-28T22:18:18.494Z 22 440-459 119 119 119 217 217 217 F F F
136792024-02-28T22:18:18.494Z 23 460-479 126 126 123 232 232 227 F F F <---
136802024-02-28T22:18:18.494Z 24 480-499 126 126 124 234 234 229 F F F <---
136812024-02-28T22:18:18.494Z 25 500-519 126 126 124 234 234 229 T T F <---
136822024-02-28T22:18:18.494Z 26 520-539 124 124 124 229 229 229 T T F <---
136832024-02-28T22:18:18.495Z 27 540-559 119 119 119 217 217 217 F F F
136842024-02-28T22:18:18.495Z 28 560-579 125 125 125 231 231 231 F F F
136852024-02-28T22:18:18.495Z 29 580-599 126 126 114 232 232 206 F F F <---
136862024-02-28T22:18:18.495ZMax gen: 126, Max flush: 234
136872024-02-28T22:18:18.495ZError: Difference in extent metadata found!
136882024-02-28T22:18:18.495ZA Difference in extent metadata is expected here
136892024-02-28T22:18:18.495ZOn loop 63
136902024-02-28T22:18:18.495Z
136912024-02-28T22:18:18.500ZVerifying data now
136922024-02-28T22:18:23.277ZNo change after verify
136932024-02-28T22:18:23.277ZLoop: 63 Downstairs dump after verify (and repair):
136942024-02-28T22:18:23.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136952024-02-28T22:18:23.285Z 0 000-019 123 123 123 227 227 227 F F F
136962024-02-28T22:18:23.285Z 1 020-039 126 126 126 232 232 232 F F F
136972024-02-28T22:18:23.285Z 2 040-059 126 126 126 232 232 232 F F F
136982024-02-28T22:18:23.285Z 3 060-079 126 126 126 233 233 233 F F F
136992024-02-28T22:18:23.286Z 4 080-099 126 126 126 233 233 233 F F F
137002024-02-28T22:18:23.286Z 5 100-119 125 125 125 231 231 231 F F F
137012024-02-28T22:18:23.286Z 6 120-139 122 122 122 224 224 224 F F F
137022024-02-28T22:18:23.286Z 7 140-159 124 124 124 230 230 230 F F F
137032024-02-28T22:18:23.286Z 8 160-179 126 126 126 232 232 232 F F F
137042024-02-28T22:18:23.290Z 9 180-199 127 127 127 235 235 235 F F F
137052024-02-28T22:18:23.290Z 10 200-219 127 127 127 235 235 235 F F F
137062024-02-28T22:18:23.290Z 11 220-239 120 120 120 220 220 220 F F F
137072024-02-28T22:18:23.290Z 12 240-259 125 125 125 231 231 231 F F F
137082024-02-28T22:18:23.290Z 13 260-279 126 126 126 232 232 232 F F F
137092024-02-28T22:18:23.290Z 14 280-299 126 126 126 233 233 233 F F F
137102024-02-28T22:18:23.291Z 15 300-319 127 127 127 235 235 235 F F F
137112024-02-28T22:18:23.291Z 16 320-339 122 122 122 225 225 225 F F F
137122024-02-28T22:18:23.291Z 17 340-359 122 122 122 224 224 224 F F F
137132024-02-28T22:18:23.291Z 18 360-379 124 124 124 229 229 229 F F F
137142024-02-28T22:18:23.291Z 19 380-399 125 125 125 231 231 231 F F F
137152024-02-28T22:18:23.291Z 20 400-419 127 127 127 235 235 235 F F F
137162024-02-28T22:18:23.291Z 21 420-439 122 122 122 224 224 224 F F F
137172024-02-28T22:18:23.291Z 22 440-459 119 119 119 217 217 217 F F F
137182024-02-28T22:18:23.291Z 23 460-479 126 126 126 232 232 232 F F F
137192024-02-28T22:18:23.291Z 24 480-499 126 126 126 234 234 234 F F F
137202024-02-28T22:18:23.291Z 25 500-519 127 127 127 235 235 235 F F F
137212024-02-28T22:18:23.291Z 26 520-539 127 127 127 235 235 235 F F F
137222024-02-28T22:18:23.292Z 27 540-559 119 119 119 217 217 217 F F F
137232024-02-28T22:18:23.292Z 28 560-579 125 125 125 231 231 231 F F F
137242024-02-28T22:18:23.292Z 29 580-599 126 126 126 232 232 232 F F F
137252024-02-28T22:18:23.292ZMax gen: 127, Max flush: 235
137262024-02-28T22:18:23.292Z
137272024-02-28T22:18:23.292ZBegin loop 64
137282024-02-28T22:18:23.292ZDownstairs to restart: 2
137292024-02-28T22:18:23.307Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.192642469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
137302024-02-28T22:18:23.307Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\nCrucible runtime is spawned
137312024-02-28T22:18:23.319Z opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193020259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
137322024-02-28T22:18:23.319Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193063677Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
137332024-02-28T22:18:23.320Z{"msg":"Crucible af6b960d-d057-4f20-8946-9601cd399754 has session id: 64357a9e-eedb-41c0-9000-e1b6dd080cf5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193094187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137342024-02-28T22:18:23.320Z{"msg":"Upstairs opts: Upstairs UUID: af6b960d-d057-4f20-8946-9601cd399754, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193143695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137352024-02-28T22:18:23.320Z{"msg":"Crucible stats registered with UUID: af6b960d-d057-4f20-8946-9601cd399754","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193185464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137362024-02-28T22:18:23.320Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193214973Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
137372024-02-28T22:18:23.320Z{"msg":"af6b960d-d057-4f20-8946-9601cd399754 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193250302Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137382024-02-28T22:18:23.320Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193287821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137392024-02-28T22:18:23.320Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.19332588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137402024-02-28T22:18:23.320Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193359819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137412024-02-28T22:18:23.320Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193402458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137422024-02-28T22:18:23.320Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193452837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137432024-02-28T22:18:23.320Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193490276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137442024-02-28T22:18:23.320Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193518695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137452024-02-28T22:18:23.320Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193553334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137462024-02-28T22:18:23.320Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193595263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137472024-02-28T22:18:23.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193629412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137482024-02-28T22:18:23.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.193657101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137492024-02-28T22:18:23.321Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.19369393Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137502024-02-28T22:18:23.321Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194211136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137512024-02-28T22:18:23.321Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194249145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137522024-02-28T22:18:23.321Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194266274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137532024-02-28T22:18:23.321Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194288364Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137542024-02-28T22:18:23.321Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194302993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137552024-02-28T22:18:23.321Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194334273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137562024-02-28T22:18:23.321Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.194896617Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137572024-02-28T22:18:23.321Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195037963Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137582024-02-28T22:18:23.321Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195077152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137592024-02-28T22:18:23.321Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195405883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137602024-02-28T22:18:23.321Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195444332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137612024-02-28T22:18:23.321Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195460562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137622024-02-28T22:18:23.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195572248Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137632024-02-28T22:18:23.322Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195615737Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137642024-02-28T22:18:23.322Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195652186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137652024-02-28T22:18:23.322Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195672156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137662024-02-28T22:18:23.322Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195701205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137672024-02-28T22:18:23.322Z{"msg":"[0]R flush_numbers[0..12]: [227, 232, 232, 233, 233, 231, 224, 230, 232, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195721434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137682024-02-28T22:18:23.322Z{"msg":"[0]R generation[0..12]: [123, 126, 126, 126, 126, 125, 122, 124, 126, 127, 127, 120]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
137692024-02-28T22:18:23.322Z,"time":"2024-02-28T22:18:22.195751114Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137702024-02-28T22:18:23.322Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195787393Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137712024-02-28T22:18:23.322Z{"msg":"[1]R flush_numbers[0..12]: [227, 232, 232, 233, 233, 231, 224, 230, 232, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195809712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137722024-02-28T22:18:23.322Z{"msg":"[1]R generation[0..12]: [123, 126, 126, 126, 126, 125, 122, 124, 126, 127, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195838261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137732024-02-28T22:18:23.322Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195858791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137742024-02-28T22:18:23.323Z{"msg":"[2]R flush_numbers[0..12]: [227, 232, 232, 233, 233, 231, 224, 230, 232, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.19588924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137752024-02-28T22:18:23.323Z{"msg":"[2]R generation[0..12]: [123, 126, 126, 126, 126, 125, 122, 124, 126, 127, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195906909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137762024-02-28T22:18:23.323Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195937698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137772024-02-28T22:18:23.323Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195957398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137782024-02-28T22:18:23.323Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.195986577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137792024-02-28T22:18:23.323Z{"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196002096Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137802024-02-28T22:18:23.323Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196019216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"":"downstairs","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137812024-02-28T22:18:23.323Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196048425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137822024-02-28T22:18:23.323Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196063365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"0","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137832024-02-28T22:18:23.323Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196094114Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"1","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137842024-02-28T22:18:23.324Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196113324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"client":"2","":"downstairs","session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137852024-02-28T22:18:23.324Z{"msg":"All required reconciliation work is completed","v":0,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137862024-02-28T22:18:23.324Z"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196148392Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137872024-02-28T22:18:23.324Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196169862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137882024-02-28T22:18:23.324Z{"msg":"af6b960d-d057-4f20-8946-9601cd399754 is now active with session: 64357a9e-eedb-41c0-9000-e1b6dd080cf5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196196711Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137892024-02-28T22:18:23.324Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.196213731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647,"session_id":"64357a9e-eedb-41c0-9000-e1b6dd080cf5"}
137902024-02-28T22:18:23.324Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:22.19624125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
137912024-02-28T22:18:23.324ZLoading write count information from file "/tmp/test_repair_verify.data"
137922024-02-28T22:18:23.324ZRun Repair workload
137932024-02-28T22:18:23.324Z01/30 Write block 351 len 20480 data: 8 8 9 9 8
137942024-02-28T22:18:23.324Z02/30 Write block 462 len 8192 data: 7 8
137952024-02-28T22:18:23.324Z03/30 Write block 292 len 8192 data: 7 6
137962024-02-28T22:18:23.324Z04/30 Write block 88 len 8192 data: 11 12
137972024-02-28T22:18:23.324Z05/30 Write block 251 len 8192 data: 11 9
137982024-02-28T22:18:23.324Z06/30 Read block 270 len 28672
137992024-02-28T22:18:23.340Z07/30 Read block 289 len 36864
138002024-02-28T22:18:23.340Z08/30 Write block 269 len 28672 data: 6 6 6 4 5 5 5
138012024-02-28T22:18:23.340Z09/30 Write block 584 len 8192 data: 7 9
138022024-02-28T22:18:23.340Z10/30 Write block 163 len 36864 data: 8 9 10 11 15 14 14 15 14
138032024-02-28T22:18:23.341Z11/30 Write block 529 len 8192 data: 7 7
138042024-02-28T22:18:23.341Z12/30 Flush
138052024-02-28T22:18:23.341Z13/30 Flush
138062024-02-28T22:18:23.341Z14/30 Write block 534 len 16384 data: 13 10 12 11
138072024-02-28T22:18:23.341Z15/30 Write block 487 len 20480 data: 11 8 13 13 15
138082024-02-28T22:18:23.341Z16/30 Read block 346 len 20480
138092024-02-28T22:18:23.341Z17/30 Read block 419 len 4096
138102024-02-28T22:18:23.341Z18/30 Write block 97 len 4096 data: 9
138112024-02-28T22:18:23.341Z19/30 Read block 235 len 24576
138122024-02-28T22:18:23.341Z20/30 Read block 365 len 4096
138132024-02-28T22:18:23.341Z21/30 Read block 27 len 20480
138142024-02-28T22:18:23.346Z22/30 Read block 14 len 12288
138152024-02-28T22:18:23.346Z23/30 Write block 165 len 4096 data: 11
138162024-02-28T22:18:23.347Z24/30 Read block 217 len 28672
138172024-02-28T22:18:23.347Z25/30 Read block 463 len 12288
138182024-02-28T22:18:23.347Z26/30 Read block 26 len 40960
138192024-02-28T22:18:23.347Z27/30 Read block 434 len 20480
138202024-02-28T22:18:23.347Z28/30 Write block 521 len 28672 data: 12 10 7 7 9 8 7
138212024-02-28T22:18:23.347Z29/30 Write block 356 len 24576 data: 8 8 7 5 5 5
138222024-02-28T22:18:23.347Z30/30 Read block 183 len 24576
138232024-02-28T22:18:23.347Z----------------------------------------------------------------
138242024-02-28T22:18:23.353Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138252024-02-28T22:18:23.353ZStates: Active Active Active
138262024-02-28T22:18:23.354ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138272024-02-28T22:18:23.354Z 1 Acked 1000 Write 5 Done Done Sent false
138282024-02-28T22:18:23.354Z 2 Acked 1001 Write 2 Done Done Sent false
138292024-02-28T22:18:23.354Z 3 Acked 1002 Write 2 Done Done Sent false
138302024-02-28T22:18:23.354Z 4 Acked 1003 Write 2 Done Done Sent false
138312024-02-28T22:18:23.354Z 5 Acked 1004 Write 2 Done Done Sent false
138322024-02-28T22:18:23.354Z 6 Acked 1005 Read 7 Done Done Sent false
138332024-02-28T22:18:23.355Z 7 Acked 1006 Read 9 Done Done Sent false
138342024-02-28T22:18:23.355Z 8 Acked 1007 Write 7 Done Done Sent false
138352024-02-28T22:18:23.355Z 9 Acked 1008 Write 2 Done Done Sent false
138362024-02-28T22:18:23.355Z 10 Acked 1009 Write 9 Done Done Sent false
138372024-02-28T22:18:23.355Z 11 Acked 1010 Write 2 Done Done Sent false
138382024-02-28T22:18:23.355Z 12 Acked 1011 Flush 0 Done Done Sent false
138392024-02-28T22:18:23.355Z 13 Acked 1012 Flush 0 Done Done Sent false
138402024-02-28T22:18:23.355Z 14 Acked 1013 Write 4 Done Done Sent false
138412024-02-28T22:18:23.355Z 15 Acked 1014 Write 5 Done Done Sent false
138422024-02-28T22:18:23.355Z 16 Acked 1015 Read 5 Done Done Sent false
138432024-02-28T22:18:23.355Z 17 Acked 1016 Read 1 Done Done Sent false
138442024-02-28T22:18:23.355Z 18 Acked 1017 Write 1 Done Done Sent false
138452024-02-28T22:18:23.355Z 19 Acked 1018 Read 6 Done Done Sent false
138462024-02-28T22:18:23.356Z 20 Acked 1019 Read 1 Done Done Sent false
138472024-02-28T22:18:23.356Z 21 Acked 1020 Read 5 Done Done Sent false
138482024-02-28T22:18:23.356Z 22 Acked 1021 Read 3 Done Done Sent false
138492024-02-28T22:18:23.356Z 23 Acked 1022 Write 1 Done Done Sent false
138502024-02-28T22:18:23.356Z 24 Acked 1023 Read 7 Done Done Sent false
138512024-02-28T22:18:23.356Z 25 Acked 1024 Read 3 Done Done Sent false
138522024-02-28T22:18:23.356Z 26 Acked 1025 Read 10 Done Done Sent false
138532024-02-28T22:18:23.356Z 27 Acked 1026 Read 5 Done Done Sent false
138542024-02-28T22:18:23.356Z 28 Acked 1027 Write 7 Done Done Sent false
138552024-02-28T22:18:23.356Z 29 Acked 1028 Write 6 Done Done Sent false
138562024-02-28T22:18:23.356Z 30 Acked 1029 Read 6 Sent Done Sent false
138572024-02-28T22:18:23.356Z STATES DS:0 DS:1 DS:2 TOTAL
138582024-02-28T22:18:23.356Z New 0 0 0 0
138592024-02-28T22:18:23.357Z Sent 1 0 30 31
138602024-02-28T22:18:23.357Z Done 29 30 0 59
138612024-02-28T22:18:23.357Z Skipped 0 0 0 0
138622024-02-28T22:18:23.357Z Error 0 0 0 0
138632024-02-28T22:18:23.357ZLast Flush: 1012 1012 0
138642024-02-28T22:18:23.357ZDownstairs last five completed:
138652024-02-28T22:18:23.357ZUpstairs last five completed: 30 29 28 27 26
138662024-02-28T22:18:23.357Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:22.222847317Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1647}
138672024-02-28T22:18:23.357ZWrote out file "/tmp/test_repair_verify.data"
138682024-02-28T22:18:23.357Z
138692024-02-28T22:18:23.362ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138702024-02-28T22:18:23.362ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138712024-02-28T22:18:23.362Z 0 000-019 123 123 123 227 227 227 F F F
138722024-02-28T22:18:23.362Z 1 020-039 126 126 126 232 232 232 F F F
138732024-02-28T22:18:23.362Z 2 040-059 126 126 126 232 232 232 F F F
138742024-02-28T22:18:23.363Z 3 060-079 126 126 126 233 233 233 F F F
138752024-02-28T22:18:23.363Z 4 080-099 128 128 126 236 236 233 T T F <---
138762024-02-28T22:18:23.363Z 5 100-119 125 125 125 231 231 231 F F F
138772024-02-28T22:18:23.363Z 6 120-139 122 122 122 224 224 224 F F F
138782024-02-28T22:18:23.363Z 7 140-159 124 124 124 230 230 230 F F F
138792024-02-28T22:18:23.363Z 8 160-179 128 128 126 236 236 232 T T F <---
138802024-02-28T22:18:23.363Z 9 180-199 127 127 127 235 235 235 F F F
138812024-02-28T22:18:23.363Z 10 200-219 127 127 127 235 235 235 F F F
138822024-02-28T22:18:23.365Z 11 220-239 120 120 120 220 220 220 F F F
138832024-02-28T22:18:23.365Z 12 240-259 128 128 125 236 236 231 F F F <---
138842024-02-28T22:18:23.365Z 13 260-279 128 128 126 236 236 232 F F F <---
138852024-02-28T22:18:23.366Z 14 280-299 128 128 126 236 236 233 F F F <---
138862024-02-28T22:18:23.366Z 15 300-319 127 127 127 235 235 235 F F F
138872024-02-28T22:18:23.366Z 16 320-339 122 122 122 225 225 225 F F F
138882024-02-28T22:18:23.366Z 17 340-359 128 128 122 236 236 224 T T F <---
138892024-02-28T22:18:23.370Z 18 360-379 124 124 124 229 229 229 T T F <---
138902024-02-28T22:18:23.370Z 19 380-399 125 125 125 231 231 231 F F F
138912024-02-28T22:18:23.370Z 20 400-419 127 127 127 235 235 235 F F F
138922024-02-28T22:18:23.370Z 21 420-439 122 122 122 224 224 224 F F F
138932024-02-28T22:18:23.370Z 22 440-459 119 119 119 217 217 217 F F F
138942024-02-28T22:18:23.371Z 23 460-479 128 128 126 236 236 232 F F F <---
138952024-02-28T22:18:23.371Z 24 480-499 126 126 126 234 234 234 T T F <---
138962024-02-28T22:18:23.371Z 25 500-519 127 127 127 235 235 235 F F F
138972024-02-28T22:18:23.371Z 26 520-539 128 128 127 236 236 235 T T F <---
138982024-02-28T22:18:23.371Z 27 540-559 119 119 119 217 217 217 F F F
138992024-02-28T22:18:23.371Z 28 560-579 125 125 125 231 231 231 F F F
139002024-02-28T22:18:23.371Z 29 580-599 128 128 126 236 236 232 F F F <---
139012024-02-28T22:18:23.371ZMax gen: 128, Max flush: 236
139022024-02-28T22:18:23.371ZError: Difference in extent metadata found!
139032024-02-28T22:18:23.371ZA Difference in extent metadata is expected here
139042024-02-28T22:18:23.371ZOn loop 64
139052024-02-28T22:18:23.371Z
139062024-02-28T22:18:23.371ZVerifying data now
139072024-02-28T22:18:27.958ZNo change after verify
139082024-02-28T22:18:27.958ZLoop: 64 Downstairs dump after verify (and repair):
139092024-02-28T22:18:27.972ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139102024-02-28T22:18:27.972Z 0 000-019 123 123 123 227 227 227 F F F
139112024-02-28T22:18:27.973Z 1 020-039 126 126 126 232 232 232 F F F
139122024-02-28T22:18:27.973Z 2 040-059 126 126 126 232 232 232 F F F
139132024-02-28T22:18:27.973Z 3 060-079 126 126 126 233 233 233 F F F
139142024-02-28T22:18:27.973Z 4 080-099 129 129 129 237 237 237 F F F
139152024-02-28T22:18:27.973Z 5 100-119 125 125 125 231 231 231 F F F
139162024-02-28T22:18:27.973Z 6 120-139 122 122 122 224 224 224 F F F
139172024-02-28T22:18:27.973Z 7 140-159 124 124 124 230 230 230 F F F
139182024-02-28T22:18:27.973Z 8 160-179 129 129 129 237 237 237 F F F
139192024-02-28T22:18:27.973Z 9 180-199 127 127 127 235 235 235 F F F
139202024-02-28T22:18:27.973Z 10 200-219 127 127 127 235 235 235 F F F
139212024-02-28T22:18:27.973Z 11 220-239 120 120 120 220 220 220 F F F
139222024-02-28T22:18:27.976Z 12 240-259 128 128 128 236 236 236 F F F
139232024-02-28T22:18:27.976Z 13 260-279 128 128 128 236 236 236 F F F
139242024-02-28T22:18:27.976Z 14 280-299 128 128 128 236 236 236 F F F
139252024-02-28T22:18:27.976Z 15 300-319 127 127 127 235 235 235 F F F
139262024-02-28T22:18:27.976Z 16 320-339 122 122 122 225 225 225 F F F
139272024-02-28T22:18:27.976Z 17 340-359 129 129 129 237 237 237 F F F
139282024-02-28T22:18:27.976Z 18 360-379 129 129 129 237 237 237 F F F
139292024-02-28T22:18:27.976Z 19 380-399 125 125 125 231 231 231 F F F
139302024-02-28T22:18:27.976Z 20 400-419 127 127 127 235 235 235 F F F
139312024-02-28T22:18:27.976Z 21 420-439 122 122 122 224 224 224 F F F
139322024-02-28T22:18:27.977Z 22 440-459 119 119 119 217 217 217 F F F
139332024-02-28T22:18:27.977Z 23 460-479 128 128 128 236 236 236 F F F
139342024-02-28T22:18:27.977Z 24 480-499 129 129 129 237 237 237 F F F
139352024-02-28T22:18:27.977Z 25 500-519 127 127 127 235 235 235 F F F
139362024-02-28T22:18:27.977Z 26 520-539 129 129 129 237 237 237 F F F
139372024-02-28T22:18:27.977Z 27 540-559 119 119 119 217 217 217 F F F
139382024-02-28T22:18:27.977Z 28 560-579 125 125 125 231 231 231 F F F
139392024-02-28T22:18:27.977Z 29 580-599 128 128 128 236 236 236 F F F
139402024-02-28T22:18:27.977ZMax gen: 129, Max flush: 237
139412024-02-28T22:18:27.980Z
139422024-02-28T22:18:27.980ZBegin loop 65
139432024-02-28T22:18:27.980ZDownstairs to restart: 2
139442024-02-28T22:18:27.997Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.882664384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
139452024-02-28T22:18:27.997Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.882989375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
139462024-02-28T22:18:27.997ZCrucible runtime is spawned
139472024-02-28T22:18:27.997Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883037124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
139482024-02-28T22:18:28.003Z{"msg":"Crucible 3262161a-3bac-40c2-8289-17347ebd8081 has session id: 02b39692-23c1-40cd-88e4-1b830476b8fa","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883064663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139492024-02-28T22:18:28.003Z{"msg":"Upstairs opts: Upstairs UUID: 3262161a-3bac-40c2-8289-17347ebd8081, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883116702Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139502024-02-28T22:18:28.004Z{"msg":"Crucible stats registered with UUID: 3262161a-3bac-40c2-8289-17347ebd8081","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88315928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139512024-02-28T22:18:28.004Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883194469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
139522024-02-28T22:18:28.004Z{"msg":"3262161a-3bac-40c2-8289-17347ebd8081 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883230978Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139532024-02-28T22:18:28.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883258898Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139542024-02-28T22:18:28.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883306246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139552024-02-28T22:18:28.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883367315Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139562024-02-28T22:18:28.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883406523Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139572024-02-28T22:18:28.004Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883441782Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139582024-02-28T22:18:28.004Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883468162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139592024-02-28T22:18:28.004Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883503351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139602024-02-28T22:18:28.004Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88353861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139612024-02-28T22:18:28.004Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883564239Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139622024-02-28T22:18:28.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883589078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139632024-02-28T22:18:28.004Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883622737Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139642024-02-28T22:18:28.005Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.883663426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139652024-02-28T22:18:28.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.884330998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139662024-02-28T22:18:28.005Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.884403126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139672024-02-28T22:18:28.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.884455865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139682024-02-28T22:18:28.005Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.884484904Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139692024-02-28T22:18:28.005Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.884866373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139702024-02-28T22:18:28.005Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885009589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139712024-02-28T22:18:28.005Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885091687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139722024-02-28T22:18:28.005Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885142126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139732024-02-28T22:18:28.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88535708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139742024-02-28T22:18:28.005Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885393479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139752024-02-28T22:18:28.005Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885409278Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139762024-02-28T22:18:28.005Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885564874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139772024-02-28T22:18:28.005Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885596013Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139782024-02-28T22:18:28.005Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885612053Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139792024-02-28T22:18:28.006Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.885795378Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139802024-02-28T22:18:28.006Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886211346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139812024-02-28T22:18:28.006Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886246905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139822024-02-28T22:18:28.006Z{"msg":"[0]R flush_numbers[0..12]: [227, 232, 232, 233, 237, 231, 224, 230, 237, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886263955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139832024-02-28T22:18:28.006Z{"msg":"[0]R generation[0..12]: [123, 126, 126, 126, 129, 125, 122, 124, 129, 127, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886278994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139842024-02-28T22:18:28.006Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886301014Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139852024-02-28T22:18:28.006Z{"msg":"[1]R flush_numbers[0..12]: [227, 232, 232, 233, 237, 231, 224, 230, 237, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886315363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139862024-02-28T22:18:28.006Z{"msg":"[1]R generation[0..12]: [123, 126, 126, 126, 129, 125, 122, 124, 129, 127, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886335933Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139872024-02-28T22:18:28.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886350422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139882024-02-28T22:18:28.006Z{"msg":"[2]R flush_numbers[0..12]: [227, 232, 232, 233, 237, 231, 224, 230, 237, 235, 235, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886377202Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139892024-02-28T22:18:28.006Z{"msg":"[2]R generation[0..12]: [123, 126, 126, 126, 129, 125, 122, 124, 129, 127, 127, 120]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886392321Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139902024-02-28T22:18:28.006ZWait for a query_work_queue command to finish before sending IO
139912024-02-28T22:18:28.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88641834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139922024-02-28T22:18:28.006Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88643324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139932024-02-28T22:18:28.007Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.88645429Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139942024-02-28T22:18:28.007Z{"msg":"Next flush: 238","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886468339Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139952024-02-28T22:18:28.007Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886494748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"":"downstairs","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139962024-02-28T22:18:28.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886508838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139972024-02-28T22:18:28.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886531727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"0","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139982024-02-28T22:18:28.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886572926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"1","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
139992024-02-28T22:18:28.011Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886597195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"client":"2","":"downstairs","session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
140002024-02-28T22:18:28.011Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886632115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
140012024-02-28T22:18:28.011Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886654654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
140022024-02-28T22:18:28.011Z{"msg":"3262161a-3bac-40c2-8289-17347ebd8081 is now active with session: 02b39692-23c1-40cd-88e4-1b830476b8fa","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886684573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
140032024-02-28T22:18:28.012Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886703773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655,"session_id":"02b39692-23c1-40cd-88e4-1b830476b8fa"}
140042024-02-28T22:18:28.012Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:26.886731272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
140052024-02-28T22:18:28.012ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140062024-02-28T22:18:28.012ZLoading write count information from file "/tmp/test_repair_verify.data"
140072024-02-28T22:18:28.012ZRun Repair workload
140082024-02-28T22:18:28.012Z01/30 Write block 272 len 32768 data: 5 6 6 6 6 6 7 8
140092024-02-28T22:18:28.012Z02/30 Write block 141 len 32768 data: 10 11 13 12 13 13 12 13
140102024-02-28T22:18:28.012Z03/30 Flush
140112024-02-28T22:18:28.012Z04/30 Write block 77 len 12288 data: 10 10 9
140122024-02-28T22:18:28.012Z05/30 Write block 114 len 32768 data: 12 11 11 9 7 7 8 10
140132024-02-28T22:18:28.012Z06/30 Read block 312 len 16384
140142024-02-28T22:18:28.012Z07/30 Read block 492 len 20480
140152024-02-28T22:18:28.013Z08/30 Read block 508 len 12288
140162024-02-28T22:18:28.017Z09/30 Flush
140172024-02-28T22:18:28.017Z10/30 Write block 433 len 24576 data: 11 12 12 13 12 12
140182024-02-28T22:18:28.017Z11/30 Read block 573 len 40960
140192024-02-28T22:18:28.017Z12/30 Read block 531 len 36864
140202024-02-28T22:18:28.020Z13/30 Write block 404 len 36864 data: 10 11 12 12 11 9 8 7 8
140212024-02-28T22:18:28.020Z14/30 Write block 438 len 32768 data: 13 11 10 9 10 7 10 11
140222024-02-28T22:18:28.020Z15/30 Write block 522 len 20480 data: 11 8 8 10 9
140232024-02-28T22:18:28.020Z16/30 Write block 208 len 36864 data: 8 7 9 10 11 10 9 6 6
140242024-02-28T22:18:28.024Z17/30 Read block 27 len 20480
140252024-02-28T22:18:28.024Z18/30 Read block 420 len 40960
140262024-02-28T22:18:28.024Z19/30 Write block 220 len 12288 data: 5 6 5
140272024-02-28T22:18:28.024Z20/30 Write block 372 len 8192 data: 15 12
140282024-02-28T22:18:28.024Z21/30 Read block 284 len 32768
140292024-02-28T22:18:28.028Z22/30 Flush
140302024-02-28T22:18:28.031Z23/30 Write block 226 len 36864 data: 10 8 10 14 13 17 16 14 14
140312024-02-28T22:18:28.031Z24/30 Read block 92 len 8192
140322024-02-28T22:18:28.031Z25/30 Flush
140332024-02-28T22:18:28.035Z26/30 Write block 404 len 28672 data: 11 12 13 13 12 10 9
140342024-02-28T22:18:28.035Z27/30 Write block 383 len 40960 data: 15 15 14 14 11 9 8 11 12 11
140352024-02-28T22:18:28.035Z28/30 Write block 108 len 12288 data: 14 18 16
140362024-02-28T22:18:28.035Z29/30 Write block 386 len 24576 data: 15 12 10 9 12 13
140372024-02-28T22:18:28.035Z30/30 Read block 436 len 4096
140382024-02-28T22:18:28.040Z----------------------------------------------------------------
140392024-02-28T22:18:28.040Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:27
140402024-02-28T22:18:28.040ZStates: Active Active Active
140412024-02-28T22:18:28.040ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140422024-02-28T22:18:28.040Z 4 Acked 1003 Write 3 Done Done Done false
140432024-02-28T22:18:28.040Z 5 Acked 1004 Write 8 Done Done Done false
140442024-02-28T22:18:28.040Z 6 Acked 1005 Read 4 Done Done Done false
140452024-02-28T22:18:28.040Z 7 Acked 1006 Read 5 Done Done Sent false
140462024-02-28T22:18:28.040Z 8 Acked 1007 Read 3 Done Done Sent false
140472024-02-28T22:18:28.040Z 9 Acked 1008 Flush 0 Done Done Sent false
140482024-02-28T22:18:28.040Z 10 Acked 1009 Write 6 Done Done Sent false
140492024-02-28T22:18:28.040Z 11 Acked 1010 Read 10 Done Done Sent false
140502024-02-28T22:18:28.040Z 12 Acked 1011 Read 9 Done Done Sent false
140512024-02-28T22:18:28.040Z 13 Acked 1012 Write 9 Done Done Sent false
140522024-02-28T22:18:28.041Z 14 Acked 1013 Write 8 Done Done Sent false
140532024-02-28T22:18:28.041Z 15 Acked 1014 Write 5 Done Done Sent false
140542024-02-28T22:18:28.041Z 16 Acked 1015 Write 9 Done Done Sent false
140552024-02-28T22:18:28.041Z 17 Acked 1016 Read 5 Done Done Sent false
140562024-02-28T22:18:28.041Z 18 Acked 1017 Read 10 Done Done Sent false
140572024-02-28T22:18:28.041Z 19 Acked 1018 Write 3 Done Done Sent false
140582024-02-28T22:18:28.041Z 20 Acked 1019 Write 2 Done Done Sent false
140592024-02-28T22:18:28.041Z 21 Acked 1020 Read 8 Done Done Sent false
140602024-02-28T22:18:28.041Z 22 Acked 1021 Flush 0 Done Done Sent false
140612024-02-28T22:18:28.041Z 23 Acked 1022 Write 9 Done Done Sent false
140622024-02-28T22:18:28.041Z 24 Acked 1023 Read 2 Done Done Sent false
140632024-02-28T22:18:28.041Z 25 Acked 1024 Flush 0 Done Done Sent false
140642024-02-28T22:18:28.041Z 26 Acked 1025 Write 7 Done Done Sent false
140652024-02-28T22:18:28.041Z 27 Acked 1026 Write 10 Done Done Sent false
140662024-02-28T22:18:28.042Z 28 Acked 1027 Write 3 Done Done Sent false
140672024-02-28T22:18:28.042Z 29 Acked 1028 Write 6 Done Done Sent false
140682024-02-28T22:18:28.042Z 30 Acked 1029 Read 1 Sent Done Sent false
140692024-02-28T22:18:28.042Z STATES DS:0 DS:1 DS:2 TOTAL
140702024-02-28T22:18:28.042Z New 0 0 0 0
140712024-02-28T22:18:28.042Z Sent 1 0 24 25
140722024-02-28T22:18:28.042Z Done 26 27 3 56
140732024-02-28T22:18:28.042Z Skipped 0 0 0 0
140742024-02-28T22:18:28.042Z Error 0 0 0 0
140752024-02-28T22:18:28.042ZLast Flush: 1024 1024 1002
140762024-02-28T22:18:28.042ZDownstairs last five completed: 1002 1001 1000
140772024-02-28T22:18:28.042ZUpstairs last five completed: 30 29 28 27 26
140782024-02-28T22:18:28.042Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:26.921680819Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1655}
140792024-02-28T22:18:28.042ZWrote out file "/tmp/test_repair_verify.data"
140802024-02-28T22:18:28.048Z
140812024-02-28T22:18:28.048ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140822024-02-28T22:18:28.060ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140832024-02-28T22:18:28.060Z 0 000-019 123 123 123 227 227 227 F F F
140842024-02-28T22:18:28.060Z 1 020-039 126 126 126 232 232 232 F F F
140852024-02-28T22:18:28.069Z 2 040-059 126 126 126 232 232 232 F F F
140862024-02-28T22:18:28.069Z 3 060-079 130 130 126 239 239 233 F F T <---
140872024-02-28T22:18:28.069Z 4 080-099 129 129 129 237 237 237 F F F
140882024-02-28T22:18:28.069Z 5 100-119 130 130 125 239 239 231 T T T <---
140892024-02-28T22:18:28.069Z 6 120-139 130 130 122 239 239 224 F F T <---
140902024-02-28T22:18:28.069Z 7 140-159 130 130 130 238 238 238 F F F
140912024-02-28T22:18:28.070Z 8 160-179 129 129 129 237 237 237 F F F
140922024-02-28T22:18:28.083Z 9 180-199 127 127 127 235 235 235 F F F
140932024-02-28T22:18:28.083Z 10 200-219 130 130 127 240 240 235 F F F <---
140942024-02-28T22:18:28.083Z 11 220-239 130 130 120 241 241 220 F F F <---
140952024-02-28T22:18:28.083Z 12 240-259 128 128 128 236 236 236 F F F
140962024-02-28T22:18:28.083Z 13 260-279 130 130 130 238 238 238 F F F
140972024-02-28T22:18:28.083Z 14 280-299 128 128 128 236 236 236 F F F
140982024-02-28T22:18:28.083Z 15 300-319 127 127 127 235 235 235 F F F
140992024-02-28T22:18:28.083Z 16 320-339 122 122 122 225 225 225 F F F
141002024-02-28T22:18:28.083Z 17 340-359 129 129 129 237 237 237 F F F
141012024-02-28T22:18:28.083Z 18 360-379 130 130 129 240 240 237 F F F <---
141022024-02-28T22:18:28.083Z 19 380-399 125 125 125 231 231 231 T T F <---
141032024-02-28T22:18:28.084Z 20 400-419 130 130 127 240 240 235 T T F <---
141042024-02-28T22:18:28.084Z 21 420-439 130 130 122 240 240 224 F F F <---
141052024-02-28T22:18:28.084Z 22 440-459 130 130 119 240 240 217 F F F <---
141062024-02-28T22:18:28.084Z 23 460-479 128 128 128 236 236 236 F F F
141072024-02-28T22:18:28.084Z 24 480-499 129 129 129 237 237 237 F F F
141082024-02-28T22:18:28.084Z 25 500-519 127 127 127 235 235 235 F F F
141092024-02-28T22:18:28.084Z 26 520-539 130 130 129 240 240 237 F F F <---
141102024-02-28T22:18:28.084Z 27 540-559 119 119 119 217 217 217 F F F
141112024-02-28T22:18:28.084Z 28 560-579 125 125 125 231 231 231 F F F
141122024-02-28T22:18:28.084Z 29 580-599 128 128 128 236 236 236 F F F
141132024-02-28T22:18:28.084ZMax gen: 130, Max flush: 241
141142024-02-28T22:18:28.084ZError: Difference in extent metadata found!
141152024-02-28T22:18:28.085ZA Difference in extent metadata is expected here
141162024-02-28T22:18:28.085ZOn loop 65
141172024-02-28T22:18:28.085Z
141182024-02-28T22:18:28.085ZVerifying data now
141192024-02-28T22:18:33.646ZNo change after verify
141202024-02-28T22:18:33.646ZLoop: 65 Downstairs dump after verify (and repair):
141212024-02-28T22:18:33.660ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141222024-02-28T22:18:33.660Z 0 000-019 123 123 123 227 227 227 F F F
141232024-02-28T22:18:33.660Z 1 020-039 126 126 126 232 232 232 F F F
141242024-02-28T22:18:33.660Z 2 040-059 126 126 126 232 232 232 F F F
141252024-02-28T22:18:33.661Z 3 060-079 130 130 130 239 239 239 F F F
141262024-02-28T22:18:33.661Z 4 080-099 129 129 129 237 237 237 F F F
141272024-02-28T22:18:33.661Z 5 100-119 131 131 131 242 242 242 F F F
141282024-02-28T22:18:33.661Z 6 120-139 130 130 130 239 239 239 F F F
141292024-02-28T22:18:33.661Z 7 140-159 130 130 130 238 238 238 F F F
141302024-02-28T22:18:33.661Z 8 160-179 129 129 129 237 237 237 F F F
141312024-02-28T22:18:33.661Z 9 180-199 127 127 127 235 235 235 F F F
141322024-02-28T22:18:33.661Z 10 200-219 130 130 130 240 240 240 F F F
141332024-02-28T22:18:33.661Z 11 220-239 130 130 130 241 241 241 F F F
141342024-02-28T22:18:33.661Z 12 240-259 128 128 128 236 236 236 F F F
141352024-02-28T22:18:33.661Z 13 260-279 130 130 130 238 238 238 F F F
141362024-02-28T22:18:33.662Z 14 280-299 128 128 128 236 236 236 F F F
141372024-02-28T22:18:33.662Z 15 300-319 127 127 127 235 235 235 F F F
141382024-02-28T22:18:33.662Z 16 320-339 122 122 122 225 225 225 F F F
141392024-02-28T22:18:33.666Z 17 340-359 129 129 129 237 237 237 F F F
141402024-02-28T22:18:33.666Z 18 360-379 130 130 130 240 240 240 F F F
141412024-02-28T22:18:33.666Z 19 380-399 131 131 131 242 242 242 F F F
141422024-02-28T22:18:33.666Z 20 400-419 131 131 131 242 242 242 F F F
141432024-02-28T22:18:33.666Z 21 420-439 130 130 130 240 240 240 F F F
141442024-02-28T22:18:33.667Z 22 440-459 130 130 130 240 240 240 F F F
141452024-02-28T22:18:33.667Z 23 460-479 128 128 128 236 236 236 F F F
141462024-02-28T22:18:33.667Z 24 480-499 129 129 129 237 237 237 F F F
141472024-02-28T22:18:33.667Z 25 500-519 127 127 127 235 235 235 F F F
141482024-02-28T22:18:33.667Z 26 520-539 130 130 130 240 240 240 F F F
141492024-02-28T22:18:33.667Z 27 540-559 119 119 119 217 217 217 F F F
141502024-02-28T22:18:33.667Z 28 560-579 125 125 125 231 231 231 F F F
141512024-02-28T22:18:33.667Z 29 580-599 128 128 128 236 236 236 F F F
141522024-02-28T22:18:33.667ZMax gen: 131, Max flush: 242
141532024-02-28T22:18:33.667Z
141542024-02-28T22:18:33.668ZBegin loop 66
141552024-02-28T22:18:33.668ZDownstairs to restart: 2
141562024-02-28T22:18:33.683Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568350299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
141572024-02-28T22:18:33.683Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
141582024-02-28T22:18:33.683Z,"time":"2024-02-28T22:18:32.56868771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
141592024-02-28T22:18:33.691Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568726309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
141602024-02-28T22:18:33.691Z{"msg":"Crucible 0e792044-5170-484f-b710-5eefd55f16ac has session id: f463560f-10d8-499a-8cf3-6c2ddefcac1f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568752378Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141612024-02-28T22:18:33.691Z{"msg":"Upstairs opts: Upstairs UUID: 0e792044-5170-484f-b710-5eefd55f16ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568790227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141622024-02-28T22:18:33.691Z{"msg":"Crucible stats registered with UUID: 0e792044-5170-484f-b710-5eefd55f16ac","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568833256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141632024-02-28T22:18:33.691Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568860995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
141642024-02-28T22:18:33.692Z{"msg":"0e792044-5170-484f-b710-5eefd55f16ac active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568913133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141652024-02-28T22:18:33.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.568959632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141662024-02-28T22:18:33.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569015001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141672024-02-28T22:18:33.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.56905037Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141682024-02-28T22:18:33.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569093768Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141692024-02-28T22:18:33.692Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569123598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141702024-02-28T22:18:33.692Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569154937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141712024-02-28T22:18:33.692Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569212045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141722024-02-28T22:18:33.693Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569267114Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141732024-02-28T22:18:33.693Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569295613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141742024-02-28T22:18:33.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569331822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141752024-02-28T22:18:33.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569360541Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141762024-02-28T22:18:33.693Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.56939867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141772024-02-28T22:18:33.693Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.569972174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141782024-02-28T22:18:33.693Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570007983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141792024-02-28T22:18:33.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570025003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141802024-02-28T22:18:33.694Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570044502Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141812024-02-28T22:18:33.694Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570078471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141822024-02-28T22:18:33.694Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.57010682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141832024-02-28T22:18:33.694Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570622736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141842024-02-28T22:18:33.694Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570657395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141852024-02-28T22:18:33.694Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.570674625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141862024-02-28T22:18:33.695Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571161801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141872024-02-28T22:18:33.695Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.57120046Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141882024-02-28T22:18:33.695Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571221369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141892024-02-28T22:18:33.695Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571323797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141902024-02-28T22:18:33.695Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571370245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141912024-02-28T22:18:33.695Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571413864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141922024-02-28T22:18:33.695Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571431494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663Wait for a query_work_queue command to finish before sending IO
141932024-02-28T22:18:33.695Z,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141942024-02-28T22:18:33.696Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571472493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141952024-02-28T22:18:33.696Z{"msg":"[0]R flush_numbers[0..12]: [227, 232, 232, 239, 237, 242, 239, 238, 237, 235, 240, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571499662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141962024-02-28T22:18:33.696Z{"msg":"[0]R generation[0..12]: [123, 126, 126, 130, 129, 131, 130, 130, 129, 127, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571532801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141972024-02-28T22:18:33.696Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.57156716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141982024-02-28T22:18:33.696Z{"msg":"[1]R flush_numbers[0..12]: [227, 232, 232, 239, 237, 242, 239, 238, 237, 235, 240, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571601069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
141992024-02-28T22:18:33.696Z{"msg":"[1]R generation[0..12]: [123, 126, 126, 130, 129, 131, 130, 130, 129, 127, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571629758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142002024-02-28T22:18:33.696Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571644788Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142012024-02-28T22:18:33.696Z{"msg":"[2]R flush_numbers[0..12]: [227, 232, 232, 239, 237, 242, 239, 238, 237, 235, 240, 241]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571676087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142022024-02-28T22:18:33.696Z{"msg":"[2]R generation[0..12]: [123, 126, 126, 130, 129, 131, 130, 130, 129, 127, 130, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571702506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142032024-02-28T22:18:33.696Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571727426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142042024-02-28T22:18:33.697Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571764985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142052024-02-28T22:18:33.697Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571788164Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142062024-02-28T22:18:33.697Z{"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571817243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142072024-02-28T22:18:33.697Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571840883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"":"downstairs","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142082024-02-28T22:18:33.697Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571872822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142092024-02-28T22:18:33.697Z{"msg":"Transition from WaitQuorum to Active"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142102024-02-28T22:18:33.697Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571899421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"0","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142112024-02-28T22:18:33.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.57192952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"1","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142122024-02-28T22:18:33.697Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571953059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"client":"2","":"downstairs","session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142132024-02-28T22:18:33.698Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.571979939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142142024-02-28T22:18:33.698Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.572001858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142152024-02-28T22:18:33.698Z{"msg":"0e792044-5170-484f-b710-5eefd55f16ac is now active with session: f463560f-10d8-499a-8cf3-6c2ddefcac1f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.572030637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142162024-02-28T22:18:33.698Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.572052287Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663,"session_id":"f463560f-10d8-499a-8cf3-6c2ddefcac1f"}
142172024-02-28T22:18:33.698Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:32.572078756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
142182024-02-28T22:18:33.698ZLoading write count information from file "/tmp/test_repair_verify.data"
142192024-02-28T22:18:33.698ZRun Repair workload
142202024-02-28T22:18:33.698Z01/30 Write block 564 len 28672 data: 13 14 16 18 15 12 11
142212024-02-28T22:18:33.703Z02/30 Flush
142222024-02-28T22:18:33.703Z03/30 Write block 217 len 20480 data: 6 6 6 6 7
142232024-02-28T22:18:33.703Z04/30 Write block 164 len 28672 data: 10 12 12 16 15 15 16
142242024-02-28T22:18:33.704Z05/30 Write block 262 len 36864 data: 13 11 10 9 11 9 8 7 7
142252024-02-28T22:18:33.704Z06/30 Read block 544 len 20480
142262024-02-28T22:18:33.704Z07/30 Read block 454 len 24576
142272024-02-28T22:18:33.704Z08/30 Read block 228 len 28672
142282024-02-28T22:18:33.704Z09/30 Read block 75 len 28672
142292024-02-28T22:18:33.704Z10/30 Read block 302 len 28672
142302024-02-28T22:18:33.704Z11/30 Write block 421 len 36864 data: 14 14 12 11 10 7 7 7 10
142312024-02-28T22:18:33.704Z12/30 Read block 337 len 40960
142322024-02-28T22:18:33.705Z13/30 Read block 244 len 40960
142332024-02-28T22:18:33.705Z14/30 Flush
142342024-02-28T22:18:33.709Z15/30 Write block 31 len 16384 data: 9 9 9 8
142352024-02-28T22:18:33.709Z16/30 Read block 299 len 20480
142362024-02-28T22:18:33.709Z17/30 Write block 111 len 16384 data: 14 14 11 13
142372024-02-28T22:18:33.709Z18/30 Read block 38 len 12288
142382024-02-28T22:18:33.709Z19/30 Write block 517 len 28672 data: 15 14 14 14 13 12 9
142392024-02-28T22:18:33.709Z20/30 Write block 436 len 16384 data: 14 13 14 12
142402024-02-28T22:18:33.725Z21/30 Read block 361 len 12288
142412024-02-28T22:18:33.725Z22/30 Read block 102 len 36864
142422024-02-28T22:18:33.725Z23/30 Write block 386 len 24576 data: 16 13 11 10 13 14
142432024-02-28T22:18:33.725Z24/30 Write block 91 len 8192 data: 12 13
142442024-02-28T22:18:33.725Z25/30 Write block 483 len 28672 data: 10 12 14 11 12 9 14
142452024-02-28T22:18:33.729Z26/30 Write block 374 len 40960 data: 13 12 12 12 12 11 12 11 15 16
142462024-02-28T22:18:33.730Z27/30 Write block 577 len 36864 data: 9 10 11 8 7 4 6 8 10
142472024-02-28T22:18:33.730Z28/30 Write block 150 len 12288 data: 9 7 7
142482024-02-28T22:18:33.730Z29/30 Read block 355 len 12288
142492024-02-28T22:18:33.730Z30/30 Write block 85 len 12288 data: 12 11 11
142502024-02-28T22:18:33.730Z----------------------------------------------------------------
142512024-02-28T22:18:33.730Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142522024-02-28T22:18:33.730ZStates: Active Active Active
142532024-02-28T22:18:33.730ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142542024-02-28T22:18:33.730Z 1 Acked 1000 Write 7 Done Done Sent false
142552024-02-28T22:18:33.730Z 2 Acked 1001 Flush 0 Done Done Sent false
142562024-02-28T22:18:33.730Z 3 Acked 1002 Write 5 Done Done Sent false
142572024-02-28T22:18:33.731Z 4 Acked 1003 Write 7 Done Done Sent false
142582024-02-28T22:18:33.731Z 5 Acked 1004 Write 9 Done Done Sent false
142592024-02-28T22:18:33.731Z 6 Acked 1005 Read 5 Done Done Sent false
142602024-02-28T22:18:33.731Z 7 Acked 1006 Read 6 Done Done Sent false
142612024-02-28T22:18:33.731Z 8 Acked 1007 Read 7 Done Done Sent false
142622024-02-28T22:18:33.731Z 9 Acked 1008 Read 7 Done Done Sent false
142632024-02-28T22:18:33.731Z 10 Acked 1009 Read 7 Done Done Sent false
142642024-02-28T22:18:33.731Z 11 Acked 1010 Write 9 Done Done Sent false
142652024-02-28T22:18:33.731Z 12 Acked 1011 Read 10 Done Done Sent false
142662024-02-28T22:18:33.731Z 13 Acked 1012 Read 10 Done Done Sent false
142672024-02-28T22:18:33.731Z 14 Acked 1013 Flush 0 Done Done Sent false
142682024-02-28T22:18:33.731Z 15 Acked 1014 Write 4 Done Done Sent false
142692024-02-28T22:18:33.731Z 16 Acked 1015 Read 5 Done Done Sent false
142702024-02-28T22:18:33.732Z 17 Acked 1016 Write 4 Done Done Sent false
142712024-02-28T22:18:33.732Z 18 Acked 1017 Read 3 Done Done Sent false
142722024-02-28T22:18:33.732Z 19 Acked 1018 Write 7 Done Done Sent false
142732024-02-28T22:18:33.732Z 20 Acked 1019 Write 4 Done Done Sent false
142742024-02-28T22:18:33.732Z 21 Acked 1020 Read 3 Done Done Sent false
142752024-02-28T22:18:33.732Z 22 Acked 1021 Read 9 Done Done Sent false
142762024-02-28T22:18:33.732Z 23 Acked 1022 Write 6 Done Done Sent false
142772024-02-28T22:18:33.732Z 24 Acked 1023 Write 2 Done Done Sent false
142782024-02-28T22:18:33.732Z 25 Acked 1024 Write 7 Done Done Sent false
142792024-02-28T22:18:33.732Z 26 Acked 1025 Write 10 Done Done Sent false
142802024-02-28T22:18:33.732Z 27 Acked 1026 Write 9 Done Done Sent false
142812024-02-28T22:18:33.733Z 28 Acked 1027 Write 3 Done Done Sent false
142822024-02-28T22:18:33.733Z 29 Acked 1028 Read 3 Done Done Sent false
142832024-02-28T22:18:33.733Z 30 Acked 1029 Write 3 Sent Sent Sent false
142842024-02-28T22:18:33.733Z STATES DS:0 DS:1 DS:2 TOTAL
142852024-02-28T22:18:33.733Z New 0 0 0 0
142862024-02-28T22:18:33.733Z Sent 1 1 30 32
142872024-02-28T22:18:33.733Z Done 29 29 0 58
142882024-02-28T22:18:33.733Z Skipped 0 0 0 0
142892024-02-28T22:18:33.733Z Error 0 0 0 0
142902024-02-28T22:18:33.733ZLast Flush: 1013 1013 0
142912024-02-28T22:18:33.733ZDownstairs last five completed:
142922024-02-28T22:18:33.733ZUpstairs last five completed: 30 29 28 27 26
142932024-02-28T22:18:33.733Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:32.602841389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1663}
142942024-02-28T22:18:33.734ZWrote out file "/tmp/test_repair_verify.data"
142952024-02-28T22:18:33.734Z
142962024-02-28T22:18:33.734ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
142972024-02-28T22:18:33.754ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
142982024-02-28T22:18:33.755Z 0 000-019 123 123 123 227 227 227 F F F
142992024-02-28T22:18:33.755Z 1 020-039 126 126 126 232 232 232 T T F <---
143002024-02-28T22:18:33.755Z 2 040-059 126 126 126 232 232 232 F F F
143012024-02-28T22:18:33.755Z 3 060-079 130 130 130 239 239 239 F F F
143022024-02-28T22:18:33.755Z 4 080-099 129 129 129 237 237 237 T T F <---
143032024-02-28T22:18:33.755Z 5 100-119 131 131 131 242 242 242 T T F <---
143042024-02-28T22:18:33.755Z 6 120-139 130 130 130 239 239 239 F F F
143052024-02-28T22:18:33.755Z 7 140-159 130 130 130 238 238 238 T T F <---
143062024-02-28T22:18:33.755Z 8 160-179 132 132 129 244 244 237 F F F <---
143072024-02-28T22:18:33.755Z 9 180-199 127 127 127 235 235 235 F F F
143082024-02-28T22:18:33.756Z 10 200-219 132 132 130 244 244 240 F F F <---
143092024-02-28T22:18:33.756Z 11 220-239 132 132 130 244 244 241 F F F <---
143102024-02-28T22:18:33.756Z 12 240-259 128 128 128 236 236 236 F F F
143112024-02-28T22:18:33.756Z 13 260-279 132 132 130 244 244 238 F F F <---
143122024-02-28T22:18:33.756Z 14 280-299 128 128 128 236 236 236 F F F
143132024-02-28T22:18:33.756Z 15 300-319 127 127 127 235 235 235 F F F
143142024-02-28T22:18:33.756Z 16 320-339 122 122 122 225 225 225 F F F
143152024-02-28T22:18:33.756Z 17 340-359 129 129 129 237 237 237 F F F
143162024-02-28T22:18:33.757Z 18 360-379 130 130 130 240 240 240 T T F <---
143172024-02-28T22:18:33.757Z 19 380-399 131 131 131 242 242 242 T T F <---
143182024-02-28T22:18:33.757Z 20 400-419 131 131 131 242 242 242 F F F
143192024-02-28T22:18:33.757Z 21 420-439 132 132 130 244 244 240 T T F <---
143202024-02-28T22:18:33.757Z 22 440-459 130 130 130 240 240 240 F F F
143212024-02-28T22:18:33.757Z 23 460-479 128 128 128 236 236 236 F F F
143222024-02-28T22:18:33.757Z 24 480-499 129 129 129 237 237 237 T T F <---
143232024-02-28T22:18:33.757Z 25 500-519 127 127 127 235 235 235 T T F <---
143242024-02-28T22:18:33.757Z 26 520-539 130 130 130 240 240 240 T T F <---
143252024-02-28T22:18:33.758Z 27 540-559 119 119 119 217 217 217 F F F
143262024-02-28T22:18:33.758Z 28 560-579 132 132 125 243 243 231 T T F <---
143272024-02-28T22:18:33.758Z 29 580-599 128 128 128 236 236 236 T T F <---
143282024-02-28T22:18:33.758ZMax gen: 132, Max flush: 244
143292024-02-28T22:18:33.758ZError: Difference in extent metadata found!
143302024-02-28T22:18:33.758ZA Difference in extent metadata is expected here
143312024-02-28T22:18:33.758ZOn loop 66
143322024-02-28T22:18:33.758Z
143332024-02-28T22:18:33.758ZVerifying data now
143342024-02-28T22:18:38.553ZNo change after verify
143352024-02-28T22:18:38.554ZLoop: 66 Downstairs dump after verify (and repair):
143362024-02-28T22:18:38.568ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143372024-02-28T22:18:38.568Z 0 000-019 123 123 123 227 227 227 F F F
143382024-02-28T22:18:38.569Z 1 020-039 133 133 133 245 245 245 F F F
143392024-02-28T22:18:38.569Z 2 040-059 126 126 126 232 232 232 F F F
143402024-02-28T22:18:38.569Z 3 060-079 130 130 130 239 239 239 F F F
143412024-02-28T22:18:38.569Z 4 080-099 133 133 133 245 245 245 F F F
143422024-02-28T22:18:38.569Z 5 100-119 133 133 133 245 245 245 F F F
143432024-02-28T22:18:38.569Z 6 120-139 130 130 130 239 239 239 F F F
143442024-02-28T22:18:38.570Z 7 140-159 133 133 133 245 245 245 F F F
143452024-02-28T22:18:38.570Z 8 160-179 132 132 132 244 244 244 F F F
143462024-02-28T22:18:38.570Z 9 180-199 127 127 127 235 235 235 F F F
143472024-02-28T22:18:38.574Z 10 200-219 132 132 132 244 244 244 F F F
143482024-02-28T22:18:38.574Z 11 220-239 132 132 132 244 244 244 F F F
143492024-02-28T22:18:38.574Z 12 240-259 128 128 128 236 236 236 F F F
143502024-02-28T22:18:38.574Z 13 260-279 132 132 132 244 244 244 F F F
143512024-02-28T22:18:38.575Z 14 280-299 128 128 128 236 236 236 F F F
143522024-02-28T22:18:38.575Z 15 300-319 127 127 127 235 235 235 F F F
143532024-02-28T22:18:38.575Z 16 320-339 122 122 122 225 225 225 F F F
143542024-02-28T22:18:38.575Z 17 340-359 129 129 129 237 237 237 F F F
143552024-02-28T22:18:38.575Z 18 360-379 133 133 133 245 245 245 F F F
143562024-02-28T22:18:38.575Z 19 380-399 133 133 133 245 245 245 F F F
143572024-02-28T22:18:38.575Z 20 400-419 131 131 131 242 242 242 F F F
143582024-02-28T22:18:38.575Z 21 420-439 133 133 133 245 245 245 F F F
143592024-02-28T22:18:38.575Z 22 440-459 130 130 130 240 240 240 F F F
143602024-02-28T22:18:38.576Z 23 460-479 128 128 128 236 236 236 F F F
143612024-02-28T22:18:38.576Z 24 480-499 133 133 133 245 245 245 F F F
143622024-02-28T22:18:38.576Z 25 500-519 133 133 133 245 245 245 F F F
143632024-02-28T22:18:38.576Z 26 520-539 133 133 133 245 245 245 F F F
143642024-02-28T22:18:38.576Z 27 540-559 119 119 119 217 217 217 F F F
143652024-02-28T22:18:38.576Z 28 560-579 133 133 133 245 245 245 F F F
143662024-02-28T22:18:38.576Z 29 580-599 133 133 133 245 245 245 F F F
143672024-02-28T22:18:38.577ZMax gen: 133, Max flush: 245
143682024-02-28T22:18:38.577Z
143692024-02-28T22:18:38.577ZBegin loop 67
143702024-02-28T22:18:38.577ZDownstairs to restart: 2
143712024-02-28T22:18:38.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47521542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
143722024-02-28T22:18:38.591Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475541501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
143732024-02-28T22:18:38.598Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47557942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
143742024-02-28T22:18:38.599Z{"msg":"Crucible runtime is spawned
143752024-02-28T22:18:38.599ZCrucible 4b28614c-65cf-4352-9571-a5fa61dd69d2 has session id: 2842f78e-984a-4851-9234-60e2a2575783","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475650928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143762024-02-28T22:18:38.599Z{"msg":"Upstairs opts: Upstairs UUID: 4b28614c-65cf-4352-9571-a5fa61dd69d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475708236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143772024-02-28T22:18:38.599Z{"msg":"Crucible stats registered with UUID: 4b28614c-65cf-4352-9571-a5fa61dd69d2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475752095Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143782024-02-28T22:18:38.599Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475795554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
143792024-02-28T22:18:38.599Z{"msg":"4b28614c-65cf-4352-9571-a5fa61dd69d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475879362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143802024-02-28T22:18:38.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475913701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143812024-02-28T22:18:38.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47593097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143822024-02-28T22:18:38.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475960539Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143832024-02-28T22:18:38.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.475986069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143842024-02-28T22:18:38.600Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476027407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143852024-02-28T22:18:38.600Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476057117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143862024-02-28T22:18:38.600Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476091906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143872024-02-28T22:18:38.600Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476133024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143882024-02-28T22:18:38.601Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476156504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143892024-02-28T22:18:38.601Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47631507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143902024-02-28T22:18:38.601Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476370288Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143912024-02-28T22:18:38.601Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.476400777Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143922024-02-28T22:18:38.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47701896Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143932024-02-28T22:18:38.601Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477077238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143942024-02-28T22:18:38.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477105218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143952024-02-28T22:18:38.601Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477129247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143962024-02-28T22:18:38.601Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477545426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143972024-02-28T22:18:38.601Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477590264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143982024-02-28T22:18:38.601Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477623934Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
143992024-02-28T22:18:38.601Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.477668852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144002024-02-28T22:18:38.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478240217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144012024-02-28T22:18:38.601Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478276685Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144022024-02-28T22:18:38.601Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478335234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144032024-02-28T22:18:38.601Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478378463Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144042024-02-28T22:18:38.602Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478396672Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144052024-02-28T22:18:38.602Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478418781Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144062024-02-28T22:18:38.602Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478433201Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144072024-02-28T22:18:38.602Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478757022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144082024-02-28T22:18:38.602Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478776782Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144092024-02-28T22:18:38.602Z{"msg":"[0]R flush_numbers[0..12]: [227, 245, 232, 239, 245, 245, 239, 245, 244, 235, 244, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478791861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144102024-02-28T22:18:38.602Z{"msg":"[0]R generation[0..12]: [123, 133, 126, 130, 133, 133, 130, 133, 132, 127, 132, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478806161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144112024-02-28T22:18:38.602Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478821261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144122024-02-28T22:18:38.602Z{"msg":"[1]R flush_numbers[0..12]: [227, 245, 232, 239, 245, 245, 239, 245, 244, 235, 244, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47883521Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144132024-02-28T22:18:38.602Z{"msg":"[1]R generation[0..12]: [123, 133, 126, 130, 133, 133, 130, 133, 132, 127, 132, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.4788582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144142024-02-28T22:18:38.602Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478873379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144152024-02-28T22:18:38.602Z{"msg":"[2]R flush_numbers[0..12]: [227, 245, 232, 239, 245, 245, 239, 245, 244, 235, 244, 244]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478895359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144162024-02-28T22:18:38.602Z{"msg":"[2]R generation[0..12]: [123, 133, 126, 130, 133, 133, 130, 133, 132, 127, 132, 132]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478909998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairsWait for a query_work_queue command to finish before sending IO
144172024-02-28T22:18:38.602Z","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144182024-02-28T22:18:38.602Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478940247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144192024-02-28T22:18:38.602Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478954257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144202024-02-28T22:18:38.602Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478974726Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144212024-02-28T22:18:38.602Z{"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.478992546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144222024-02-28T22:18:38.602Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479027385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"":"downstairs","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144232024-02-28T22:18:38.602Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479051974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144242024-02-28T22:18:38.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479066784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"0","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144252024-02-28T22:18:38.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479099873Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"1","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144262024-02-28T22:18:38.603Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479130732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"client":"2","":"downstairs","session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144272024-02-28T22:18:38.603Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479164241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144282024-02-28T22:18:38.603Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.47919803Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144292024-02-28T22:18:38.603Z{"msg":"4b28614c-65cf-4352-9571-a5fa61dd69d2 is now active with session: 2842f78e-984a-4851-9234-60e2a2575783","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479234459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144302024-02-28T22:18:38.603Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:37.479264828Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671,"session_id":"2842f78e-984a-4851-9234-60e2a2575783"}
144312024-02-28T22:18:38.603Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144322024-02-28T22:18:38.603Z,"time":"2024-02-28T22:18:37.479295277Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
144332024-02-28T22:18:38.603ZLoading write count information from file "/tmp/test_repair_verify.data"
144342024-02-28T22:18:38.603ZRun Repair workload
144352024-02-28T22:18:38.609Z01/30 Write block 397 len 20480 data: 9 8 7 9 9
144362024-02-28T22:18:38.609Z02/30 Write block 432 len 4096 data: 10
144372024-02-28T22:18:38.609Z03/30 Read block 121 len 16384
144382024-02-28T22:18:38.609Z04/30 Read block 153 len 20480
144392024-02-28T22:18:38.609Z05/30 Read block 496 len 28672
144402024-02-28T22:18:38.609Z06/30 Read block 405 len 8192
144412024-02-28T22:18:38.609Z07/30 Write block 131 len 28672 data: 15 14 15 14 14 14 14
144422024-02-28T22:18:38.610Z08/30 Read block 472 len 4096
144432024-02-28T22:18:38.610Z09/30 Write block 44 len 8192 data: 6 8
144442024-02-28T22:18:38.610Z10/30 Read block 139 len 32768
144452024-02-28T22:18:38.610Z11/30 Flush
144462024-02-28T22:18:38.610Z12/30 Flush
144472024-02-28T22:18:38.626Z13/30 Write block 225 len 4096 data: 8
144482024-02-28T22:18:38.626Z14/30 Write block 314 len 16384 data: 11 8 8 6
144492024-02-28T22:18:38.626Z15/30 Write block 220 len 36864 data: 7 8 6 5 6 9 11 9 11
144502024-02-28T22:18:38.626Z16/30 Read block 126 len 24576
144512024-02-28T22:18:38.626Z17/30 Read block 408 len 28672
144522024-02-28T22:18:38.626Z18/30 Read block 8 len 36864
144532024-02-28T22:18:38.626Z19/30 Read block 115 len 24576
144542024-02-28T22:18:38.626Z20/30 Read block 176 len 36864
144552024-02-28T22:18:38.632Z21/30 Read block 6 len 20480
144562024-02-28T22:18:38.632Z22/30 Write block 387 len 32768 data: 14 12 11 14 15 12 9 8
144572024-02-28T22:18:38.632Z23/30 Read block 339 len 32768
144582024-02-28T22:18:38.632Z24/30 Read block 123 len 36864
144592024-02-28T22:18:38.632Z25/30 Read block 492 len 32768
144602024-02-28T22:18:38.633Z26/30 Flush
144612024-02-28T22:18:38.633Z27/30 Write block 172 len 40960 data: 13 14 14 12 10 9 7 5 5 6
144622024-02-28T22:18:38.633Z28/30 Read block 369 len 4096
144632024-02-28T22:18:38.633Z29/30 Write block 262 len 28672 data: 14 12 11 10 12 10 9
144642024-02-28T22:18:38.633Z30/30 Write block 352 len 16384 data: 9 10 10 9
144652024-02-28T22:18:38.633Z----------------------------------------------------------------
144662024-02-28T22:18:38.633Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144672024-02-28T22:18:38.634ZStates: Active Active Active
144682024-02-28T22:18:38.634ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144692024-02-28T22:18:38.634Z 1 Acked 1000 Write 5 Done Done Done false
144702024-02-28T22:18:38.634Z 2 Acked 1001 Write 1 Done Done Done false
144712024-02-28T22:18:38.634Z 3 Acked 1002 Read 4 Done Done Done false
144722024-02-28T22:18:38.634Z 4 Acked 1003 Read 5 Done Done Done false
144732024-02-28T22:18:38.634Z 5 Acked 1004 Read 7 Done Done Done false
144742024-02-28T22:18:38.635Z 6 Acked 1005 Read 2 Done Done Done false
144752024-02-28T22:18:38.635Z 7 Acked 1006 Write 7 Done Done Sent false
144762024-02-28T22:18:38.635Z 8 Acked 1007 Read 1 Done Done Sent false
144772024-02-28T22:18:38.635Z 9 Acked 1008 Write 2 Done Done Sent false
144782024-02-28T22:18:38.635Z 10 Acked 1009 Read 8 Done Done Sent false
144792024-02-28T22:18:38.635Z 11 Acked 1010 Flush 0 Done Done Sent false
144802024-02-28T22:18:38.635Z 12 Acked 1011 Flush 0 Done Done Sent false
144812024-02-28T22:18:38.636Z 13 Acked 1012 Write 1 Done Done Sent false
144822024-02-28T22:18:38.636Z 14 Acked 1013 Write 4 Done Done Sent false
144832024-02-28T22:18:38.636Z 15 Acked 1014 Write 9 Done Done Sent false
144842024-02-28T22:18:38.636Z 16 Acked 1015 Read 6 Done Done Sent false
144852024-02-28T22:18:38.636Z 17 Acked 1016 Read 7 Done Done Sent false
144862024-02-28T22:18:38.636Z 18 Acked 1017 Read 9 Done Done Sent false
144872024-02-28T22:18:38.636Z 19 Acked 1018 Read 6 Done Done Sent false
144882024-02-28T22:18:38.636Z 20 Acked 1019 Read 9 Done Done Sent false
144892024-02-28T22:18:38.637Z 21 Acked 1020 Read 5 Done Done Sent false
144902024-02-28T22:18:38.637Z 22 Acked 1021 Write 8 Done Done Sent false
144912024-02-28T22:18:38.637Z 23 Acked 1022 Read 8 Done Done Sent false
144922024-02-28T22:18:38.637Z 24 Acked 1023 Read 9 Done Done Sent false
144932024-02-28T22:18:38.637Z 25 Acked 1024 Read 8 Done Done Sent false
144942024-02-28T22:18:38.637Z 26 Acked 1025 Flush 0 Done Done Sent false
144952024-02-28T22:18:38.637Z 27 Acked 1026 Write 10 Done Done Sent false
144962024-02-28T22:18:38.637Z 28 Acked 1027 Read 1 Done Done Sent false
144972024-02-28T22:18:38.637Z 29 Acked 1028 Write 7 Done Done Sent false
144982024-02-28T22:18:38.638Z 30 Acked 1029 Write 4 Sent Sent Sent false
144992024-02-28T22:18:38.638Z STATES DS:0 DS:1 DS:2 TOTAL
145002024-02-28T22:18:38.638Z New 0 0 0 0
145012024-02-28T22:18:38.638Z Sent 1 1 24 26
145022024-02-28T22:18:38.638Z Done 29 29 6 64
145032024-02-28T22:18:38.638Z Skipped 0 0 0 0
145042024-02-28T22:18:38.638Z Error 0 0 0 0
145052024-02-28T22:18:38.638ZLast Flush: 1025 1025 0
145062024-02-28T22:18:38.639ZDownstairs last five completed:
145072024-02-28T22:18:38.639ZUpstairs last five completed: 30 29 28 27 26
145082024-02-28T22:18:38.639Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:37.511542119Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1671}
145092024-02-28T22:18:38.639ZWrote out file "/tmp/test_repair_verify.data"
145102024-02-28T22:18:38.639Z
145112024-02-28T22:18:38.643ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145122024-02-28T22:18:38.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145132024-02-28T22:18:38.650Z 0 000-019 123 123 123 227 227 227 F F F
145142024-02-28T22:18:38.650Z 1 020-039 133 133 133 245 245 245 F F F
145152024-02-28T22:18:38.650Z 2 040-059 134 134 126 246 246 232 F F F <---
145162024-02-28T22:18:38.650Z 3 060-079 130 130 130 239 239 239 F F F
145172024-02-28T22:18:38.651Z 4 080-099 133 133 133 245 245 245 F F F
145182024-02-28T22:18:38.651Z 5 100-119 133 133 133 245 245 245 F F F
145192024-02-28T22:18:38.651Z 6 120-139 134 134 130 246 246 239 F F F <---
145202024-02-28T22:18:38.651Z 7 140-159 133 133 133 245 245 245 F F F
145212024-02-28T22:18:38.655Z 8 160-179 132 132 132 244 244 244 T T F <---
145222024-02-28T22:18:38.655Z 9 180-199 127 127 127 235 235 235 T T F <---
145232024-02-28T22:18:38.655Z 10 200-219 132 132 132 244 244 244 F F F
145242024-02-28T22:18:38.655Z 11 220-239 134 134 132 248 248 244 F F F <---
145252024-02-28T22:18:38.656Z 12 240-259 128 128 128 236 236 236 F F F
145262024-02-28T22:18:38.656Z 13 260-279 132 132 132 244 244 244 T T F <---
145272024-02-28T22:18:38.656Z 14 280-299 128 128 128 236 236 236 F F F
145282024-02-28T22:18:38.656Z 15 300-319 134 134 127 248 248 235 F F F <---
145292024-02-28T22:18:38.656Z 16 320-339 122 122 122 225 225 225 F F F
145302024-02-28T22:18:38.656Z 17 340-359 129 129 129 237 237 237 T T F <---
145312024-02-28T22:18:38.657Z 18 360-379 133 133 133 245 245 245 F F F
145322024-02-28T22:18:38.657Z 19 380-399 134 134 133 248 248 245 F F T <---
145332024-02-28T22:18:38.657Z 20 400-419 134 134 131 246 246 242 F F T <---
145342024-02-28T22:18:38.657Z 21 420-439 134 134 133 246 246 245 F F T <---
145352024-02-28T22:18:38.657Z 22 440-459 130 130 130 240 240 240 F F F
145362024-02-28T22:18:38.657Z 23 460-479 128 128 128 236 236 236 F F F
145372024-02-28T22:18:38.657Z 24 480-499 133 133 133 245 245 245 F F F
145382024-02-28T22:18:38.657Z 25 500-519 133 133 133 245 245 245 F F F
145392024-02-28T22:18:38.658Z 26 520-539 133 133 133 245 245 245 F F F
145402024-02-28T22:18:38.658Z 27 540-559 119 119 119 217 217 217 F F F
145412024-02-28T22:18:38.658Z 28 560-579 133 133 133 245 245 245 F F F
145422024-02-28T22:18:38.658Z 29 580-599 133 133 133 245 245 245 F F F
145432024-02-28T22:18:38.658ZMax gen: 134, Max flush: 248
145442024-02-28T22:18:38.659ZError: Difference in extent metadata found!
145452024-02-28T22:18:38.659ZA Difference in extent metadata is expected here
145462024-02-28T22:18:38.659ZOn loop 67
145472024-02-28T22:18:38.659Z
145482024-02-28T22:18:38.664ZVerifying data now
145492024-02-28T22:18:44.238ZNo change after verify
145502024-02-28T22:18:44.238ZLoop: 67 Downstairs dump after verify (and repair):
145512024-02-28T22:18:44.252ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145522024-02-28T22:18:44.252Z 0 000-019 123 123 123 227 227 227 F F F
145532024-02-28T22:18:44.252Z 1 020-039 133 133 133 245 245 245 F F F
145542024-02-28T22:18:44.252Z 2 040-059 134 134 134 246 246 246 F F F
145552024-02-28T22:18:44.252Z 3 060-079 130 130 130 239 239 239 F F F
145562024-02-28T22:18:44.252Z 4 080-099 133 133 133 245 245 245 F F F
145572024-02-28T22:18:44.253Z 5 100-119 133 133 133 245 245 245 F F F
145582024-02-28T22:18:44.253Z 6 120-139 134 134 134 246 246 246 F F F
145592024-02-28T22:18:44.253Z 7 140-159 133 133 133 245 245 245 F F F
145602024-02-28T22:18:44.253Z 8 160-179 135 135 135 249 249 249 F F F
145612024-02-28T22:18:44.253Z 9 180-199 135 135 135 249 249 249 F F F
145622024-02-28T22:18:44.253Z 10 200-219 132 132 132 244 244 244 F F F
145632024-02-28T22:18:44.253Z 11 220-239 134 134 134 248 248 248 F F F
145642024-02-28T22:18:44.253Z 12 240-259 128 128 128 236 236 236 F F F
145652024-02-28T22:18:44.253Z 13 260-279 135 135 135 249 249 249 F F F
145662024-02-28T22:18:44.258Z 14 280-299 128 128 128 236 236 236 F F F
145672024-02-28T22:18:44.258Z 15 300-319 134 134 134 248 248 248 F F F
145682024-02-28T22:18:44.258Z 16 320-339 122 122 122 225 225 225 F F F
145692024-02-28T22:18:44.258Z 17 340-359 135 135 135 249 249 249 F F F
145702024-02-28T22:18:44.258Z 18 360-379 133 133 133 245 245 245 F F F
145712024-02-28T22:18:44.258Z 19 380-399 134 134 134 248 248 248 F F F
145722024-02-28T22:18:44.259Z 20 400-419 134 134 134 246 246 246 F F F
145732024-02-28T22:18:44.259Z 21 420-439 134 134 134 246 246 246 F F F
145742024-02-28T22:18:44.259Z 22 440-459 130 130 130 240 240 240 F F F
145752024-02-28T22:18:44.259Z 23 460-479 128 128 128 236 236 236 F F F
145762024-02-28T22:18:44.259Z 24 480-499 133 133 133 245 245 245 F F F
145772024-02-28T22:18:44.259Z 25 500-519 133 133 133 245 245 245 F F F
145782024-02-28T22:18:44.259Z 26 520-539 133 133 133 245 245 245 F F F
145792024-02-28T22:18:44.259Z 27 540-559 119 119 119 217 217 217 F F F
145802024-02-28T22:18:44.260Z 28 560-579 133 133 133 245 245 245 F F F
145812024-02-28T22:18:44.260Z 29 580-599 133 133 133 245 245 245 F F F
145822024-02-28T22:18:44.260ZMax gen: 135, Max flush: 249
145832024-02-28T22:18:44.260Z
145842024-02-28T22:18:44.260ZBegin loop 68
145852024-02-28T22:18:44.260ZDownstairs to restart: 1
145862024-02-28T22:18:44.279Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160138956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
145872024-02-28T22:18:44.279Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160407448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
145882024-02-28T22:18:44.287Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160447587Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
145892024-02-28T22:18:44.287Z{"msg":"Crucible runtime is spawned
145902024-02-28T22:18:44.287ZCrucible c80e7f3d-dbf8-428e-8652-ead47814bdc8 has session id: cb3c9346-0e7b-48d6-8679-4cdbde36f70d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160486976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145912024-02-28T22:18:44.287Z{"msg":"Upstairs opts: Upstairs UUID: c80e7f3d-dbf8-428e-8652-ead47814bdc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160531835Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145922024-02-28T22:18:44.288Z{"msg":"Crucible stats registered with UUID: c80e7f3d-dbf8-428e-8652-ead47814bdc8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160568304Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145932024-02-28T22:18:44.288Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160602693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
145942024-02-28T22:18:44.288Z{"msg":"c80e7f3d-dbf8-428e-8652-ead47814bdc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160675111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145952024-02-28T22:18:44.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.16071637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145962024-02-28T22:18:44.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160761718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145972024-02-28T22:18:44.288Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160780798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145982024-02-28T22:18:44.288Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160804827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
145992024-02-28T22:18:44.289Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160826767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146002024-02-28T22:18:44.289Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160850716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146012024-02-28T22:18:44.289Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160875086Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146022024-02-28T22:18:44.289Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160907234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146032024-02-28T22:18:44.289Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.160934274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146042024-02-28T22:18:44.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.161013852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146052024-02-28T22:18:44.289Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.16106145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146062024-02-28T22:18:44.290Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.161149288Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146072024-02-28T22:18:44.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.161696473Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146082024-02-28T22:18:44.290Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.161751981Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146092024-02-28T22:18:44.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.16180324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146102024-02-28T22:18:44.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.161832049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146112024-02-28T22:18:44.290Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162210059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146122024-02-28T22:18:44.290Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162278447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146132024-02-28T22:18:44.290Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162306996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146142024-02-28T22:18:44.290Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162338235Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146152024-02-28T22:18:44.291Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162903579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146162024-02-28T22:18:44.291Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162935049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146172024-02-28T22:18:44.291Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.162960458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146182024-02-28T22:18:44.291Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163007067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146192024-02-28T22:18:44.291Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163027006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146202024-02-28T22:18:44.291Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163054875Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146212024-02-28T22:18:44.291Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163069385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146222024-02-28T22:18:44.292Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163428195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146232024-02-28T22:18:44.292Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163474044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146242024-02-28T22:18:44.292Z{"msg":"[0]R flush_numbers[0..12]: [227, 245, 246, 239, 245, 245, 246, 245, 249, 249, 244, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163500923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146252024-02-28T22:18:44.292Z{"msg":"[0]R generation[0..12]: [123, 133, 134, 130, 133, 133, 134, 133, 135, 135, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163536422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146262024-02-28T22:18:44.292Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163572661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146272024-02-28T22:18:44.292Z{"msg":"[1]R flush_numbers[0..12]: [227, 245, 246, 239, 245, 245, 246, 245, 249, 249, 244, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.16360946Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146282024-02-28T22:18:44.292ZWait for a query_work_queue command to finish before sending IO
146292024-02-28T22:18:44.292Z{"msg":"[1]R generation[0..12]: [123, 133, 134, 130, 133, 133, 134, 133, 135, 135, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163652309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146302024-02-28T22:18:44.293Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163694518Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146312024-02-28T22:18:44.293Z{"msg":"[2]R flush_numbers[0..12]: [227, 245, 246, 239, 245, 245, 246, 245, 249, 249, 244, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163733987Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146322024-02-28T22:18:44.293Z{"msg":"[2]R generation[0..12]: [123, 133, 134, 130, 133, 133, 134, 133, 135, 135, 132, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163770076Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146332024-02-28T22:18:44.293Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163810585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146342024-02-28T22:18:44.293Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163850033Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146352024-02-28T22:18:44.293Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163888163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146362024-02-28T22:18:44.293Z{"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.163925601Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146372024-02-28T22:18:44.293Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.16396312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"":"downstairs","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146382024-02-28T22:18:44.293Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164000039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146392024-02-28T22:18:44.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146402024-02-28T22:18:44.294Z,"time":"2024-02-28T22:18:43.164025689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"0","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146412024-02-28T22:18:44.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164057638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"1","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146422024-02-28T22:18:44.294Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164082467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"client":"2","":"downstairs","session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146432024-02-28T22:18:44.294Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164096817Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146442024-02-28T22:18:44.294Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164116526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146452024-02-28T22:18:44.294Z{"msg":"c80e7f3d-dbf8-428e-8652-ead47814bdc8 is now active with session: cb3c9346-0e7b-48d6-8679-4cdbde36f70d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164130266Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146462024-02-28T22:18:44.295Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164149455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679,"session_id":"cb3c9346-0e7b-48d6-8679-4cdbde36f70d"}
146472024-02-28T22:18:44.295Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:43.164162315Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
146482024-02-28T22:18:44.295ZLoading write count information from file "/tmp/test_repair_verify.data"
146492024-02-28T22:18:44.295ZRun Repair workload
146502024-02-28T22:18:44.295Z01/30 Write block 421 len 24576 data: 15 15 13 12 11 8
146512024-02-28T22:18:44.295Z02/30 Flush
146522024-02-28T22:18:44.300Z03/30 Write block 355 len 36864 data: 10 9 9 8 6 6 6 9 10
146532024-02-28T22:18:44.300Z04/30 Read block 496 len 16384
146542024-02-28T22:18:44.300Z05/30 Write block 317 len 24576 data: 7 7 7 7 8 9
146552024-02-28T22:18:44.300Z06/30 Flush
146562024-02-28T22:18:44.300Z07/30 Write block 371 len 4096 data: 13
146572024-02-28T22:18:44.300Z08/30 Write block 253 len 28672 data: 10 10 9 12 13 13 12
146582024-02-28T22:18:44.300Z09/30 Write block 366 len 4096 data: 6
146592024-02-28T22:18:44.301Z10/30 Read block 352 len 4096
146602024-02-28T22:18:44.301Z11/30 Read block 308 len 4096
146612024-02-28T22:18:44.301Z12/30 Write block 88 len 4096 data: 12
146622024-02-28T22:18:44.301Z13/30 Write block 378 len 32768 data: 13 12 13 12 16 17 16 15
146632024-02-28T22:18:44.301Z14/30 Write block 217 len 4096 data: 7
146642024-02-28T22:18:44.301Z15/30 Read block 275 len 36864
146652024-02-28T22:18:44.301Z16/30 Flush
146662024-02-28T22:18:44.301Z17/30 Write block 199 len 36864 data: 8 7 9 9 8 10 12 9 7
146672024-02-28T22:18:44.307Z18/30 Read block 425 len 20480
146682024-02-28T22:18:44.307Z19/30 Write block 305 len 28672 data: 10 10 13 12 12 8 13
146692024-02-28T22:18:44.307Z20/30 Read block 116 len 12288
146702024-02-28T22:18:44.307Z21/30 Write block 257 len 40960 data: 14 14 13 12 12 15 13 12 11 13
146712024-02-28T22:18:44.307Z22/30 Read block 40 len 20480
146722024-02-28T22:18:44.307Z23/30 Read block 145 len 40960
146732024-02-28T22:18:44.311Z24/30 Write block 449 len 40960 data: 13 13 13 11 13 11 11 10 9 11
146742024-02-28T22:18:44.311Z25/30 Write block 551 len 36864 data: 7 7 7 5 6 7 8 8 8
146752024-02-28T22:18:44.311Z26/30 Write block 404 len 16384 data: 12 13 14 14
146762024-02-28T22:18:44.311Z27/30 Read block 424 len 32768
146772024-02-28T22:18:44.311Z28/30 Write block 462 len 8192 data: 8 9
146782024-02-28T22:18:44.311Z29/30 Read block 499 len 8192
146792024-02-28T22:18:44.316Z30/30 Read block 506 len 8192
146802024-02-28T22:18:44.316Z----------------------------------------------------------------
146812024-02-28T22:18:44.316Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:28
146822024-02-28T22:18:44.316ZStates: Active Active Active
146832024-02-28T22:18:44.316ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
146842024-02-28T22:18:44.316Z 3 Acked 1002 Write 9 Done Done Done false
146852024-02-28T22:18:44.316Z 4 Acked 1003 Read 4 Done Done Done false
146862024-02-28T22:18:44.316Z 5 Acked 1004 Write 6 Done Done Done false
146872024-02-28T22:18:44.317Z 6 Acked 1005 Flush 0 Done Sent Done false
146882024-02-28T22:18:44.317Z 7 Acked 1006 Write 1 Done Sent Done false
146892024-02-28T22:18:44.317Z 8 Acked 1007 Write 7 Done Sent Done false
146902024-02-28T22:18:44.317Z 9 Acked 1008 Write 1 Done Sent Done false
146912024-02-28T22:18:44.317Z 10 Acked 1009 Read 1 Done Sent Done false
146922024-02-28T22:18:44.317Z 11 Acked 1010 Read 1 Done Sent Done false
146932024-02-28T22:18:44.317Z 12 Acked 1011 Write 1 Done Sent Done false
146942024-02-28T22:18:44.317Z 13 Acked 1012 Write 8 Done Sent Done false
146952024-02-28T22:18:44.317Z 14 Acked 1013 Write 1 Done Sent Done false
146962024-02-28T22:18:44.317Z 15 Acked 1014 Read 9 Done Sent Done false
146972024-02-28T22:18:44.317Z 16 Acked 1015 Flush 0 Done Sent Done false
146982024-02-28T22:18:44.318Z 17 Acked 1016 Write 9 Done Sent Done false
146992024-02-28T22:18:44.318Z 18 Acked 1017 Read 5 Done Sent Done false
147002024-02-28T22:18:44.318Z 19 Acked 1018 Write 7 Done Sent Done false
147012024-02-28T22:18:44.318Z 20 Acked 1019 Read 3 Done Sent Done false
147022024-02-28T22:18:44.318Z 21 Acked 1020 Write 10 Done Sent Done false
147032024-02-28T22:18:44.318Z 22 Acked 1021 Read 5 Done Sent Done false
147042024-02-28T22:18:44.318Z 23 Acked 1022 Read 10 Done Sent Done false
147052024-02-28T22:18:44.318Z 24 Acked 1023 Write 10 Done Sent Done false
147062024-02-28T22:18:44.318Z 25 Acked 1024 Write 9 Done Sent Done false
147072024-02-28T22:18:44.318Z 26 Acked 1025 Write 4 Done Sent Done false
147082024-02-28T22:18:44.319Z 27 Acked 1026 Read 8 Done Sent Done false
147092024-02-28T22:18:44.319Z 28 Acked 1027 Write 2 Done Sent Done false
147102024-02-28T22:18:44.319Z 29 Acked 1028 Read 2 Done Sent Done false
147112024-02-28T22:18:44.319Z 30 Acked 1029 Read 2 Sent Sent Done false
147122024-02-28T22:18:44.319Z STATES DS:0 DS:1 DS:2 TOTAL
147132024-02-28T22:18:44.319Z New 0 0 0 0
147142024-02-28T22:18:44.319Z Sent 1 25 0 26
147152024-02-28T22:18:44.319Z Done 27 3 28 58
147162024-02-28T22:18:44.319Z Skipped 0 0 0 0
147172024-02-28T22:18:44.320Z Error 0 0 0 0
147182024-02-28T22:18:44.320ZLast Flush: 1015 1001 1015
147192024-02-28T22:18:44.320ZDownstairs last five completed: 1001 1000
147202024-02-28T22:18:44.320ZUpstairs last five completed: 30 29 28 27 26
147212024-02-28T22:18:44.320Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:43.196853534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1679}
147222024-02-28T22:18:44.320ZWrote out file "/tmp/test_repair_verify.data"
147232024-02-28T22:18:44.324Z
147242024-02-28T22:18:44.324ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147252024-02-28T22:18:44.338ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147262024-02-28T22:18:44.338Z 0 000-019 123 123 123 227 227 227 F F F
147272024-02-28T22:18:44.339Z 1 020-039 133 133 133 245 245 245 F F F
147282024-02-28T22:18:44.339Z 2 040-059 134 134 134 246 246 246 F F F
147292024-02-28T22:18:44.339Z 3 060-079 130 130 130 239 239 239 F F F
147302024-02-28T22:18:44.339Z 4 080-099 136 133 136 252 245 252 F F F <---
147312024-02-28T22:18:44.339Z 5 100-119 133 133 133 245 245 245 F F F
147322024-02-28T22:18:44.339Z 6 120-139 134 134 134 246 246 246 F F F
147332024-02-28T22:18:44.339Z 7 140-159 133 133 133 245 245 245 F F F
147342024-02-28T22:18:44.340Z 8 160-179 135 135 135 249 249 249 F F F
147352024-02-28T22:18:44.340Z 9 180-199 135 135 135 249 249 249 T F T <---
147362024-02-28T22:18:44.340Z 10 200-219 136 132 136 252 244 252 T F T <---
147372024-02-28T22:18:44.340Z 11 220-239 134 134 134 248 248 248 F F F
147382024-02-28T22:18:44.340Z 12 240-259 136 128 136 252 236 252 T F T <---
147392024-02-28T22:18:44.345Z 13 260-279 135 135 135 249 249 249 T F T <---
147402024-02-28T22:18:44.345Z 14 280-299 128 128 128 236 236 236 F F F
147412024-02-28T22:18:44.346Z 15 300-319 136 134 136 251 248 251 T T T <---
147422024-02-28T22:18:44.346Z 16 320-339 136 122 136 251 225 251 F T F <---
147432024-02-28T22:18:44.346Z 17 340-359 136 135 136 251 249 251 F T F <---
147442024-02-28T22:18:44.346Z 18 360-379 136 133 136 252 245 252 F T F <---
147452024-02-28T22:18:44.346Z 19 380-399 136 134 136 252 248 252 F F F <---
147462024-02-28T22:18:44.346Z 20 400-419 134 134 134 246 246 246 T F T <---
147472024-02-28T22:18:44.347Z 21 420-439 136 136 136 250 250 250 F F F
147482024-02-28T22:18:44.347Z 22 440-459 130 130 130 240 240 240 T F T <---
147492024-02-28T22:18:44.347Z 23 460-479 128 128 128 236 236 236 T F T <---
147502024-02-28T22:18:44.347Z 24 480-499 133 133 133 245 245 245 F F F
147512024-02-28T22:18:44.347Z 25 500-519 133 133 133 245 245 245 F F F
147522024-02-28T22:18:44.348Z 26 520-539 133 133 133 245 245 245 F F F
147532024-02-28T22:18:44.348Z 27 540-559 119 119 119 217 217 217 T F T <---
147542024-02-28T22:18:44.348Z 28 560-579 133 133 133 245 245 245 F F F
147552024-02-28T22:18:44.348Z 29 580-599 133 133 133 245 245 245 F F F
147562024-02-28T22:18:44.348ZMax gen: 136, Max flush: 252
147572024-02-28T22:18:44.348ZError: Difference in extent metadata found!
147582024-02-28T22:18:44.348ZA Difference in extent metadata is expected here
147592024-02-28T22:18:44.348ZOn loop 68
147602024-02-28T22:18:44.349Z
147612024-02-28T22:18:44.349ZVerifying data now
147622024-02-28T22:18:50.051ZNo change after verify
147632024-02-28T22:18:50.051ZLoop: 68 Downstairs dump after verify (and repair):
147642024-02-28T22:18:50.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147652024-02-28T22:18:50.091Z 0 000-019 123 123 123 227 227 227 F F F
147662024-02-28T22:18:50.091Z 1 020-039 133 133 133 245 245 245 F F F
147672024-02-28T22:18:50.091Z 2 040-059 134 134 134 246 246 246 F F F
147682024-02-28T22:18:50.091Z 3 060-079 130 130 130 239 239 239 F F F
147692024-02-28T22:18:50.091Z 4 080-099 136 136 136 252 252 252 F F F
147702024-02-28T22:18:50.092Z 5 100-119 133 133 133 245 245 245 F F F
147712024-02-28T22:18:50.092Z 6 120-139 134 134 134 246 246 246 F F F
147722024-02-28T22:18:50.092Z 7 140-159 133 133 133 245 245 245 F F F
147732024-02-28T22:18:50.092Z 8 160-179 135 135 135 249 249 249 F F F
147742024-02-28T22:18:50.092Z 9 180-199 137 137 137 253 253 253 F F F
147752024-02-28T22:18:50.092Z 10 200-219 137 137 137 253 253 253 F F F
147762024-02-28T22:18:50.092Z 11 220-239 134 134 134 248 248 248 F F F
147772024-02-28T22:18:50.092Z 12 240-259 137 137 137 253 253 253 F F F
147782024-02-28T22:18:50.092Z 13 260-279 137 137 137 253 253 253 F F F
147792024-02-28T22:18:50.092Z 14 280-299 128 128 128 236 236 236 F F F
147802024-02-28T22:18:50.093Z 15 300-319 137 137 137 253 253 253 F F F
147812024-02-28T22:18:50.093Z 16 320-339 136 136 136 251 251 251 F F F
147822024-02-28T22:18:50.093Z 17 340-359 136 136 136 251 251 251 F F F
147832024-02-28T22:18:50.093Z 18 360-379 136 136 136 252 252 252 F F F
147842024-02-28T22:18:50.093Z 19 380-399 136 136 136 252 252 252 F F F
147852024-02-28T22:18:50.093Z 20 400-419 137 137 137 253 253 253 F F F
147862024-02-28T22:18:50.093Z 21 420-439 136 136 136 250 250 250 F F F
147872024-02-28T22:18:50.093Z 22 440-459 137 137 137 253 253 253 F F F
147882024-02-28T22:18:50.093Z 23 460-479 137 137 137 253 253 253 F F F
147892024-02-28T22:18:50.093Z 24 480-499 133 133 133 245 245 245 F F F
147902024-02-28T22:18:50.093Z 25 500-519 133 133 133 245 245 245 F F F
147912024-02-28T22:18:50.094Z 26 520-539 133 133 133 245 245 245 F F F
147922024-02-28T22:18:50.094Z 27 540-559 137 137 137 253 253 253 F F F
147932024-02-28T22:18:50.094Z 28 560-579 133 133 133 245 245 245 F F F
147942024-02-28T22:18:50.094Z 29 580-599 133 133 133 245 245 245 F F F
147952024-02-28T22:18:50.094ZMax gen: 137, Max flush: 253
147962024-02-28T22:18:50.094Z
147972024-02-28T22:18:50.094ZBegin loop 69
147982024-02-28T22:18:50.094ZDownstairs to restart: 2
147992024-02-28T22:18:50.123Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.96891353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
148002024-02-28T22:18:50.123Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"Crucible runtime is spawned
148012024-02-28T22:18:50.123Zname":"crucible","level":30,"time":"2024-02-28T22:18:48.969284079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
148022024-02-28T22:18:50.123Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969383857Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
148032024-02-28T22:18:50.123Z{"msg":"Crucible 00631008-9d1e-4ff3-a0d6-9796c79021ee has session id: d8930ba3-675b-49b7-a59a-6608036f57a6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969417806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148042024-02-28T22:18:50.124Z{"msg":"Upstairs opts: Upstairs UUID: 00631008-9d1e-4ff3-a0d6-9796c79021ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969471204Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148052024-02-28T22:18:50.124Z{"msg":"Crucible stats registered with UUID: 00631008-9d1e-4ff3-a0d6-9796c79021ee","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969520223Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148062024-02-28T22:18:50.124Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969565492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
148072024-02-28T22:18:50.124Z{"msg":"00631008-9d1e-4ff3-a0d6-9796c79021ee active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969593731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148082024-02-28T22:18:50.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.96963673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148092024-02-28T22:18:50.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969705568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148102024-02-28T22:18:50.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969736227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148112024-02-28T22:18:50.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969808075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148122024-02-28T22:18:50.124Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969895243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148132024-02-28T22:18:50.124Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.969950051Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148142024-02-28T22:18:50.124Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.96998745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148152024-02-28T22:18:50.124Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970015909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148162024-02-28T22:18:50.125Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970033099Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148172024-02-28T22:18:50.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970067978Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148182024-02-28T22:18:50.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970085668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148192024-02-28T22:18:50.125Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970110627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148202024-02-28T22:18:50.125Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970611363Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148212024-02-28T22:18:50.125Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970648452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148222024-02-28T22:18:50.125Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970684271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148232024-02-28T22:18:50.125Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.97072004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148242024-02-28T22:18:50.125Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970758719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148252024-02-28T22:18:50.125Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.970787198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148262024-02-28T22:18:50.125Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971273655Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148272024-02-28T22:18:50.125Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971326173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148282024-02-28T22:18:50.125Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971360552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148292024-02-28T22:18:50.126Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971874708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148302024-02-28T22:18:50.126Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971925367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148312024-02-28T22:18:50.126Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971947936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148322024-02-28T22:18:50.126Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971971045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148332024-02-28T22:18:50.126Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.971985625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148342024-02-28T22:18:50.126Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972006335Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148352024-02-28T22:18:50.126Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972019314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148362024-02-28T22:18:50.126Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972039324Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148372024-02-28T22:18:50.126Z{"msg":"[0]R flush_numbers[0..12]: [227, 245, 246, 239, 252, 245, 246, 245, 249, 253, 253, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972053593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148382024-02-28T22:18:50.126Z{"msg":"[0]R generation[0..12]: [123, 133, 134, 130, 136, 133, 134, 133, 135, 137, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972074283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148392024-02-28T22:18:50.126Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972090152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148402024-02-28T22:18:50.126Z{"msg":"[1]R flush_numbers[0..12]: [227, 245, 246, 239, 252, 245, 246, 245, 249, 253, 253, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972116621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148412024-02-28T22:18:50.126Z{"msg":"[1]R generation[0..12]: [123, 133, 134, 130, 136, 133, 134, 133, 135, 137, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972132291Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148422024-02-28T22:18:50.127Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.97216907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148432024-02-28T22:18:50.127Z{"msg":"Wait for a query_work_queue command to finish before sending IO
148442024-02-28T22:18:50.127Z[2]R flush_numbers[0..12]: [227, 245, 246, 239, 252, 245, 246, 245, 249, 253, 253, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972199629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148452024-02-28T22:18:50.127Z{"msg":"[2]R generation[0..12]: [123, 133, 134, 130, 136, 133, 134, 133, 135, 137, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972231798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148462024-02-28T22:18:50.127Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972258757Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148472024-02-28T22:18:50.127Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972288587Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148482024-02-28T22:18:50.127Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972313526Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148492024-02-28T22:18:50.127Z{"msg":"Next flush: 254","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972340645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148502024-02-28T22:18:50.127Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972363835Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"":"downstairs","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148512024-02-28T22:18:50.127Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972391284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148522024-02-28T22:18:50.127Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972412713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"0","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148532024-02-28T22:18:50.127Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972441012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"1","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148542024-02-28T22:18:50.128Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972465852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"client":"2","":"downstairs","session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148552024-02-28T22:18:50.128Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972496931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148562024-02-28T22:18:50.128Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148572024-02-28T22:18:50.128Z,"time":"2024-02-28T22:18:48.97251936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148582024-02-28T22:18:50.128Z{"msg":"00631008-9d1e-4ff3-a0d6-9796c79021ee is now active with session: d8930ba3-675b-49b7-a59a-6608036f57a6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.97255372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148592024-02-28T22:18:50.128Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972583919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687,"session_id":"d8930ba3-675b-49b7-a59a-6608036f57a6"}
148602024-02-28T22:18:50.128Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:48.972606328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
148612024-02-28T22:18:50.128ZLoading write count information from file "/tmp/test_repair_verify.data"
148622024-02-28T22:18:50.128ZRun Repair workload
148632024-02-28T22:18:50.128Z01/30 Write block 348 len 40960 data: 10 7 8 9 10 11 11 11 10 10
148642024-02-28T22:18:50.128Z02/30 Read block 424 len 36864
148652024-02-28T22:18:50.128Z03/30 Read block 43 len 40960
148662024-02-28T22:18:50.128Z04/30 Read block 477 len 8192
148672024-02-28T22:18:50.129Z05/30 Read block 32 len 36864
148682024-02-28T22:18:50.129Z06/30 Write block 517 len 24576 data: 16 15 15 15 14 13
148692024-02-28T22:18:50.129Z07/30 Read block 261 len 8192
148702024-02-28T22:18:50.129Z08/30 Write block 439 len 40960 data: 13 11 10 11 8 11 12 13 13 12
148712024-02-28T22:18:50.129Z09/30 Read block 150 len 24576
148722024-02-28T22:18:50.135Z10/30 Write block 189 len 40960 data: 10 9 9 12 11 10 10 10 11 11
148732024-02-28T22:18:50.135Z11/30 Write block 416 len 20480 data: 13 9 12 12 14
148742024-02-28T22:18:50.135Z12/30 Read block 425 len 36864
148752024-02-28T22:18:50.135Z13/30 Write block 523 len 28672 data: 10 9 11 10 8 6 8
148762024-02-28T22:18:50.135Z14/30 Write block 289 len 4096 data: 8
148772024-02-28T22:18:50.135Z15/30 Read block 236 len 36864
148782024-02-28T22:18:50.135Z16/30 Read block 160 len 12288
148792024-02-28T22:18:50.135Z17/30 Write block 439 len 12288 data: 14 12 11
148802024-02-28T22:18:50.135Z18/30 Write block 271 len 4096 data: 7
148812024-02-28T22:18:50.135Z19/30 Write block 265 len 32768 data: 12 14 11 10 8 8 8 6
148822024-02-28T22:18:50.135Z20/30 Write block 582 len 4096 data: 5
148832024-02-28T22:18:50.135Z21/30 Read block 79 len 8192
148842024-02-28T22:18:50.136Z22/30 Read block 90 len 20480
148852024-02-28T22:18:50.136Z23/30 Read block 125 len 16384
148862024-02-28T22:18:50.136Z24/30 Write block 3 len 24576 data: 10 12 11 13 16 16
148872024-02-28T22:18:50.136Z25/30 Read block 428 len 28672
148882024-02-28T22:18:50.136Z26/30 Write block 262 len 24576 data: 16 14 13 13 15 12
148892024-02-28T22:18:50.136Z27/30 Write block 96 len 8192 data: 9 10
148902024-02-28T22:18:50.136Z28/30 Read block 373 len 20480
148912024-02-28T22:18:50.136Z29/30 Read block 562 len 28672
148922024-02-28T22:18:50.136Z30/30 Write block 64 len 4096 data: 13
148932024-02-28T22:18:50.136Z----------------------------------------------------------------
148942024-02-28T22:18:50.136Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
148952024-02-28T22:18:50.136ZStates: Active Active Active
148962024-02-28T22:18:50.136ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148972024-02-28T22:18:50.136Z 1 Acked 1000 Write 10 Done Done Done false
148982024-02-28T22:18:50.137Z 2 Acked 1001 Read 9 Done Done Done false
148992024-02-28T22:18:50.137Z 3 Acked 1002 Read 10 Done Done Done false
149002024-02-28T22:18:50.137Z 4 Acked 1003 Read 2 Done Done Done false
149012024-02-28T22:18:50.137Z 5 Acked 1004 Read 9 Done Done Done false
149022024-02-28T22:18:50.137Z 6 Acked 1005 Write 6 Done Done Done false
149032024-02-28T22:18:50.137Z 7 Acked 1006 Read 2 Done Done Sent false
149042024-02-28T22:18:50.137Z 8 Acked 1007 Write 10 Done Done Sent false
149052024-02-28T22:18:50.137Z 9 Acked 1008 Read 6 Done Done Sent false
149062024-02-28T22:18:50.137Z 10 Acked 1009 Write 10 Done Done Sent false
149072024-02-28T22:18:50.137Z 11 Acked 1010 Write 5 Done Done Sent false
149082024-02-28T22:18:50.137Z 12 Acked 1011 Read 9 Done Done Sent false
149092024-02-28T22:18:50.137Z 13 Acked 1012 Write 7 Done Done Sent false
149102024-02-28T22:18:50.137Z 14 Acked 1013 Write 1 Done Done Sent false
149112024-02-28T22:18:50.137Z 15 Acked 1014 Read 9 Done Done Sent false
149122024-02-28T22:18:50.137Z 16 Acked 1015 Read 3 Done Done Sent false
149132024-02-28T22:18:50.137Z 17 Acked 1016 Write 3 Done Done Sent false
149142024-02-28T22:18:50.137Z 18 Acked 1017 Write 1 Done Done Sent false
149152024-02-28T22:18:50.138Z 19 Acked 1018 Write 8 Done Done Sent false
149162024-02-28T22:18:50.138Z 20 Acked 1019 Write 1 Done Done Sent false
149172024-02-28T22:18:50.138Z 21 Acked 1020 Read 2 Done Done Sent false
149182024-02-28T22:18:50.138Z 22 Acked 1021 Read 5 Done Done Sent false
149192024-02-28T22:18:50.138Z 23 Acked 1022 Read 4 Done Done Sent false
149202024-02-28T22:18:50.138Z 24 Acked 1023 Write 6 Done Done Sent false
149212024-02-28T22:18:50.138Z 25 Acked 1024 Read 7 Done Done Sent false
149222024-02-28T22:18:50.138Z 26 Acked 1025 Write 6 Done Done Sent false
149232024-02-28T22:18:50.138Z 27 Acked 1026 Write 2 Done Done Sent false
149242024-02-28T22:18:50.138Z 28 Acked 1027 Read 5 Done Done Sent false
149252024-02-28T22:18:50.138Z 29 Acked 1028 Read 7 Done Done Sent false
149262024-02-28T22:18:50.138Z 30 Acked 1029 Write 1 Sent Sent Sent false
149272024-02-28T22:18:50.138Z STATES DS:0 DS:1 DS:2 TOTAL
149282024-02-28T22:18:50.138Z New 0 0 0 0
149292024-02-28T22:18:50.138Z Sent 1 1 24 26
149302024-02-28T22:18:50.138Z Done 29 29 6 64
149312024-02-28T22:18:50.138Z Skipped 0 0 0 0
149322024-02-28T22:18:50.138Z Error 0 0 0 0
149332024-02-28T22:18:50.138ZLast Flush: 0 0 0
149342024-02-28T22:18:50.139ZDownstairs last five completed:
149352024-02-28T22:18:50.139ZUpstairs last five completed: 30 29 28 27 26
149362024-02-28T22:18:50.139Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:48.999644853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1687}
149372024-02-28T22:18:50.139ZWrote out file "/tmp/test_repair_verify.data"
149382024-02-28T22:18:50.139Z
149392024-02-28T22:18:50.139ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149402024-02-28T22:18:50.144ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149412024-02-28T22:18:50.144Z 0 000-019 123 123 123 227 227 227 T T F <---
149422024-02-28T22:18:50.144Z 1 020-039 133 133 133 245 245 245 F F F
149432024-02-28T22:18:50.144Z 2 040-059 134 134 134 246 246 246 F F F
149442024-02-28T22:18:50.144Z 3 060-079 130 130 130 239 239 239 T T F <---
149452024-02-28T22:18:50.144Z 4 080-099 136 136 136 252 252 252 T T F <---
149462024-02-28T22:18:50.144Z 5 100-119 133 133 133 245 245 245 F F F
149472024-02-28T22:18:50.144Z 6 120-139 134 134 134 246 246 246 F F F
149482024-02-28T22:18:50.145Z 7 140-159 133 133 133 245 245 245 F F F
149492024-02-28T22:18:50.145Z 8 160-179 135 135 135 249 249 249 F F F
149502024-02-28T22:18:50.145Z 9 180-199 137 137 137 253 253 253 T T F <---
149512024-02-28T22:18:50.145Z 10 200-219 137 137 137 253 253 253 F F F
149522024-02-28T22:18:50.145Z 11 220-239 134 134 134 248 248 248 F F F
149532024-02-28T22:18:50.145Z 12 240-259 137 137 137 253 253 253 F F F
149542024-02-28T22:18:50.145Z 13 260-279 137 137 137 253 253 253 T T F <---
149552024-02-28T22:18:50.145Z 14 280-299 128 128 128 236 236 236 T T F <---
149562024-02-28T22:18:50.145Z 15 300-319 137 137 137 253 253 253 F F F
149572024-02-28T22:18:50.145Z 16 320-339 136 136 136 251 251 251 F F F
149582024-02-28T22:18:50.145Z 17 340-359 136 136 136 251 251 251 T T T
149592024-02-28T22:18:50.145Z 18 360-379 136 136 136 252 252 252 F F F
149602024-02-28T22:18:50.145Z 19 380-399 136 136 136 252 252 252 F F F
149612024-02-28T22:18:50.145Z 20 400-419 137 137 137 253 253 253 T T F <---
149622024-02-28T22:18:50.145Z 21 420-439 136 136 136 250 250 250 T T F <---
149632024-02-28T22:18:50.145Z 22 440-459 137 137 137 253 253 253 T T F <---
149642024-02-28T22:18:50.146Z 23 460-479 137 137 137 253 253 253 F F F
149652024-02-28T22:18:50.146Z 24 480-499 133 133 133 245 245 245 F F F
149662024-02-28T22:18:50.146Z 25 500-519 133 133 133 245 245 245 T T T
149672024-02-28T22:18:50.146Z 26 520-539 133 133 133 245 245 245 T T T
149682024-02-28T22:18:50.146Z 27 540-559 137 137 137 253 253 253 F F F
149692024-02-28T22:18:50.146Z 28 560-579 133 133 133 245 245 245 F F F
149702024-02-28T22:18:50.146Z 29 580-599 133 133 133 245 245 245 T T F <---
149712024-02-28T22:18:50.146ZMax gen: 137, Max flush: 253
149722024-02-28T22:18:50.146ZError: Difference in extent metadata found!
149732024-02-28T22:18:50.150ZA Difference in extent metadata is expected here
149742024-02-28T22:18:50.150ZOn loop 69
149752024-02-28T22:18:50.150Z
149762024-02-28T22:18:50.150ZVerifying data now
149772024-02-28T22:18:55.830ZNo change after verify
149782024-02-28T22:18:55.830ZLoop: 69 Downstairs dump after verify (and repair):
149792024-02-28T22:18:55.844ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149802024-02-28T22:18:55.844Z 0 000-019 138 138 138 254 254 254 F F F
149812024-02-28T22:18:55.844Z 1 020-039 133 133 133 245 245 245 F F F
149822024-02-28T22:18:55.844Z 2 040-059 134 134 134 246 246 246 F F F
149832024-02-28T22:18:55.844Z 3 060-079 138 138 138 254 254 254 F F F
149842024-02-28T22:18:55.844Z 4 080-099 138 138 138 254 254 254 F F F
149852024-02-28T22:18:55.845Z 5 100-119 133 133 133 245 245 245 F F F
149862024-02-28T22:18:55.845Z 6 120-139 134 134 134 246 246 246 F F F
149872024-02-28T22:18:55.845Z 7 140-159 133 133 133 245 245 245 F F F
149882024-02-28T22:18:55.845Z 8 160-179 135 135 135 249 249 249 F F F
149892024-02-28T22:18:55.848Z 9 180-199 138 138 138 254 254 254 F F F
149902024-02-28T22:18:55.848Z 10 200-219 137 137 137 253 253 253 F F F
149912024-02-28T22:18:55.848Z 11 220-239 134 134 134 248 248 248 F F F
149922024-02-28T22:18:55.848Z 12 240-259 137 137 137 253 253 253 F F F
149932024-02-28T22:18:55.849Z 13 260-279 138 138 138 254 254 254 F F F
149942024-02-28T22:18:55.849Z 14 280-299 138 138 138 254 254 254 F F F
149952024-02-28T22:18:55.849Z 15 300-319 137 137 137 253 253 253 F F F
149962024-02-28T22:18:55.849Z 16 320-339 136 136 136 251 251 251 F F F
149972024-02-28T22:18:55.849Z 17 340-359 138 138 138 254 254 254 F F F
149982024-02-28T22:18:55.849Z 18 360-379 136 136 136 252 252 252 F F F
149992024-02-28T22:18:55.849Z 19 380-399 136 136 136 252 252 252 F F F
150002024-02-28T22:18:55.849Z 20 400-419 138 138 138 254 254 254 F F F
150012024-02-28T22:18:55.849Z 21 420-439 138 138 138 254 254 254 F F F
150022024-02-28T22:18:55.849Z 22 440-459 138 138 138 254 254 254 F F F
150032024-02-28T22:18:55.849Z 23 460-479 137 137 137 253 253 253 F F F
150042024-02-28T22:18:55.849Z 24 480-499 133 133 133 245 245 245 F F F
150052024-02-28T22:18:55.849Z 25 500-519 138 138 138 254 254 254 F F F
150062024-02-28T22:18:55.849Z 26 520-539 138 138 138 254 254 254 F F F
150072024-02-28T22:18:55.849Z 27 540-559 137 137 137 253 253 253 F F F
150082024-02-28T22:18:55.849Z 28 560-579 133 133 133 245 245 245 F F F
150092024-02-28T22:18:55.849Z 29 580-599 138 138 138 254 254 254 F F F
150102024-02-28T22:18:55.849ZMax gen: 138, Max flush: 254
150112024-02-28T22:18:55.849Z
150122024-02-28T22:18:55.849ZBegin loop 70
150132024-02-28T22:18:55.849ZDownstairs to restart: 0
150142024-02-28T22:18:55.866Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.750852023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
150152024-02-28T22:18:55.866Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"Crucible runtime is spawned
150162024-02-28T22:18:55.866Z1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751158814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
150172024-02-28T22:18:55.873Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751216533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
150182024-02-28T22:18:55.873Z{"msg":"Crucible 10439243-e70f-4e9c-a72b-57f33b46d89d has session id: ffee0e7d-9b3e-4752-ba4c-cacb02efe6db","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751259682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150192024-02-28T22:18:55.873Z{"msg":"Upstairs opts: Upstairs UUID: 10439243-e70f-4e9c-a72b-57f33b46d89d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.75131588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150202024-02-28T22:18:55.873Z{"msg":"Crucible stats registered with UUID: 10439243-e70f-4e9c-a72b-57f33b46d89d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751361259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150212024-02-28T22:18:55.873Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751388498Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
150222024-02-28T22:18:55.873Z{"msg":"10439243-e70f-4e9c-a72b-57f33b46d89d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751411027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150232024-02-28T22:18:55.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751436857Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150242024-02-28T22:18:55.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751466646Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150252024-02-28T22:18:55.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751505645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150262024-02-28T22:18:55.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751537644Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150272024-02-28T22:18:55.873Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751577783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150282024-02-28T22:18:55.873Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751617232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150292024-02-28T22:18:55.873Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751665431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150302024-02-28T22:18:55.873Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.75170775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150312024-02-28T22:18:55.873Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751740449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150322024-02-28T22:18:55.873Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751770868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150332024-02-28T22:18:55.873Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751846615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150342024-02-28T22:18:55.874Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.751882275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150352024-02-28T22:18:55.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752439669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150362024-02-28T22:18:55.874Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752497268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150372024-02-28T22:18:55.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752581365Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150382024-02-28T22:18:55.874Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752632454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150392024-02-28T22:18:55.874Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752662953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150402024-02-28T22:18:55.874Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.752697622Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150412024-02-28T22:18:55.874Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753000634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150422024-02-28T22:18:55.874Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753049642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150432024-02-28T22:18:55.874Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753446751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150442024-02-28T22:18:55.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753637596Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150452024-02-28T22:18:55.874Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753674155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150462024-02-28T22:18:55.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753690545Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150472024-02-28T22:18:55.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753718784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150482024-02-28T22:18:55.874Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753743503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150492024-02-28T22:18:55.874Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753770522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150502024-02-28T22:18:55.874Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.753944668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150512024-02-28T22:18:55.874Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754020416Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150522024-02-28T22:18:55.874Z{"msg":"[0]R flush_numbers[0..12]: [254, 245, 246, 254, 254, 245, 246, 245, 249, 254, 253, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754058905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150532024-02-28T22:18:55.874Z{"msg":"[0]R generation[0..12]: [138, 133, 134, 138, 138, 133, 134, 133, 135, 138, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754086134Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150542024-02-28T22:18:55.874Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754117773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150552024-02-28T22:18:55.874Z{"msg":"[1]R flush_numbers[0..12]: [254, 245, 246, 254, 254, 245, 246, 245, 249, 254, 253, 248]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
150562024-02-28T22:18:55.875Zcrucible","level":30,"time":"2024-02-28T22:18:54.754148122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150572024-02-28T22:18:55.875Z{"msg":"[1]R generation[0..12]: [138, 133, 134, 138, 138, 133, 134, 133, 135, 138, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754181721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150582024-02-28T22:18:55.875Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.75422389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150592024-02-28T22:18:55.875Z{"msg":"[2]R flush_numbers[0..12]: [254, 245, 246, 254, 254, 245, 246, 245, 249, 254, 253, 248]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754260179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150602024-02-28T22:18:55.875Z{"msg":"[2]R generation[0..12]: [138, 133, 134, 138, 138, 133, 134, 133, 135, 138, 137, 134]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754294878Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150612024-02-28T22:18:55.875Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754328697Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150622024-02-28T22:18:55.875Z{"msg":"Max found gen is 139","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754349877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150632024-02-28T22:18:55.875Z{"msg":"Generation requested: 140 >= found:139","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754374136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150642024-02-28T22:18:55.875Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754388165Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150652024-02-28T22:18:55.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754402175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"":"downstairs","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150662024-02-28T22:18:55.875Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754415725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150672024-02-28T22:18:55.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754428075Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"0","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150682024-02-28T22:18:55.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754456864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"1","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150692024-02-28T22:18:55.875Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754472343Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"client":"2","":"downstairs","session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150702024-02-28T22:18:55.875Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754493793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150712024-02-28T22:18:55.875Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754506662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150722024-02-28T22:18:55.875Z{"msg":"10439243-e70f-4e9c-a72b-57f33b46d89d is now active with session: ffee0e7d-9b3e-4752-ba4c-cacb02efe6db","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754519562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150732024-02-28T22:18:55.875Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754538752Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695,"session_id":"ffee0e7d-9b3e-4752-ba4c-cacb02efe6db"}
150742024-02-28T22:18:55.875Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:18:54.754551421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
150752024-02-28T22:18:55.875ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150762024-02-28T22:18:55.879ZLoading write count information from file "/tmp/test_repair_verify.data"
150772024-02-28T22:18:55.879ZRun Repair workload
150782024-02-28T22:18:55.879Z01/30 Write block 128 len 28672 data: 17 15 14 16 15 16 15
150792024-02-28T22:18:55.879Z02/30 Write block 357 len 40960 data: 11 9 7 7 7 10 11 7 7 7
150802024-02-28T22:18:55.879Z03/30 Write block 249 len 8192 data: 11 11
150812024-02-28T22:18:55.879Z04/30 Read block 184 len 24576
150822024-02-28T22:18:55.880Z05/30 Write block 573 len 32768 data: 9 8 9 9 10 11 12 9
150832024-02-28T22:18:55.880Z06/30 Read block 19 len 4096
150842024-02-28T22:18:55.880Z07/30 Read block 286 len 32768
150852024-02-28T22:18:55.880Z08/30 Read block 277 len 32768
150862024-02-28T22:18:55.880Z09/30 Write block 452 len 24576 data: 12 14 12 12 11 10
150872024-02-28T22:18:55.883Z10/30 Write block 27 len 20480 data: 6 6 9 9 10
150882024-02-28T22:18:55.883Z11/30 Write block 17 len 28672 data: 11 10 10 7 6 5 4
150892024-02-28T22:18:55.883Z12/30 Read block 189 len 24576
150902024-02-28T22:18:55.883Z13/30 Flush
150912024-02-28T22:18:55.886Z14/30 Write block 451 len 24576 data: 14 13 15 13 13 12
150922024-02-28T22:18:55.889Z15/30 Read block 154 len 32768
150932024-02-28T22:18:55.889Z16/30 Write block 512 len 24576 data: 11 13 13 15 16 17
150942024-02-28T22:18:55.889Z17/30 Read block 346 len 20480
150952024-02-28T22:18:55.892Z18/30 Write block 100 len 24576 data: 10 9 9 9 9 8
150962024-02-28T22:18:55.892Z19/30 Read block 55 len 8192
150972024-02-28T22:18:55.892Z20/30 Write block 245 len 36864 data: 12 8 14 16 12 12 12 10 11
150982024-02-28T22:18:55.895Z21/30 Write block 324 len 16384 data: 11 11 13 13
150992024-02-28T22:18:55.895Z22/30 Write block 322 len 32768 data: 10 9 12 12 14 14 16 16
151002024-02-28T22:18:55.895Z23/30 Read block 212 len 24576
151012024-02-28T22:18:55.895Z24/30 Read block 139 len 32768
151022024-02-28T22:18:55.899Z25/30 Write block 541 len 36864 data: 10 10 10 10 11 11 10 7 8
151032024-02-28T22:18:55.899Z26/30 Write block 227 len 20480 data: 10 12 15 14 18
151042024-02-28T22:18:55.899Z27/30 Write block 435 len 40960 data: 13 15 14 15 15 13 12 12 9 12
151052024-02-28T22:18:55.899Z28/30 Write block 344 len 4096 data: 10
151062024-02-28T22:18:55.902Z29/30 Write block 234 len 20480 data: 15 10 9 10 10
151072024-02-28T22:18:55.903Z30/30 Read block 546 len 32768
151082024-02-28T22:18:55.903Z----------------------------------------------------------------
151092024-02-28T22:18:55.903Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
151102024-02-28T22:18:55.903ZStates: Active Active Active
151112024-02-28T22:18:55.903ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151122024-02-28T22:18:55.903Z 1 Acked 1000 Write 7 Done Done Done false
151132024-02-28T22:18:55.903Z 2 Acked 1001 Write 10 Done Done Done false
151142024-02-28T22:18:55.903Z 3 Acked 1002 Write 2 Sent Done Done false
151152024-02-28T22:18:55.903Z 4 Acked 1003 Read 6 Sent Done Done false
151162024-02-28T22:18:55.903Z 5 Acked 1004 Write 8 Sent Done Done false
151172024-02-28T22:18:55.903Z 6 Acked 1005 Read 1 Sent Done Done false
151182024-02-28T22:18:55.903Z 7 Acked 1006 Read 8 Sent Done Done false
151192024-02-28T22:18:55.903Z 8 Acked 1007 Read 8 Sent Done Done false
151202024-02-28T22:18:55.903Z 9 Acked 1008 Write 6 Sent Done Done false
151212024-02-28T22:18:55.903Z 10 Acked 1009 Write 5 Sent Done Done false
151222024-02-28T22:18:55.903Z 11 Acked 1010 Write 7 Sent Done Done false
151232024-02-28T22:18:55.903Z 12 Acked 1011 Read 6 Sent Done Done false
151242024-02-28T22:18:55.903Z 13 Acked 1012 Flush 0 Sent Done Done false
151252024-02-28T22:18:55.903Z 14 Acked 1013 Write 6 Sent Done Done false
151262024-02-28T22:18:55.903Z 15 Acked 1014 Read 8 Sent Done Done false
151272024-02-28T22:18:55.904Z 16 Acked 1015 Write 6 Sent Done Done false
151282024-02-28T22:18:55.904Z 17 Acked 1016 Read 5 Sent Done Done false
151292024-02-28T22:18:55.904Z 18 Acked 1017 Write 6 Sent Done Done false
151302024-02-28T22:18:55.904Z 19 Acked 1018 Read 2 Sent Done Done false
151312024-02-28T22:18:55.904Z 20 Acked 1019 Write 9 Sent Done Done false
151322024-02-28T22:18:55.904Z 21 Acked 1020 Write 4 Sent Done Done false
151332024-02-28T22:18:55.904Z 22 Acked 1021 Write 8 Sent Done Done false
151342024-02-28T22:18:55.904Z 23 Acked 1022 Read 6 Sent Done Done false
151352024-02-28T22:18:55.904Z 24 Acked 1023 Read 8 Sent Done Done false
151362024-02-28T22:18:55.904Z 25 Acked 1024 Write 9 Sent Done Done false
151372024-02-28T22:18:55.904Z 26 Acked 1025 Write 5 Sent Done Done false
151382024-02-28T22:18:55.904Z 27 Acked 1026 Write 10 Sent Done Done false
151392024-02-28T22:18:55.904Z 28 Acked 1027 Write 1 Sent Done Done false
151402024-02-28T22:18:55.904Z 29 Acked 1028 Write 5 Sent Done Done false
151412024-02-28T22:18:55.904Z 30 Acked 1029 Read 8 Sent Done Sent false
151422024-02-28T22:18:55.904Z STATES DS:0 DS:1 DS:2 TOTAL
151432024-02-28T22:18:55.904Z New 0 0 0 0
151442024-02-28T22:18:55.904Z Sent 28 0 1 29
151452024-02-28T22:18:55.904Z Done 2 30 29 61
151462024-02-28T22:18:55.904Z Skipped 0 0 0 0
151472024-02-28T22:18:55.904Z Error 0 0 0 0
151482024-02-28T22:18:55.904ZLast Flush: 0 1012 1012
151492024-02-28T22:18:55.904ZDownstairs last five completed:
151502024-02-28T22:18:55.905ZUpstairs last five completed: 30 29 28 27 26
151512024-02-28T22:18:55.905Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:18:54.785843419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1695}
151522024-02-28T22:18:55.905ZWrote out file "/tmp/test_repair_verify.data"
151532024-02-28T22:18:55.908Z
151542024-02-28T22:18:55.911ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151552024-02-28T22:18:55.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151562024-02-28T22:18:55.927Z 0 000-019 138 140 140 254 255 255 F F F <---
151572024-02-28T22:18:55.927Z 1 020-039 133 140 140 245 255 255 F F F <---
151582024-02-28T22:18:55.927Z 2 040-059 134 134 134 246 246 246 F F F
151592024-02-28T22:18:55.927Z 3 060-079 138 138 138 254 254 254 F F F
151602024-02-28T22:18:55.927Z 4 080-099 138 138 138 254 254 254 F F F
151612024-02-28T22:18:55.927Z 5 100-119 133 133 133 245 245 245 F T T <---
151622024-02-28T22:18:55.927Z 6 120-139 134 140 140 246 255 255 T F F <---
151632024-02-28T22:18:55.927Z 7 140-159 133 133 133 245 245 245 F F F
151642024-02-28T22:18:55.927Z 8 160-179 135 135 135 249 249 249 F F F
151652024-02-28T22:18:55.927Z 9 180-199 138 138 138 254 254 254 F F F
151662024-02-28T22:18:55.927Z 10 200-219 137 137 137 253 253 253 F F F
151672024-02-28T22:18:55.927Z 11 220-239 134 134 134 248 248 248 F T T <---
151682024-02-28T22:18:55.927Z 12 240-259 137 140 140 253 255 255 F T T <---
151692024-02-28T22:18:55.930Z 13 260-279 138 138 138 254 254 254 F F F
151702024-02-28T22:18:55.930Z 14 280-299 138 138 138 254 254 254 F F F
151712024-02-28T22:18:55.931Z 15 300-319 137 137 137 253 253 253 F F F
151722024-02-28T22:18:55.931Z 16 320-339 136 136 136 251 251 251 F T T <---
151732024-02-28T22:18:55.931Z 17 340-359 138 140 140 254 255 255 T T T <---
151742024-02-28T22:18:55.931Z 18 360-379 136 140 140 252 255 255 T F F <---
151752024-02-28T22:18:55.931Z 19 380-399 136 136 136 252 252 252 F F F
151762024-02-28T22:18:55.931Z 20 400-419 138 138 138 254 254 254 F F F
151772024-02-28T22:18:55.931Z 21 420-439 138 138 138 254 254 254 F T T <---
151782024-02-28T22:18:55.931Z 22 440-459 138 140 140 254 255 255 F T T <---
151792024-02-28T22:18:55.931Z 23 460-479 137 137 137 253 253 253 F F F
151802024-02-28T22:18:55.931Z 24 480-499 133 133 133 245 245 245 F F F
151812024-02-28T22:18:55.931Z 25 500-519 138 138 138 254 254 254 F T T <---
151822024-02-28T22:18:55.931Z 26 520-539 138 138 138 254 254 254 F F F
151832024-02-28T22:18:55.931Z 27 540-559 137 137 137 253 253 253 F T T <---
151842024-02-28T22:18:55.931Z 28 560-579 133 140 140 245 255 255 F F F <---
151852024-02-28T22:18:55.931Z 29 580-599 138 140 140 254 255 255 F F F <---
151862024-02-28T22:18:55.931ZMax gen: 140, Max flush: 255
151872024-02-28T22:18:55.931ZError: Difference in extent metadata found!
151882024-02-28T22:18:55.931ZA Difference in extent metadata is expected here
151892024-02-28T22:18:55.931ZOn loop 70
151902024-02-28T22:18:55.931Z
151912024-02-28T22:18:55.935ZVerifying data now
151922024-02-28T22:19:01.687ZNo change after verify
151932024-02-28T22:19:01.687ZLoop: 70 Downstairs dump after verify (and repair):
151942024-02-28T22:19:01.702ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151952024-02-28T22:19:01.702Z 0 000-019 140 140 140 255 255 255 F F F
151962024-02-28T22:19:01.702Z 1 020-039 140 140 140 255 255 255 F F F
151972024-02-28T22:19:01.703Z 2 040-059 134 134 134 246 246 246 F F F
151982024-02-28T22:19:01.703Z 3 060-079 138 138 138 254 254 254 F F F
151992024-02-28T22:19:01.703Z 4 080-099 138 138 138 254 254 254 F F F
152002024-02-28T22:19:01.703Z 5 100-119 141 141 141 256 256 256 F F F
152012024-02-28T22:19:01.704Z 6 120-139 140 140 140 255 255 255 F F F
152022024-02-28T22:19:01.704Z 7 140-159 133 133 133 245 245 245 F F F
152032024-02-28T22:19:01.704Z 8 160-179 135 135 135 249 249 249 F F F
152042024-02-28T22:19:01.704Z 9 180-199 138 138 138 254 254 254 F F F
152052024-02-28T22:19:01.704Z 10 200-219 137 137 137 253 253 253 F F F
152062024-02-28T22:19:01.709Z 11 220-239 141 141 141 256 256 256 F F F
152072024-02-28T22:19:01.709Z 12 240-259 141 141 141 256 256 256 F F F
152082024-02-28T22:19:01.709Z 13 260-279 138 138 138 254 254 254 F F F
152092024-02-28T22:19:01.709Z 14 280-299 138 138 138 254 254 254 F F F
152102024-02-28T22:19:01.710Z 15 300-319 137 137 137 253 253 253 F F F
152112024-02-28T22:19:01.710Z 16 320-339 141 141 141 256 256 256 F F F
152122024-02-28T22:19:01.710Z 17 340-359 141 141 141 256 256 256 F F F
152132024-02-28T22:19:01.710Z 18 360-379 140 140 140 255 255 255 F F F
152142024-02-28T22:19:01.710Z 19 380-399 136 136 136 252 252 252 F F F
152152024-02-28T22:19:01.710Z 20 400-419 138 138 138 254 254 254 F F F
152162024-02-28T22:19:01.710Z 21 420-439 141 141 141 256 256 256 F F F
152172024-02-28T22:19:01.711Z 22 440-459 141 141 141 256 256 256 F F F
152182024-02-28T22:19:01.711Z 23 460-479 137 137 137 253 253 253 F F F
152192024-02-28T22:19:01.711Z 24 480-499 133 133 133 245 245 245 F F F
152202024-02-28T22:19:01.711Z 25 500-519 141 141 141 256 256 256 F F F
152212024-02-28T22:19:01.711Z 26 520-539 138 138 138 254 254 254 F F F
152222024-02-28T22:19:01.712Z 27 540-559 141 141 141 256 256 256 F F F
152232024-02-28T22:19:01.712Z 28 560-579 140 140 140 255 255 255 F F F
152242024-02-28T22:19:01.712Z 29 580-599 140 140 140 255 255 255 F F F
152252024-02-28T22:19:01.712ZMax gen: 141, Max flush: 256
152262024-02-28T22:19:01.712Z
152272024-02-28T22:19:01.712ZBegin loop 71
152282024-02-28T22:19:01.712ZDownstairs to restart: 1
152292024-02-28T22:19:01.722Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605433475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
152302024-02-28T22:19:01.731Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"Crucible runtime is spawned
152312024-02-28T22:19:01.731Zcrucible","level":30,"time":"2024-02-28T22:19:00.605768256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
152322024-02-28T22:19:01.731Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605797845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
152332024-02-28T22:19:01.732Z{"msg":"Crucible 7fcc4b5d-1e3d-4c7a-907e-309acfc212b4 has session id: e3cbe84d-5e1b-4eea-a3d6-81114b898de0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605834184Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152342024-02-28T22:19:01.732Z{"msg":"Upstairs opts: Upstairs UUID: 7fcc4b5d-1e3d-4c7a-907e-309acfc212b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605878583Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152352024-02-28T22:19:01.732Z{"msg":"Crucible stats registered with UUID: 7fcc4b5d-1e3d-4c7a-907e-309acfc212b4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605901042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152362024-02-28T22:19:01.732Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605923082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
152372024-02-28T22:19:01.732Z{"msg":"7fcc4b5d-1e3d-4c7a-907e-309acfc212b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.605943041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152382024-02-28T22:19:01.732Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60598386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152392024-02-28T22:19:01.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606027449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152402024-02-28T22:19:01.733Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606051968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152412024-02-28T22:19:01.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606088477Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152422024-02-28T22:19:01.733Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606115146Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152432024-02-28T22:19:01.733Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606151435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152442024-02-28T22:19:01.733Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606176995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152452024-02-28T22:19:01.733Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606218733Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152462024-02-28T22:19:01.734Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606252252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152472024-02-28T22:19:01.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606276582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152482024-02-28T22:19:01.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606483326Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152492024-02-28T22:19:01.734Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606543035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152502024-02-28T22:19:01.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.606981192Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152512024-02-28T22:19:01.734Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607031381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152522024-02-28T22:19:01.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.6070642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152532024-02-28T22:19:01.734Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607094299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152542024-02-28T22:19:01.734Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607132698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152552024-02-28T22:19:01.734Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607164767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152562024-02-28T22:19:01.735Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607555657Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152572024-02-28T22:19:01.735Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.607729312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152582024-02-28T22:19:01.735Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60778722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152592024-02-28T22:19:01.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608073862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152602024-02-28T22:19:01.735Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608122941Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152612024-02-28T22:19:01.735Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60814966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152622024-02-28T22:19:01.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608346415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152632024-02-28T22:19:01.735Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608399273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152642024-02-28T22:19:01.735Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608423533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152652024-02-28T22:19:01.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608499231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152662024-02-28T22:19:01.735Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60852605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152672024-02-28T22:19:01.735Z{"msg":"[0]R flush_numbers[0..12]: [255, 255, 246, 254, 254, 256, 255, 245, 249, 254, 253, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60854124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152682024-02-28T22:19:01.736Z{"msg":"[0]R generation[0..12]: [140, 140, 134, 138, 138, 141, 140, 133, 135, 138, 137, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608562719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152692024-02-28T22:19:01.736Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608577528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152702024-02-28T22:19:01.736Z{"msg":"[1]R flush_numbers[0..12]: [255, 255, 246, 254, 254, 256, 255, 245, 249, 254, 253, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608599068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152712024-02-28T22:19:01.736Z{"msg":"[1]R generation[0..12]: [140, 140, 134, 138, 138, 141, 140, 133, 135, 138, 137, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608613398Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152722024-02-28T22:19:01.736Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608628267Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152732024-02-28T22:19:01.736Z{"msg":"[2]R flush_numbers[0..12]: [255, 255, 246, 254, 254, 256, 255, 245, 249, 254, 253, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608642387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152742024-02-28T22:19:01.736Z{"msg":"[2]R generation[0..12]: [140, 140, 134, 138, 138, 141, 140, 133, 135, 138, 137, 141]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608657286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152752024-02-28T22:19:01.736Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608672136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"Wait for a query_work_queue command to finish before sending IO
152762024-02-28T22:19:01.736Zdownstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152772024-02-28T22:19:01.736Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608689345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152782024-02-28T22:19:01.737Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608710715Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152792024-02-28T22:19:01.737Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608724235Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152802024-02-28T22:19:01.737Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608737624Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"":"downstairs","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152812024-02-28T22:19:01.737Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608760034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152822024-02-28T22:19:01.737Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608772613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"0","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152832024-02-28T22:19:01.737Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608785913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"1","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152842024-02-28T22:19:01.737Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608806152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"client":"2","":"downstairs","session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152852024-02-28T22:19:01.737Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608819792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152862024-02-28T22:19:01.737Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608832272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152872024-02-28T22:19:01.737Z{"msg":"7fcc4b5d-1e3d-4c7a-907e-309acfc212b4 is now active with session: e3cbe84d-5e1b-4eea-a3d6-81114b898de0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.608852301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152882024-02-28T22:19:01.737Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60886456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703,"session_id":"e3cbe84d-5e1b-4eea-a3d6-81114b898de0"}
152892024-02-28T22:19:01.738Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:00.60887711Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
152902024-02-28T22:19:01.738ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152912024-02-28T22:19:01.738ZLoading write count information from file "/tmp/test_repair_verify.data"
152922024-02-28T22:19:01.738ZRun Repair workload
152932024-02-28T22:19:01.738Z01/30 Write block 383 len 16384 data: 18 17 16 17
152942024-02-28T22:19:01.744Z02/30 Write block 123 len 24576 data: 10 10 11 11 15 18
152952024-02-28T22:19:01.745Z03/30 Read block 448 len 40960
152962024-02-28T22:19:01.745Z04/30 Read block 400 len 20480
152972024-02-28T22:19:01.745Z05/30 Write block 365 len 24576 data: 8 8 8 10 12 13
152982024-02-28T22:19:01.745Z06/30 Read block 175 len 40960
152992024-02-28T22:19:01.745Z07/30 Flush
153002024-02-28T22:19:01.745Z08/30 Write block 117 len 40960 data: 10 8 8 9 11 12 11 11 12 12
153012024-02-28T22:19:01.746Z09/30 Read block 124 len 20480
153022024-02-28T22:19:01.746Z10/30 Write block 228 len 8192 data: 13 16
153032024-02-28T22:19:01.746Z11/30 Flush
153042024-02-28T22:19:01.746Z12/30 Write block 24 len 28672 data: 4 3 4 7 7 10 10
153052024-02-28T22:19:01.746Z13/30 Write block 579 len 12288 data: 13 10 8
153062024-02-28T22:19:01.746Z14/30 Read block 433 len 8192
153072024-02-28T22:19:01.746Z15/30 Read block 99 len 40960
153082024-02-28T22:19:01.776Z16/30 Read block 186 len 36864
153092024-02-28T22:19:01.776Z17/30 Read block 518 len 12288
153102024-02-28T22:19:01.777Z18/30 Write block 293 len 16384 data: 7 5 8 9
153112024-02-28T22:19:01.777Z19/30 Read block 248 len 24576
153122024-02-28T22:19:01.777Z20/30 Read block 416 len 4096
153132024-02-28T22:19:01.777Z21/30 Write block 590 len 40960 data: 10 11 11 12 11 11 8 8 8 7
153142024-02-28T22:19:01.777Z22/30 Read block 542 len 8192
153152024-02-28T22:19:01.777Z23/30 Flush
153162024-02-28T22:19:01.778Z24/30 Write block 450 len 32768 data: 14 15 14 16 14 14 13 11
153172024-02-28T22:19:01.778Z25/30 Write block 122 len 28672 data: 13 12 12 13 13 16 19
153182024-02-28T22:19:01.778Z26/30 Write block 111 len 4096 data: 15
153192024-02-28T22:19:01.778Z27/30 Read block 145 len 16384
153202024-02-28T22:19:01.778Z28/30 Read block 530 len 24576
153212024-02-28T22:19:01.778Z29/30 Write block 282 len 32768 data: 9 10 12 11 13 10 9 9
153222024-02-28T22:19:01.779Z30/30 Write block 517 len 20480 data: 18 16 16 16 15
153232024-02-28T22:19:01.779Z----------------------------------------------------------------
153242024-02-28T22:19:01.779Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153252024-02-28T22:19:01.779ZStates: Active Active Active
153262024-02-28T22:19:01.779ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153272024-02-28T22:19:01.779Z 1 Acked 1000 Write 4 Done Done Done false
153282024-02-28T22:19:01.780Z 2 Acked 1001 Write 6 Done Done Done false
153292024-02-28T22:19:01.780Z 3 Acked 1002 Read 10 Done Done Done false
153302024-02-28T22:19:01.780Z 4 Acked 1003 Read 5 Done Done Done false
153312024-02-28T22:19:01.780Z 5 Acked 1004 Write 6 Done Sent Done false
153322024-02-28T22:19:01.780Z 6 Acked 1005 Read 10 Done Sent Done false
153332024-02-28T22:19:01.780Z 7 Acked 1006 Flush 0 Done Sent Done false
153342024-02-28T22:19:01.780Z 8 Acked 1007 Write 10 Done Sent Done false
153352024-02-28T22:19:01.780Z 9 Acked 1008 Read 5 Done Sent Done false
153362024-02-28T22:19:01.781Z 10 Acked 1009 Write 2 Done Sent Done false
153372024-02-28T22:19:01.781Z 11 Acked 1010 Flush 0 Done Sent Done false
153382024-02-28T22:19:01.781Z 12 Acked 1011 Write 7 Done Sent Done false
153392024-02-28T22:19:01.781Z 13 Acked 1012 Write 3 Done Sent Done false
153402024-02-28T22:19:01.781Z 14 Acked 1013 Read 2 Done Sent Done false
153412024-02-28T22:19:01.781Z 15 Acked 1014 Read 10 Done Sent Done false
153422024-02-28T22:19:01.781Z 16 Acked 1015 Read 9 Done Sent Done false
153432024-02-28T22:19:01.782Z 17 Acked 1016 Read 3 Done Sent Done false
153442024-02-28T22:19:01.782Z 18 Acked 1017 Write 4 Done Sent Done false
153452024-02-28T22:19:01.782Z 19 Acked 1018 Read 6 Done Sent Done false
153462024-02-28T22:19:01.782Z 20 Acked 1019 Read 1 Done Sent Done false
153472024-02-28T22:19:01.782Z 21 Acked 1020 Write 10 Done Sent Done false
153482024-02-28T22:19:01.782Z 22 Acked 1021 Read 2 Done Sent Done false
153492024-02-28T22:19:01.783Z 23 Acked 1022 Flush 0 Done Sent Done false
153502024-02-28T22:19:01.783Z 24 Acked 1023 Write 8 Done Sent Done false
153512024-02-28T22:19:01.783Z 25 Acked 1024 Write 7 Done Sent Done false
153522024-02-28T22:19:01.783Z 26 Acked 1025 Write 1 Done Sent Done false
153532024-02-28T22:19:01.783Z 27 Acked 1026 Read 4 Done Sent Done false
153542024-02-28T22:19:01.784Z 28 Acked 1027 Read 6 Done Sent Done false
153552024-02-28T22:19:01.784Z 29 Acked 1028 Write 8 Sent Sent Sent false
153562024-02-28T22:19:01.784Z 30 Acked 1029 Write 5 Sent Sent Sent false
153572024-02-28T22:19:01.784Z STATES DS:0 DS:1 DS:2 TOTAL
153582024-02-28T22:19:01.784Z New 0 0 0 0
153592024-02-28T22:19:01.784Z Sent 2 26 2 30
153602024-02-28T22:19:01.785Z Done 28 4 28 60
153612024-02-28T22:19:01.785Z Skipped 0 0 0 0
153622024-02-28T22:19:01.785Z Error 0 0 0 0
153632024-02-28T22:19:01.785ZLast Flush: 1022 0 1022
153642024-02-28T22:19:01.785ZDownstairs last five completed:
153652024-02-28T22:19:01.785ZUpstairs last five completed: 30 29 28 27 26
153662024-02-28T22:19:01.785Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:00.642180133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1703}
153672024-02-28T22:19:01.786ZWrote out file "/tmp/test_repair_verify.data"
153682024-02-28T22:19:01.786Z
153692024-02-28T22:19:01.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153702024-02-28T22:19:01.805ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153712024-02-28T22:19:01.805Z 0 000-019 140 140 140 255 255 255 F F F
153722024-02-28T22:19:01.805Z 1 020-039 142 140 142 259 255 259 F F F <---
153732024-02-28T22:19:01.805Z 2 040-059 134 134 134 246 246 246 F F F
153742024-02-28T22:19:01.806Z 3 060-079 138 138 138 254 254 254 F F F
153752024-02-28T22:19:01.806Z 4 080-099 138 138 138 254 254 254 F F F
153762024-02-28T22:19:01.806Z 5 100-119 142 141 142 258 256 258 T F T <---
153772024-02-28T22:19:01.806Z 6 120-139 142 140 142 258 255 258 T T T <---
153782024-02-28T22:19:01.806Z 7 140-159 133 133 133 245 245 245 F F F
153792024-02-28T22:19:01.806Z 8 160-179 135 135 135 249 249 249 F F F
153802024-02-28T22:19:01.806Z 9 180-199 138 138 138 254 254 254 F F F
153812024-02-28T22:19:01.806Z 10 200-219 137 137 137 253 253 253 F F F
153822024-02-28T22:19:01.806Z 11 220-239 142 141 142 258 256 258 F F F <---
153832024-02-28T22:19:01.806Z 12 240-259 141 141 141 256 256 256 F F F
153842024-02-28T22:19:01.806Z 13 260-279 138 138 138 254 254 254 F F F
153852024-02-28T22:19:01.806Z 14 280-299 142 138 142 259 254 259 T F T <---
153862024-02-28T22:19:01.806Z 15 300-319 137 137 137 253 253 253 F F F
153872024-02-28T22:19:01.807Z 16 320-339 141 141 141 256 256 256 F F F
153882024-02-28T22:19:01.807Z 17 340-359 141 141 141 256 256 256 F F F
153892024-02-28T22:19:01.807Z 18 360-379 142 140 142 257 255 257 F F F <---
153902024-02-28T22:19:01.807Z 19 380-399 142 136 142 257 252 257 F T F <---
153912024-02-28T22:19:01.807Z 20 400-419 138 138 138 254 254 254 F F F
153922024-02-28T22:19:01.807Z 21 420-439 141 141 141 256 256 256 F F F
153932024-02-28T22:19:01.807Z 22 440-459 141 141 141 256 256 256 T F T <---
153942024-02-28T22:19:01.807Z 23 460-479 137 137 137 253 253 253 F F F
153952024-02-28T22:19:01.807Z 24 480-499 133 133 133 245 245 245 F F F
153962024-02-28T22:19:01.808Z 25 500-519 141 141 141 256 256 256 T F T <---
153972024-02-28T22:19:01.808Z 26 520-539 138 138 138 254 254 254 T F T <---
153982024-02-28T22:19:01.808Z 27 540-559 141 141 141 256 256 256 F F F
153992024-02-28T22:19:01.808Z 28 560-579 142 140 142 259 255 259 F F F <---
154002024-02-28T22:19:01.808Z 29 580-599 142 140 142 259 255 259 F F F <---
154012024-02-28T22:19:01.808ZMax gen: 142, Max flush: 259
154022024-02-28T22:19:01.808ZError: Difference in extent metadata found!
154032024-02-28T22:19:01.808ZA Difference in extent metadata is expected here
154042024-02-28T22:19:01.808ZOn loop 71
154052024-02-28T22:19:01.808Z
154062024-02-28T22:19:01.809ZVerifying data now
154072024-02-28T22:19:06.414ZNo change after verify
154082024-02-28T22:19:06.414ZLoop: 71 Downstairs dump after verify (and repair):
154092024-02-28T22:19:06.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154102024-02-28T22:19:06.429Z 0 000-019 140 140 140 255 255 255 F F F
154112024-02-28T22:19:06.430Z 1 020-039 142 142 142 259 259 259 F F F
154122024-02-28T22:19:06.430Z 2 040-059 134 134 134 246 246 246 F F F
154132024-02-28T22:19:06.430Z 3 060-079 138 138 138 254 254 254 F F F
154142024-02-28T22:19:06.430Z 4 080-099 138 138 138 254 254 254 F F F
154152024-02-28T22:19:06.430Z 5 100-119 143 143 143 260 260 260 F F F
154162024-02-28T22:19:06.430Z 6 120-139 143 143 143 260 260 260 F F F
154172024-02-28T22:19:06.430Z 7 140-159 133 133 133 245 245 245 F F F
154182024-02-28T22:19:06.431Z 8 160-179 135 135 135 249 249 249 F F F
154192024-02-28T22:19:06.431Z 9 180-199 138 138 138 254 254 254 F F F
154202024-02-28T22:19:06.431Z 10 200-219 137 137 137 253 253 253 F F F
154212024-02-28T22:19:06.431Z 11 220-239 142 142 142 258 258 258 F F F
154222024-02-28T22:19:06.431Z 12 240-259 141 141 141 256 256 256 F F F
154232024-02-28T22:19:06.431Z 13 260-279 138 138 138 254 254 254 F F F
154242024-02-28T22:19:06.435Z 14 280-299 143 143 143 260 260 260 F F F
154252024-02-28T22:19:06.435Z 15 300-319 137 137 137 253 253 253 F F F
154262024-02-28T22:19:06.435Z 16 320-339 141 141 141 256 256 256 F F F
154272024-02-28T22:19:06.436Z 17 340-359 141 141 141 256 256 256 F F F
154282024-02-28T22:19:06.436Z 18 360-379 142 142 142 257 257 257 F F F
154292024-02-28T22:19:06.436Z 19 380-399 142 142 142 257 257 257 F F F
154302024-02-28T22:19:06.436Z 20 400-419 138 138 138 254 254 254 F F F
154312024-02-28T22:19:06.436Z 21 420-439 141 141 141 256 256 256 F F F
154322024-02-28T22:19:06.436Z 22 440-459 143 143 143 260 260 260 F F F
154332024-02-28T22:19:06.436Z 23 460-479 137 137 137 253 253 253 F F F
154342024-02-28T22:19:06.436Z 24 480-499 133 133 133 245 245 245 F F F
154352024-02-28T22:19:06.436Z 25 500-519 143 143 143 260 260 260 F F F
154362024-02-28T22:19:06.436Z 26 520-539 143 143 143 260 260 260 F F F
154372024-02-28T22:19:06.437Z 27 540-559 141 141 141 256 256 256 F F F
154382024-02-28T22:19:06.437Z 28 560-579 142 142 142 259 259 259 F F F
154392024-02-28T22:19:06.437Z 29 580-599 142 142 142 259 259 259 F F F
154402024-02-28T22:19:06.437ZMax gen: 143, Max flush: 260
154412024-02-28T22:19:06.437Z
154422024-02-28T22:19:06.437ZBegin loop 72
154432024-02-28T22:19:06.437ZDownstairs to restart: 2
154442024-02-28T22:19:06.450Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333253569Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
154452024-02-28T22:19:06.451Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","Crucible runtime is spawned
154462024-02-28T22:19:06.476Zv":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333588699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
154472024-02-28T22:19:06.476Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333631578Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
154482024-02-28T22:19:06.476Z{"msg":"Crucible e60facc2-90ce-4a6e-917b-586ce6731a27 has session id: df943cf7-e483-496c-9e1a-818e0681ff31","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333659937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154492024-02-28T22:19:06.476Z{"msg":"Upstairs opts: Upstairs UUID: e60facc2-90ce-4a6e-917b-586ce6731a27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333714006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154502024-02-28T22:19:06.476Z{"msg":"Crucible stats registered with UUID: e60facc2-90ce-4a6e-917b-586ce6731a27","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333754535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154512024-02-28T22:19:06.476Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333787974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
154522024-02-28T22:19:06.476Z{"msg":"e60facc2-90ce-4a6e-917b-586ce6731a27 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333812493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154532024-02-28T22:19:06.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333846882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154542024-02-28T22:19:06.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333885361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154552024-02-28T22:19:06.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33391555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154562024-02-28T22:19:06.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33394593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154572024-02-28T22:19:06.477Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.333997188Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154582024-02-28T22:19:06.477Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334025418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154592024-02-28T22:19:06.477Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334049907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154602024-02-28T22:19:06.477Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334090776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154612024-02-28T22:19:06.477Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334132224Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154622024-02-28T22:19:06.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33428906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154632024-02-28T22:19:06.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334343809Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154642024-02-28T22:19:06.477Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.334372038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154652024-02-28T22:19:06.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335047669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154662024-02-28T22:19:06.478Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335143037Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154672024-02-28T22:19:06.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335248454Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154682024-02-28T22:19:06.478Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335309992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154692024-02-28T22:19:06.478Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335340131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154702024-02-28T22:19:06.479Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33537985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154712024-02-28T22:19:06.479Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335722631Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154722024-02-28T22:19:06.479Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33576555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154732024-02-28T22:19:06.479Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.335993413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154742024-02-28T22:19:06.479Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336313274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154752024-02-28T22:19:06.479Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336357453Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154762024-02-28T22:19:06.479Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336383382Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154772024-02-28T22:19:06.479Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336406322Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154782024-02-28T22:19:06.479Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336437421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154792024-02-28T22:19:06.479Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336452751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154802024-02-28T22:19:06.480Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33648167Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154812024-02-28T22:19:06.480Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336507699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154822024-02-28T22:19:06.480Z{"msg":"[0]R flush_numbers[0..12]: [255, 259, 246, 254, 254, 260, 260, 245, 249, 254, 253, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336537998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154832024-02-28T22:19:06.480Z{"msg":"[0]R generation[0..12]: [140, 142, 134, 138, 138, 143, 143, 133, 135, 138, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336573957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154842024-02-28T22:19:06.480Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336589307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154852024-02-28T22:19:06.480Z{"msg":"[1]R flush_numbers[0..12]: [255, 259, 246, 254, 254, 260, 260, 245, 249, 254, 253, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336621456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154862024-02-28T22:19:06.480Z{"msg":"[1]R generation[0..12]: [140, 142, 134, 138, 138, 143, 143, 133, 135, 138, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336644025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154872024-02-28T22:19:06.480ZWait for a query_work_queue command to finish before sending IO
154882024-02-28T22:19:06.481Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336677314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154892024-02-28T22:19:06.481Z{"msg":"[2]R flush_numbers[0..12]: [255, 259, 246, 254, 254, 260, 260, 245, 249, 254, 253, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336710924Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154902024-02-28T22:19:06.481Z{"msg":"[2]R generation[0..12]: [140, 142, 134, 138, 138, 143, 143, 133, 135, 138, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336743912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154912024-02-28T22:19:06.481Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336777112Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154922024-02-28T22:19:06.481Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336808641Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154932024-02-28T22:19:06.482Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.33683952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154942024-02-28T22:19:06.482Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336871389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154952024-02-28T22:19:06.482Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336901468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"":"downstairs","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154962024-02-28T22:19:06.482Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336934107Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154972024-02-28T22:19:06.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336963617Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"0","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154982024-02-28T22:19:06.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.336996425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"client":"1","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
154992024-02-28T22:19:06.482Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337012725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155002024-02-28T22:19:06.482Z"pid":1711,"client":"2","":"downstairs","session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
155012024-02-28T22:19:06.482Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337044894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
155022024-02-28T22:19:06.482Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337066764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
155032024-02-28T22:19:06.482Z{"msg":"e60facc2-90ce-4a6e-917b-586ce6731a27 is now active with session: df943cf7-e483-496c-9e1a-818e0681ff31","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337091723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
155042024-02-28T22:19:06.483Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337107562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711,"session_id":"df943cf7-e483-496c-9e1a-818e0681ff31"}
155052024-02-28T22:19:06.483Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:05.337135402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
155062024-02-28T22:19:06.483ZLoading write count information from file "/tmp/test_repair_verify.data"
155072024-02-28T22:19:06.483ZRun Repair workload
155082024-02-28T22:19:06.483Z01/30 Write block 189 len 4096 data: 11
155092024-02-28T22:19:06.490Z02/30 Read block 258 len 36864
155102024-02-28T22:19:06.490Z03/30 Read block 388 len 24576
155112024-02-28T22:19:06.490Z04/30 Read block 516 len 12288
155122024-02-28T22:19:06.490Z05/30 Write block 411 len 20480 data: 8 9 10 13 14
155132024-02-28T22:19:06.490Z06/30 Flush
155142024-02-28T22:19:06.490Z07/30 Write block 44 len 8192 data: 7 9
155152024-02-28T22:19:06.491Z08/30 Write block 296 len 12288 data: 10 8 6
155162024-02-28T22:19:06.491Z09/30 Read block 228 len 24576
155172024-02-28T22:19:06.491Z10/30 Read block 346 len 8192
155182024-02-28T22:19:06.491Z11/30 Read block 498 len 40960
155192024-02-28T22:19:06.491Z12/30 Write block 6 len 32768 data: 14 17 17 11 10 12 14 12
155202024-02-28T22:19:06.491Z13/30 Write block 96 len 28672 data: 10 11 10 10 11 10 10
155212024-02-28T22:19:06.491Z14/30 Read block 290 len 32768
155222024-02-28T22:19:06.492Z15/30 Read block 312 len 36864
155232024-02-28T22:19:06.492Z16/30 Read block 106 len 4096
155242024-02-28T22:19:06.492Z17/30 Read block 242 len 40960
155252024-02-28T22:19:06.492Z18/30 Write block 113 len 4096 data: 12
155262024-02-28T22:19:06.492Z19/30 Read block 285 len 4096
155272024-02-28T22:19:06.492Z20/30 Read block 447 len 8192
155282024-02-28T22:19:06.492Z21/30 Read block 245 len 40960
155292024-02-28T22:19:06.492Z22/30 Read block 31 len 24576
155302024-02-28T22:19:06.492Z23/30 Read block 2 len 12288
155312024-02-28T22:19:06.492Z24/30 Write block 509 len 24576 data: 15 14 10 12 14 14
155322024-02-28T22:19:06.492Z25/30 Write block 349 len 28672 data: 8 9 10 11 12 12 12
155332024-02-28T22:19:06.492Z26/30 Read block 366 len 16384
155342024-02-28T22:19:06.492Z27/30 Read block 338 len 24576
155352024-02-28T22:19:06.492Z28/30 Read block 224 len 20480
155362024-02-28T22:19:06.492Z29/30 Write block 299 len 40960 data: 5 4 6 8 11 11 11 11 14 13
155372024-02-28T22:19:06.492Z30/30 Read block 442 len 4096
155382024-02-28T22:19:06.492Z----------------------------------------------------------------
155392024-02-28T22:19:06.492Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155402024-02-28T22:19:06.492ZStates: Active Active Active
155412024-02-28T22:19:06.492ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155422024-02-28T22:19:06.492Z 1 Acked 1000 Write 1 Done Done Done false
155432024-02-28T22:19:06.492Z 2 Acked 1001 Read 9 Done Done Done false
155442024-02-28T22:19:06.493Z 3 Acked 1002 Read 6 Done Done Done false
155452024-02-28T22:19:06.493Z 4 Acked 1003 Read 3 Done Done Done false
155462024-02-28T22:19:06.493Z 5 Acked 1004 Write 5 Done Done Sent false
155472024-02-28T22:19:06.493Z 6 Acked 1005 Flush 0 Done Done Sent false
155482024-02-28T22:19:06.493Z 7 Acked 1006 Write 2 Done Done Sent false
155492024-02-28T22:19:06.493Z 8 Acked 1007 Write 3 Done Done Sent false
155502024-02-28T22:19:06.493Z 9 Acked 1008 Read 6 Done Done Sent false
155512024-02-28T22:19:06.493Z 10 Acked 1009 Read 2 Done Done Sent false
155522024-02-28T22:19:06.493Z 11 Acked 1010 Read 10 Done Done Sent false
155532024-02-28T22:19:06.493Z 12 Acked 1011 Write 8 Done Done Sent false
155542024-02-28T22:19:06.493Z 13 Acked 1012 Write 7 Done Done Sent false
155552024-02-28T22:19:06.493Z 14 Acked 1013 Read 8 Done Done Sent false
155562024-02-28T22:19:06.493Z 15 Acked 1014 Read 9 Done Done Sent false
155572024-02-28T22:19:06.493Z 16 Acked 1015 Read 1 Done Done Sent false
155582024-02-28T22:19:06.493Z 17 Acked 1016 Read 10 Done Done Sent false
155592024-02-28T22:19:06.493Z 18 Acked 1017 Write 1 Done Done Sent false
155602024-02-28T22:19:06.493Z 19 Acked 1018 Read 1 Done Done Sent false
155612024-02-28T22:19:06.493Z 20 Acked 1019 Read 2 Done Done Sent false
155622024-02-28T22:19:06.493Z 21 Acked 1020 Read 10 Done Done Sent false
155632024-02-28T22:19:06.493Z 22 Acked 1021 Read 6 Done Done Sent false
155642024-02-28T22:19:06.493Z 23 Acked 1022 Read 3 Done Done Sent false
155652024-02-28T22:19:06.493Z 24 Acked 1023 Write 6 Done Done Sent false
155662024-02-28T22:19:06.493Z 25 Acked 1024 Write 7 Done Done Sent false
155672024-02-28T22:19:06.493Z 26 Acked 1025 Read 4 Done Done Sent false
155682024-02-28T22:19:06.494Z 27 Acked 1026 Read 6 Done Done Sent false
155692024-02-28T22:19:06.494Z 28 Acked 1027 Read 5 Done Done Sent false
155702024-02-28T22:19:06.494Z 29 Acked 1028 Write 10 Done Done Sent false
155712024-02-28T22:19:06.494Z 30 Acked 1029 Read 1 Done Done Sent false
155722024-02-28T22:19:06.494Z STATES DS:0 DS:1 DS:2 TOTAL
155732024-02-28T22:19:06.494Z New 0 0 0 0
155742024-02-28T22:19:06.494Z Sent 0 0 26 26
155752024-02-28T22:19:06.494Z Done 30 30 4 64
155762024-02-28T22:19:06.494Z Skipped 0 0 0 0
155772024-02-28T22:19:06.494Z Error 0 0 0 0
155782024-02-28T22:19:06.494ZLast Flush: 1005 1005 0
155792024-02-28T22:19:06.494ZDownstairs last five completed:
155802024-02-28T22:19:06.494ZUpstairs last five completed: 30 29 28 27 26
155812024-02-28T22:19:06.494Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:05.365743244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1711}
155822024-02-28T22:19:06.494ZWrote out file "/tmp/test_repair_verify.data"
155832024-02-28T22:19:06.494Z
155842024-02-28T22:19:06.498ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155852024-02-28T22:19:06.508ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155862024-02-28T22:19:06.508Z 0 000-019 140 140 140 255 255 255 T T F <---
155872024-02-28T22:19:06.509Z 1 020-039 142 142 142 259 259 259 F F F
155882024-02-28T22:19:06.509Z 2 040-059 134 134 134 246 246 246 T T F <---
155892024-02-28T22:19:06.509Z 3 060-079 138 138 138 254 254 254 F F F
155902024-02-28T22:19:06.509Z 4 080-099 138 138 138 254 254 254 T T F <---
155912024-02-28T22:19:06.509Z 5 100-119 143 143 143 260 260 260 T T F <---
155922024-02-28T22:19:06.509Z 6 120-139 143 143 143 260 260 260 F F F
155932024-02-28T22:19:06.509Z 7 140-159 133 133 133 245 245 245 F F F
155942024-02-28T22:19:06.510Z 8 160-179 135 135 135 249 249 249 F F F
155952024-02-28T22:19:06.510Z 9 180-199 144 144 138 261 261 254 F F T <---
155962024-02-28T22:19:06.510Z 10 200-219 137 137 137 253 253 253 F F F
155972024-02-28T22:19:06.510Z 11 220-239 142 142 142 258 258 258 F F F
155982024-02-28T22:19:06.510Z 12 240-259 141 141 141 256 256 256 F F F
155992024-02-28T22:19:06.510Z 13 260-279 138 138 138 254 254 254 F F F
156002024-02-28T22:19:06.515Z 14 280-299 143 143 143 260 260 260 T T F <---
156012024-02-28T22:19:06.515Z 15 300-319 137 137 137 253 253 253 T T F <---
156022024-02-28T22:19:06.515Z 16 320-339 141 141 141 256 256 256 F F F
156032024-02-28T22:19:06.515Z 17 340-359 141 141 141 256 256 256 T T F <---
156042024-02-28T22:19:06.515Z 18 360-379 142 142 142 257 257 257 F F F
156052024-02-28T22:19:06.515Z 19 380-399 142 142 142 257 257 257 F F F
156062024-02-28T22:19:06.515Z 20 400-419 144 144 138 261 261 254 F F F <---
156072024-02-28T22:19:06.515Z 21 420-439 141 141 141 256 256 256 F F F
156082024-02-28T22:19:06.516Z 22 440-459 143 143 143 260 260 260 F F F
156092024-02-28T22:19:06.516Z 23 460-479 137 137 137 253 253 253 F F F
156102024-02-28T22:19:06.516Z 24 480-499 133 133 133 245 245 245 F F F
156112024-02-28T22:19:06.516Z 25 500-519 143 143 143 260 260 260 T T F <---
156122024-02-28T22:19:06.516Z 26 520-539 143 143 143 260 260 260 F F F
156132024-02-28T22:19:06.516Z 27 540-559 141 141 141 256 256 256 F F F
156142024-02-28T22:19:06.516Z 28 560-579 142 142 142 259 259 259 F F F
156152024-02-28T22:19:06.516Z 29 580-599 142 142 142 259 259 259 F F F
156162024-02-28T22:19:06.516ZMax gen: 144, Max flush: 261
156172024-02-28T22:19:06.516ZError: Difference in extent metadata found!
156182024-02-28T22:19:06.516ZA Difference in extent metadata is expected here
156192024-02-28T22:19:06.516ZOn loop 72
156202024-02-28T22:19:06.516Z
156212024-02-28T22:19:06.516ZVerifying data now
156222024-02-28T22:19:12.062ZNo change after verify
156232024-02-28T22:19:12.062ZLoop: 72 Downstairs dump after verify (and repair):
156242024-02-28T22:19:12.077ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156252024-02-28T22:19:12.077Z 0 000-019 145 145 145 262 262 262 F F F
156262024-02-28T22:19:12.077Z 1 020-039 142 142 142 259 259 259 F F F
156272024-02-28T22:19:12.077Z 2 040-059 145 145 145 262 262 262 F F F
156282024-02-28T22:19:12.077Z 3 060-079 138 138 138 254 254 254 F F F
156292024-02-28T22:19:12.077Z 4 080-099 145 145 145 262 262 262 F F F
156302024-02-28T22:19:12.077Z 5 100-119 145 145 145 262 262 262 F F F
156312024-02-28T22:19:12.077Z 6 120-139 143 143 143 260 260 260 F F F
156322024-02-28T22:19:12.077Z 7 140-159 133 133 133 245 245 245 F F F
156332024-02-28T22:19:12.077Z 8 160-179 135 135 135 249 249 249 F F F
156342024-02-28T22:19:12.077Z 9 180-199 144 144 144 261 261 261 F F F
156352024-02-28T22:19:12.091Z 10 200-219 137 137 137 253 253 253 F F F
156362024-02-28T22:19:12.091Z 11 220-239 142 142 142 258 258 258 F F F
156372024-02-28T22:19:12.091Z 12 240-259 141 141 141 256 256 256 F F F
156382024-02-28T22:19:12.091Z 13 260-279 138 138 138 254 254 254 F F F
156392024-02-28T22:19:12.091Z 14 280-299 145 145 145 262 262 262 F F F
156402024-02-28T22:19:12.091Z 15 300-319 145 145 145 262 262 262 F F F
156412024-02-28T22:19:12.091Z 16 320-339 141 141 141 256 256 256 F F F
156422024-02-28T22:19:12.091Z 17 340-359 145 145 145 262 262 262 F F F
156432024-02-28T22:19:12.091Z 18 360-379 142 142 142 257 257 257 F F F
156442024-02-28T22:19:12.092Z 19 380-399 142 142 142 257 257 257 F F F
156452024-02-28T22:19:12.092Z 20 400-419 144 144 144 261 261 261 F F F
156462024-02-28T22:19:12.092Z 21 420-439 141 141 141 256 256 256 F F F
156472024-02-28T22:19:12.092Z 22 440-459 143 143 143 260 260 260 F F F
156482024-02-28T22:19:12.092Z 23 460-479 137 137 137 253 253 253 F F F
156492024-02-28T22:19:12.092Z 24 480-499 133 133 133 245 245 245 F F F
156502024-02-28T22:19:12.092Z 25 500-519 145 145 145 262 262 262 F F F
156512024-02-28T22:19:12.092Z 26 520-539 143 143 143 260 260 260 F F F
156522024-02-28T22:19:12.092Z 27 540-559 141 141 141 256 256 256 F F F
156532024-02-28T22:19:12.092Z 28 560-579 142 142 142 259 259 259 F F F
156542024-02-28T22:19:12.092Z 29 580-599 142 142 142 259 259 259 F F F
156552024-02-28T22:19:12.092ZMax gen: 145, Max flush: 262
156562024-02-28T22:19:12.092Z
156572024-02-28T22:19:12.093ZBegin loop 73
156582024-02-28T22:19:12.093ZDownstairs to restart: 0
156592024-02-28T22:19:12.100Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.982622404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
156602024-02-28T22:19:12.111Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
156612024-02-28T22:19:12.111Z,"time":"2024-02-28T22:19:10.982918156Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
156622024-02-28T22:19:12.111Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.982972714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
156632024-02-28T22:19:12.111Z{"msg":"Crucible 34dd7476-785e-4dbf-a1b3-50b9606cff7c has session id: 2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983043682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156642024-02-28T22:19:12.111Z{"msg":"Upstairs opts: Upstairs UUID: 34dd7476-785e-4dbf-a1b3-50b9606cff7c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983093441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156652024-02-28T22:19:12.111Z{"msg":"Crucible stats registered with UUID: 34dd7476-785e-4dbf-a1b3-50b9606cff7c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983113611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156662024-02-28T22:19:12.111Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.9831404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
156672024-02-28T22:19:12.111Z{"msg":"34dd7476-785e-4dbf-a1b3-50b9606cff7c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983159649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156682024-02-28T22:19:12.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983185969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156692024-02-28T22:19:12.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983209858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156702024-02-28T22:19:12.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983241227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156712024-02-28T22:19:12.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983296115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156722024-02-28T22:19:12.112Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983370344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156732024-02-28T22:19:12.112Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983422732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156742024-02-28T22:19:12.112Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983458961Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156752024-02-28T22:19:12.112Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983532629Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156762024-02-28T22:19:12.112Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983575348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156772024-02-28T22:19:12.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983595347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156782024-02-28T22:19:12.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983616687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156792024-02-28T22:19:12.112Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.983643286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156802024-02-28T22:19:12.112Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984298858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156812024-02-28T22:19:12.112Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984341417Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156822024-02-28T22:19:12.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984357566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156832024-02-28T22:19:12.113Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984371256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156842024-02-28T22:19:12.113Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984567551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156852024-02-28T22:19:12.113Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984621559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156862024-02-28T22:19:12.113Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984865472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156872024-02-28T22:19:12.113Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.984995069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156882024-02-28T22:19:12.113Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985263361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156892024-02-28T22:19:12.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985436727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156902024-02-28T22:19:12.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985468256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156912024-02-28T22:19:12.113Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985494555Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156922024-02-28T22:19:12.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985524474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156932024-02-28T22:19:12.113Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985551023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156942024-02-28T22:19:12.113Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985581532Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156952024-02-28T22:19:12.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985717449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156962024-02-28T22:19:12.113Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985749178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156972024-02-28T22:19:12.113Z{"msg":"[0]R flush_numbers[0..12]: [262, 259, 262, 254, 262, 262, 260, 245, 249, 261, 253, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985775227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156982024-02-28T22:19:12.113Z{"msg":"[0]R generation[0..12]: [145, 142, 145, 138, 145, 145, 143, 133, 135, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985805766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
156992024-02-28T22:19:12.113Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985839225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157002024-02-28T22:19:12.113Z{"msg":"[1]R flush_numbers[0..12]: [262, 259, 262, 254, 262, 262, 260, 245, 249, 261, 253, 258]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985872104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157012024-02-28T22:19:12.113Z{"msg":"[1]R generation[0..12]: [145, 142, 145, 138, 145, 145, 143, 133, 135, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985897034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157022024-02-28T22:19:12.113Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985933773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157032024-02-28T22:19:12.113Z{"msg":"[2]R flush_numbers[0..12]: [262, 259, 262, 254, 262, 262, 260, 245, 249, 261, 253, 258]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
157042024-02-28T22:19:12.113Z,"time":"2024-02-28T22:19:10.985956722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157052024-02-28T22:19:12.114Z{"msg":"[2]R generation[0..12]: [145, 142, 145, 138, 145, 145, 143, 133, 135, 144, 137, 142]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.985990471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157062024-02-28T22:19:12.114Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.98602275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157072024-02-28T22:19:12.114Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.98603869Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157082024-02-28T22:19:12.114Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986060379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157092024-02-28T22:19:12.114Z{"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986077459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157102024-02-28T22:19:12.114Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986111948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"":"downstairs","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157112024-02-28T22:19:12.114Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986147117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157122024-02-28T22:19:12.114Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986163376Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"0","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157132024-02-28T22:19:12.114Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986190016Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"1","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157142024-02-28T22:19:12.114Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986225645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"client":"2","":"downstairs","session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157152024-02-28T22:19:12.114Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986258634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157162024-02-28T22:19:12.114Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986273553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157172024-02-28T22:19:12.114Z{"msg":"34dd7476-785e-4dbf-a1b3-50b9606cff7c is now active with session: 2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986301433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157182024-02-28T22:19:12.114Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986336682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719,"session_id":"2d6ad956-16b3-4caa-b1e2-c0ca5f9d6160"}
157192024-02-28T22:19:12.114Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:10.986362581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
157202024-02-28T22:19:12.114ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157212024-02-28T22:19:12.114ZLoading write count information from file "/tmp/test_repair_verify.data"
157222024-02-28T22:19:12.114ZRun Repair workload
157232024-02-28T22:19:12.114Z01/30 Write block 440 len 16384 data: 14 13 13 10
157242024-02-28T22:19:12.114Z02/30 Read block 33 len 36864
157252024-02-28T22:19:12.114Z03/30 Write block 235 len 12288 data: 11 10 11
157262024-02-28T22:19:12.114Z04/30 Flush
157272024-02-28T22:19:12.119Z05/30 Write block 112 len 32768 data: 15 13 14 12 12 11 9 9
157282024-02-28T22:19:12.119Z06/30 Read block 263 len 12288
157292024-02-28T22:19:12.119Z07/30 Write block 506 len 16384 data: 16 14 13 16
157302024-02-28T22:19:12.119Z08/30 Write block 371 len 32768 data: 14 16 13 14 13 13 13 14
157312024-02-28T22:19:12.119Z09/30 Read block 232 len 36864
157322024-02-28T22:19:12.119Z10/30 Read block 313 len 20480
157332024-02-28T22:19:12.119Z11/30 Flush
157342024-02-28T22:19:12.119Z12/30 Flush
157352024-02-28T22:19:12.154Z13/30 Write block 215 len 32768 data: 7 7 8 7 7 8 9 7
157362024-02-28T22:19:12.154Z14/30 Read block 122 len 32768
157372024-02-28T22:19:12.154Z15/30 Read block 537 len 20480
157382024-02-28T22:19:12.154Z16/30 Read block 29 len 28672
157392024-02-28T22:19:12.154Z17/30 Read block 369 len 8192
157402024-02-28T22:19:12.155Z18/30 Write block 223 len 16384 data: 6 7 10 12
157412024-02-28T22:19:12.155Z19/30 Write block 355 len 20480 data: 13 11 12 10 8
157422024-02-28T22:19:12.155Z20/30 Read block 7 len 8192
157432024-02-28T22:19:12.155Z21/30 Read block 356 len 40960
157442024-02-28T22:19:12.155Z22/30 Write block 73 len 8192 data: 11 11
157452024-02-28T22:19:12.155Z23/30 Read block 151 len 8192
157462024-02-28T22:19:12.155Z24/30 Flush
157472024-02-28T22:19:12.156Z25/30 Write block 25 len 36864 data: 4 5 8 8 11 11 11 10 10
157482024-02-28T22:19:12.156Z26/30 Read block 9 len 4096
157492024-02-28T22:19:12.156Z27/30 Read block 153 len 24576
157502024-02-28T22:19:12.156Z28/30 Write block 126 len 20480 data: 14 17 20 16 15
157512024-02-28T22:19:12.156Z29/30 Write block 271 len 8192 data: 9 7
157522024-02-28T22:19:12.156Z30/30 Read block 296 len 16384
157532024-02-28T22:19:12.156Z----------------------------------------------------------------
157542024-02-28T22:19:12.156Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
157552024-02-28T22:19:12.156ZStates: Active Active Active
157562024-02-28T22:19:12.156ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157572024-02-28T22:19:12.156Z 1 Acked 1000 Write 4 Done Done Done false
157582024-02-28T22:19:12.156Z 2 Acked 1001 Read 9 Done Done Done false
157592024-02-28T22:19:12.157Z 3 Acked 1002 Write 3 Sent Done Done false
157602024-02-28T22:19:12.157Z 4 Acked 1003 Flush 0 Sent Done Done false
157612024-02-28T22:19:12.157Z 5 Acked 1004 Write 8 Sent Done Done false
157622024-02-28T22:19:12.157Z 6 Acked 1005 Read 3 Sent Done Done false
157632024-02-28T22:19:12.157Z 7 Acked 1006 Write 4 Sent Done Done false
157642024-02-28T22:19:12.157Z 8 Acked 1007 Write 8 Sent Done Done false
157652024-02-28T22:19:12.157Z 9 Acked 1008 Read 9 Sent Done Done false
157662024-02-28T22:19:12.157Z 10 Acked 1009 Read 5 Sent Done Done false
157672024-02-28T22:19:12.157Z 11 Acked 1010 Flush 0 Sent Done Done false
157682024-02-28T22:19:12.157Z 12 Acked 1011 Flush 0 Sent Done Done false
157692024-02-28T22:19:12.157Z 13 Acked 1012 Write 8 Sent Done Done false
157702024-02-28T22:19:12.157Z 14 Acked 1013 Read 8 Sent Done Done false
157712024-02-28T22:19:12.158Z 15 Acked 1014 Read 5 Sent Done Done false
157722024-02-28T22:19:12.158Z 16 Acked 1015 Read 7 Sent Done Done false
157732024-02-28T22:19:12.158Z 17 Acked 1016 Read 2 Sent Done Done false
157742024-02-28T22:19:12.158Z 18 Acked 1017 Write 4 Sent Done Done false
157752024-02-28T22:19:12.158Z 19 Acked 1018 Write 5 Sent Done Done false
157762024-02-28T22:19:12.158Z 20 Acked 1019 Read 2 Sent Done Done false
157772024-02-28T22:19:12.158Z 21 Acked 1020 Read 10 Sent Done Done false
157782024-02-28T22:19:12.158Z 22 Acked 1021 Write 2 Sent Done Done false
157792024-02-28T22:19:12.158Z 23 Acked 1022 Read 2 Sent Done Done false
157802024-02-28T22:19:12.158Z 24 Acked 1023 Flush 0 Sent Done Done false
157812024-02-28T22:19:12.158Z 25 Acked 1024 Write 9 Sent Done Done false
157822024-02-28T22:19:12.158Z 26 Acked 1025 Read 1 Sent Done Done false
157832024-02-28T22:19:12.158Z 27 Acked 1026 Read 6 Sent Done Done false
157842024-02-28T22:19:12.159Z 28 Acked 1027 Write 5 Sent Done Done false
157852024-02-28T22:19:12.159Z 29 Acked 1028 Write 2 Sent Done Done false
157862024-02-28T22:19:12.159Z 30 Acked 1029 Read 4 Sent Done Sent false
157872024-02-28T22:19:12.159Z STATES DS:0 DS:1 DS:2 TOTAL
157882024-02-28T22:19:12.159Z New 0 0 0 0
157892024-02-28T22:19:12.159Z Sent 28 0 1 29
157902024-02-28T22:19:12.159Z Done 2 30 29 61
157912024-02-28T22:19:12.159Z Skipped 0 0 0 0
157922024-02-28T22:19:12.159Z Error 0 0 0 0
157932024-02-28T22:19:12.159ZLast Flush: 0 1023 1023
157942024-02-28T22:19:12.159ZDownstairs last five completed:
157952024-02-28T22:19:12.159ZUpstairs last five completed: 30 29 28 27 26
157962024-02-28T22:19:12.160Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:11.018020549Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1719}
157972024-02-28T22:19:12.160ZWrote out file "/tmp/test_repair_verify.data"
157982024-02-28T22:19:12.160Z
157992024-02-28T22:19:12.160ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158002024-02-28T22:19:12.165ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158012024-02-28T22:19:12.165Z 0 000-019 145 145 145 262 262 262 F F F
158022024-02-28T22:19:12.165Z 1 020-039 142 142 142 259 259 259 F T T <---
158032024-02-28T22:19:12.165Z 2 040-059 145 145 145 262 262 262 F F F
158042024-02-28T22:19:12.165Z 3 060-079 138 146 146 254 266 266 F F F <---
158052024-02-28T22:19:12.165Z 4 080-099 145 145 145 262 262 262 F F F
158062024-02-28T22:19:12.165Z 5 100-119 145 146 146 262 264 264 F F F <---
158072024-02-28T22:19:12.165Z 6 120-139 143 143 143 260 260 260 F T T <---
158082024-02-28T22:19:12.165Z 7 140-159 133 133 133 245 245 245 F F F
158092024-02-28T22:19:12.166Z 8 160-179 135 135 135 249 249 249 F F F
158102024-02-28T22:19:12.166Z 9 180-199 144 144 144 261 261 261 F F F
158112024-02-28T22:19:12.166Z 10 200-219 137 146 146 253 266 266 F F F <---
158122024-02-28T22:19:12.166Z 11 220-239 142 146 146 258 266 266 F F F <---
158132024-02-28T22:19:12.166Z 12 240-259 141 141 141 256 256 256 F F F
158142024-02-28T22:19:12.166Z 13 260-279 138 138 138 254 254 254 F T T <---
158152024-02-28T22:19:12.166Z 14 280-299 145 145 145 262 262 262 F F F
158162024-02-28T22:19:12.166Z 15 300-319 145 145 145 262 262 262 F F F
158172024-02-28T22:19:12.166Z 16 320-339 141 141 141 256 256 256 F F F
158182024-02-28T22:19:12.166Z 17 340-359 145 146 146 262 266 266 F F F <---
158192024-02-28T22:19:12.166Z 18 360-379 142 146 146 257 264 264 F F F <---
158202024-02-28T22:19:12.166Z 19 380-399 142 142 142 257 257 257 F F F
158212024-02-28T22:19:12.167Z 20 400-419 144 144 144 261 261 261 F F F
158222024-02-28T22:19:12.167Z 21 420-439 141 141 141 256 256 256 F F F
158232024-02-28T22:19:12.167Z 22 440-459 143 146 146 260 263 263 T F F <---
158242024-02-28T22:19:12.167Z 23 460-479 137 137 137 253 253 253 F F F
158252024-02-28T22:19:12.167Z 24 480-499 133 133 133 245 245 245 F F F
158262024-02-28T22:19:12.167Z 25 500-519 145 146 146 262 264 264 F F F <---
158272024-02-28T22:19:12.167Z 26 520-539 143 143 143 260 260 260 F F F
158282024-02-28T22:19:12.167Z 27 540-559 141 141 141 256 256 256 F F F
158292024-02-28T22:19:12.167Z 28 560-579 142 142 142 259 259 259 F F F
158302024-02-28T22:19:12.167Z 29 580-599 142 142 142 259 259 259 F F F
158312024-02-28T22:19:12.167ZMax gen: 146, Max flush: 266
158322024-02-28T22:19:12.168ZError: Difference in extent metadata found!
158332024-02-28T22:19:12.168ZA Difference in extent metadata is expected here
158342024-02-28T22:19:12.168ZOn loop 73
158352024-02-28T22:19:12.168Z
158362024-02-28T22:19:12.168ZVerifying data now
158372024-02-28T22:19:16.733ZNo change after verify
158382024-02-28T22:19:16.733ZLoop: 73 Downstairs dump after verify (and repair):
158392024-02-28T22:19:16.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158402024-02-28T22:19:16.748Z 0 000-019 145 145 145 262 262 262 F F F
158412024-02-28T22:19:16.748Z 1 020-039 147 147 147 267 267 267 F F F
158422024-02-28T22:19:16.748Z 2 040-059 145 145 145 262 262 262 F F F
158432024-02-28T22:19:16.748Z 3 060-079 146 146 146 266 266 266 F F F
158442024-02-28T22:19:16.749Z 4 080-099 145 145 145 262 262 262 F F F
158452024-02-28T22:19:16.749Z 5 100-119 146 146 146 264 264 264 F F F
158462024-02-28T22:19:16.749Z 6 120-139 147 147 147 267 267 267 F F F
158472024-02-28T22:19:16.749Z 7 140-159 133 133 133 245 245 245 F F F
158482024-02-28T22:19:16.749Z 8 160-179 135 135 135 249 249 249 F F F
158492024-02-28T22:19:16.749Z 9 180-199 144 144 144 261 261 261 F F F
158502024-02-28T22:19:16.749Z 10 200-219 146 146 146 266 266 266 F F F
158512024-02-28T22:19:16.750Z 11 220-239 146 146 146 266 266 266 F F F
158522024-02-28T22:19:16.750Z 12 240-259 141 141 141 256 256 256 F F F
158532024-02-28T22:19:16.750Z 13 260-279 147 147 147 267 267 267 F F F
158542024-02-28T22:19:16.750Z 14 280-299 145 145 145 262 262 262 F F F
158552024-02-28T22:19:16.750Z 15 300-319 145 145 145 262 262 262 F F F
158562024-02-28T22:19:16.750Z 16 320-339 141 141 141 256 256 256 F F F
158572024-02-28T22:19:16.756Z 17 340-359 146 146 146 266 266 266 F F F
158582024-02-28T22:19:16.756Z 18 360-379 146 146 146 264 264 264 F F F
158592024-02-28T22:19:16.756Z 19 380-399 142 142 142 257 257 257 F F F
158602024-02-28T22:19:16.757Z 20 400-419 144 144 144 261 261 261 F F F
158612024-02-28T22:19:16.757Z 21 420-439 141 141 141 256 256 256 F F F
158622024-02-28T22:19:16.757Z 22 440-459 146 146 146 263 263 263 F F F
158632024-02-28T22:19:16.757Z 23 460-479 137 137 137 253 253 253 F F F
158642024-02-28T22:19:16.757Z 24 480-499 133 133 133 245 245 245 F F F
158652024-02-28T22:19:16.757Z 25 500-519 146 146 146 264 264 264 F F F
158662024-02-28T22:19:16.757Z 26 520-539 143 143 143 260 260 260 F F F
158672024-02-28T22:19:16.757Z 27 540-559 141 141 141 256 256 256 F F F
158682024-02-28T22:19:16.757Z 28 560-579 142 142 142 259 259 259 F F F
158692024-02-28T22:19:16.757Z 29 580-599 142 142 142 259 259 259 F F F
158702024-02-28T22:19:16.757ZMax gen: 147, Max flush: 267
158712024-02-28T22:19:16.758Z
158722024-02-28T22:19:16.758ZBegin loop 74
158732024-02-28T22:19:16.758ZDownstairs to restart: 2
158742024-02-28T22:19:16.807Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65515705Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
158752024-02-28T22:19:16.807Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655397274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
158762024-02-28T22:19:16.816Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655419613Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
158772024-02-28T22:19:16.816Z{"msg":"Crucible 77b63adc-4bcf-4691-8474-31c3049fdad7 has session id: 67b7d9f8-8a16-4546-a378-4f299343c027","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655453482Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158782024-02-28T22:19:16.817Z{"msg":"Upstairs opts: Upstairs UUID: 77b63adc-4bcf-4691-8474-31c3049fdad7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655501221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158792024-02-28T22:19:16.817ZCrucible runtime is spawned
158802024-02-28T22:19:16.817Z{"msg":"Crucible stats registered with UUID: 77b63adc-4bcf-4691-8474-31c3049fdad7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655550159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158812024-02-28T22:19:16.817Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655651997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
158822024-02-28T22:19:16.817Z{"msg":"77b63adc-4bcf-4691-8474-31c3049fdad7 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655770684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158832024-02-28T22:19:16.817Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655813752Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158842024-02-28T22:19:16.817Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65588347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158852024-02-28T22:19:16.817Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.655914889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158862024-02-28T22:19:16.817Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656024666Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158872024-02-28T22:19:16.818Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656076785Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158882024-02-28T22:19:16.818Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656122474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158892024-02-28T22:19:16.818Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656162703Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158902024-02-28T22:19:16.818Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656200492Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158912024-02-28T22:19:16.818Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65624242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158922024-02-28T22:19:16.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65627394Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158932024-02-28T22:19:16.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656302469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158942024-02-28T22:19:16.818Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656342568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158952024-02-28T22:19:16.818Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.656921512Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158962024-02-28T22:19:16.823Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65696612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158972024-02-28T22:19:16.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65698309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158982024-02-28T22:19:16.823Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657016699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
158992024-02-28T22:19:16.823Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657045588Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159002024-02-28T22:19:16.823Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657074308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159012024-02-28T22:19:16.823Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657593843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159022024-02-28T22:19:16.824Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657636342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159032024-02-28T22:19:16.824Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.657883435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159042024-02-28T22:19:16.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658100839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159052024-02-28T22:19:16.824Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658153848Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159062024-02-28T22:19:16.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658199346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159072024-02-28T22:19:16.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658238916Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159082024-02-28T22:19:16.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658255445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159092024-02-28T22:19:16.824Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658286684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159102024-02-28T22:19:16.824Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658359122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159112024-02-28T22:19:16.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658394391Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159122024-02-28T22:19:16.825Z{"msg":"[0]R flush_numbers[0..12]: [262, 267, 262, 266, 262, 264, 267, 245, 249, 261, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658410741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159132024-02-28T22:19:16.825Z{"msg":"[0]R generation[0..12]: [145, 147, 145, 146, 145, 146, 147, 133, 135, 144, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65843814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159142024-02-28T22:19:16.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658454039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159152024-02-28T22:19:16.825Z{"msg":"[1]R flush_numbers[0..12]: [262, 267, 262, 266, 262, 264, 267, 245, 249, 261, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658475009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159162024-02-28T22:19:16.825Z{"msg":"[1]R generation[0..12]: [145, 147, 145, 146, 145, 146, 147, 133, 135, 144, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658489598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159172024-02-28T22:19:16.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658515708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159182024-02-28T22:19:16.825Z{"msg":"[2]R flush_numbers[0..12]: [262, 267, 262, 266, 262, 264, 267, 245, 249, 261, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658536887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159192024-02-28T22:19:16.825ZWait for a query_work_queue command to finish before sending IO
159202024-02-28T22:19:16.825Z{"msg":"[2]R generation[0..12]: [145, 147, 145, 146, 145, 146, 147, 133, 135, 144, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658567307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159212024-02-28T22:19:16.825Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658584486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159222024-02-28T22:19:16.825Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658604915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159232024-02-28T22:19:16.825Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658619045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159242024-02-28T22:19:16.825Z{"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658646474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159252024-02-28T22:19:16.825Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658680273Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"":"downstairs","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159262024-02-28T22:19:16.825Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658699063Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159272024-02-28T22:19:16.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658728002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"0","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159282024-02-28T22:19:16.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658746662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"1","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159292024-02-28T22:19:16.825Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658775531Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"client":"2","":"downstairs","session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159302024-02-28T22:19:16.825Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65879338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159312024-02-28T22:19:16.826Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.65882128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159322024-02-28T22:19:16.826Z{"msg":"77b63adc-4bcf-4691-8474-31c3049fdad7 is now active with session: 67b7d9f8-8a16-4546-a378-4f299343c027","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658839509Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159332024-02-28T22:19:16.826Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658866898Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727,"session_id":"67b7d9f8-8a16-4546-a378-4f299343c027"}
159342024-02-28T22:19:16.826Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:15.658884098Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
159352024-02-28T22:19:16.826ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159362024-02-28T22:19:16.826ZLoading write count information from file "/tmp/test_repair_verify.data"
159372024-02-28T22:19:16.826ZRun Repair workload
159382024-02-28T22:19:16.826Z01/30 Write block 115 len 28672 data: 13 13 12 10 10 10 12
159392024-02-28T22:19:16.826Z02/30 Write block 366 len 8192 data: 9 9
159402024-02-28T22:19:16.826Z03/30 Read block 79 len 16384
159412024-02-28T22:19:16.826Z04/30 Write block 195 len 8192 data: 11 11
159422024-02-28T22:19:16.826Z05/30 Read block 438 len 8192
159432024-02-28T22:19:16.826Z06/30 Read block 98 len 20480
159442024-02-28T22:19:16.826Z07/30 Read block 411 len 8192
159452024-02-28T22:19:16.826Z08/30 Write block 147 len 28672 data: 13 14 12 10 8 8 5
159462024-02-28T22:19:16.826Z09/30 Read block 412 len 24576
159472024-02-28T22:19:16.826Z10/30 Read block 472 len 28672
159482024-02-28T22:19:16.826Z11/30 Read block 571 len 8192
159492024-02-28T22:19:16.827Z12/30 Read block 256 len 40960
159502024-02-28T22:19:16.827Z13/30 Read block 28 len 32768
159512024-02-28T22:19:16.827Z14/30 Write block 434 len 28672 data: 13 14 16 15 16 16 15
159522024-02-28T22:19:16.827Z15/30 Write block 12 len 40960 data: 15 13 10 10 11 12 11 11 8 7
159532024-02-28T22:19:16.827Z16/30 Write block 502 len 8192 data: 14 15
159542024-02-28T22:19:16.827Z17/30 Flush
159552024-02-28T22:19:16.827Z18/30 Write block 331 len 28672 data: 15 10 11 12 12 12 10
159562024-02-28T22:19:16.827Z19/30 Write block 448 len 4096 data: 13
159572024-02-28T22:19:16.827Z20/30 Write block 22 len 32768 data: 6 5 5 5 6 9 9 12
159582024-02-28T22:19:16.827Z21/30 Flush
159592024-02-28T22:19:16.827Z22/30 Write block 85 len 28672 data: 13 12 12 13 13 12 13
159602024-02-28T22:19:16.827Z23/30 Flush
159612024-02-28T22:19:16.827Z24/30 Write block 492 len 24576 data: 14 13 13 14 14 13
159622024-02-28T22:19:16.827Z25/30 Read block 589 len 4096
159632024-02-28T22:19:16.827Z26/30 Flush
159642024-02-28T22:19:16.827Z27/30 Write block 386 len 16384 data: 18 15 13 12
159652024-02-28T22:19:16.827Z28/30 Read block 56 len 36864
159662024-02-28T22:19:16.827Z29/30 Read block 574 len 40960
159672024-02-28T22:19:16.827Z30/30 Write block 126 len 32768 data: 15 18 21 17 16 17 16 17
159682024-02-28T22:19:16.828Z----------------------------------------------------------------
159692024-02-28T22:19:16.828Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159702024-02-28T22:19:16.828ZStates: Active Active Active
159712024-02-28T22:19:16.828ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159722024-02-28T22:19:16.828Z 1 Acked 1000 Write 7 Done Done Done false
159732024-02-28T22:19:16.828Z 2 Acked 1001 Write 2 Done Done Done false
159742024-02-28T22:19:16.828Z 3 Acked 1002 Read 4 Done Done Done false
159752024-02-28T22:19:16.828Z 4 Acked 1003 Write 2 Done Done Sent false
159762024-02-28T22:19:16.840Z 5 Acked 1004 Read 2 Done Done Sent false
159772024-02-28T22:19:16.840Z 6 Acked 1005 Read 5 Done Done Sent false
159782024-02-28T22:19:16.840Z 7 Acked 1006 Read 2 Done Done Sent false
159792024-02-28T22:19:16.840Z 8 Acked 1007 Write 7 Done Done Sent false
159802024-02-28T22:19:16.840Z 9 Acked 1008 Read 6 Done Done Sent false
159812024-02-28T22:19:16.840Z 10 Acked 1009 Read 7 Done Done Sent false
159822024-02-28T22:19:16.840Z 11 Acked 1010 Read 2 Done Done Sent false
159832024-02-28T22:19:16.840Z 12 Acked 1011 Read 10 Done Done Sent false
159842024-02-28T22:19:16.840Z 13 Acked 1012 Read 8 Done Done Sent false
159852024-02-28T22:19:16.841Z 14 Acked 1013 Write 7 Done Done Sent false
159862024-02-28T22:19:16.841Z 15 Acked 1014 Write 10 Done Done Sent false
159872024-02-28T22:19:16.841Z 16 Acked 1015 Write 2 Done Done Sent false
159882024-02-28T22:19:16.841Z 17 Acked 1016 Flush 0 Done Done Sent false
159892024-02-28T22:19:16.841Z 18 Acked 1017 Write 7 Done Done Sent false
159902024-02-28T22:19:16.841Z 19 Acked 1018 Write 1 Done Done Sent false
159912024-02-28T22:19:16.841Z 20 Acked 1019 Write 8 Done Done Sent false
159922024-02-28T22:19:16.841Z 21 Acked 1020 Flush 0 Done Done Sent false
159932024-02-28T22:19:16.841Z 22 Acked 1021 Write 7 Done Done Sent false
159942024-02-28T22:19:16.841Z 23 Acked 1022 Flush 0 Done Done Sent false
159952024-02-28T22:19:16.841Z 24 Acked 1023 Write 6 Done Done Sent false
159962024-02-28T22:19:16.841Z 25 Acked 1024 Read 1 Done Done Sent false
159972024-02-28T22:19:16.842Z 26 Acked 1025 Flush 0 Done Done Sent false
159982024-02-28T22:19:16.842Z 27 Acked 1026 Write 4 Done Done Sent false
159992024-02-28T22:19:16.842Z 28 Acked 1027 Read 9 Done Done Sent false
160002024-02-28T22:19:16.842Z 29 Acked 1028 Read 10 Done Done Sent false
160012024-02-28T22:19:16.842Z 30 Acked 1029 Write 8 Sent Sent Sent false
160022024-02-28T22:19:16.842Z STATES DS:0 DS:1 DS:2 TOTAL
160032024-02-28T22:19:16.842Z New 0 0 0 0
160042024-02-28T22:19:16.842Z Sent 1 1 27 29
160052024-02-28T22:19:16.842Z Done 29 29 3 61
160062024-02-28T22:19:16.842Z Skipped 0 0 0 0
160072024-02-28T22:19:16.842Z Error 0 0 0 0
160082024-02-28T22:19:16.843ZLast Flush: 1025 1025 0
160092024-02-28T22:19:16.843ZDownstairs last five completed:
160102024-02-28T22:19:16.843ZUpstairs last five completed: 30 29 28 27 26
160112024-02-28T22:19:16.843Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:15.693673439Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1727}
160122024-02-28T22:19:16.843ZWrote out file "/tmp/test_repair_verify.data"
160132024-02-28T22:19:16.843Z
160142024-02-28T22:19:16.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160152024-02-28T22:19:16.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160162024-02-28T22:19:16.843Z 0 000-019 148 148 145 268 268 262 F F F <---
160172024-02-28T22:19:16.843Z 1 020-039 148 148 147 269 269 267 F F F <---
160182024-02-28T22:19:16.843Z 2 040-059 145 145 145 262 262 262 F F F
160192024-02-28T22:19:16.844Z 3 060-079 146 146 146 266 266 266 F F F
160202024-02-28T22:19:16.844Z 4 080-099 148 148 145 270 270 262 F F F <---
160212024-02-28T22:19:16.844Z 5 100-119 148 148 146 268 268 264 F F T <---
160222024-02-28T22:19:16.844Z 6 120-139 148 148 147 268 268 267 T T T <---
160232024-02-28T22:19:16.844Z 7 140-159 148 148 133 268 268 245 F F F <---
160242024-02-28T22:19:16.844Z 8 160-179 135 135 135 249 249 249 F F F
160252024-02-28T22:19:16.844Z 9 180-199 148 148 144 268 268 261 F F F <---
160262024-02-28T22:19:16.844Z 10 200-219 146 146 146 266 266 266 F F F
160272024-02-28T22:19:16.844Z 11 220-239 146 146 146 266 266 266 F F F
160282024-02-28T22:19:16.844Z 12 240-259 141 141 141 256 256 256 F F F
160292024-02-28T22:19:16.844Z 13 260-279 147 147 147 267 267 267 F F F
160302024-02-28T22:19:16.844Z 14 280-299 145 145 145 262 262 262 F F F
160312024-02-28T22:19:16.845Z 15 300-319 145 145 145 262 262 262 F F F
160322024-02-28T22:19:16.845Z 16 320-339 148 148 141 269 269 256 F F F <---
160332024-02-28T22:19:16.845Z 17 340-359 146 146 146 266 266 266 F F F
160342024-02-28T22:19:16.845Z 18 360-379 148 148 146 268 268 264 F F T <---
160352024-02-28T22:19:16.845Z 19 380-399 142 142 142 257 257 257 T T F <---
160362024-02-28T22:19:16.845Z 20 400-419 144 144 144 261 261 261 F F F
160372024-02-28T22:19:16.845Z 21 420-439 148 148 141 268 268 256 F F F <---
160382024-02-28T22:19:16.845Z 22 440-459 148 148 146 269 269 263 F F F <---
160392024-02-28T22:19:16.845Z 23 460-479 137 137 137 253 253 253 F F F
160402024-02-28T22:19:16.845Z 24 480-499 148 148 133 271 271 245 F F F <---
160412024-02-28T22:19:16.845Z 25 500-519 148 148 146 268 268 264 F F F <---
160422024-02-28T22:19:16.846Z 26 520-539 143 143 143 260 260 260 F F F
160432024-02-28T22:19:16.846Z 27 540-559 141 141 141 256 256 256 F F F
160442024-02-28T22:19:16.846Z 28 560-579 142 142 142 259 259 259 F F F
160452024-02-28T22:19:16.846Z 29 580-599 142 142 142 259 259 259 F F F
160462024-02-28T22:19:16.846ZMax gen: 148, Max flush: 271
160472024-02-28T22:19:16.846ZError: Difference in extent metadata found!
160482024-02-28T22:19:16.850ZA Difference in extent metadata is expected here
160492024-02-28T22:19:16.851ZOn loop 74
160502024-02-28T22:19:16.851Z
160512024-02-28T22:19:16.851ZVerifying data now
160522024-02-28T22:19:22.528ZNo change after verify
160532024-02-28T22:19:22.529ZLoop: 74 Downstairs dump after verify (and repair):
160542024-02-28T22:19:22.542ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160552024-02-28T22:19:22.543Z 0 000-019 148 148 148 268 268 268 F F F
160562024-02-28T22:19:22.543Z 1 020-039 148 148 148 269 269 269 F F F
160572024-02-28T22:19:22.543Z 2 040-059 145 145 145 262 262 262 F F F
160582024-02-28T22:19:22.543Z 3 060-079 146 146 146 266 266 266 F F F
160592024-02-28T22:19:22.543Z 4 080-099 148 148 148 270 270 270 F F F
160602024-02-28T22:19:22.543Z 5 100-119 148 148 148 268 268 268 F F F
160612024-02-28T22:19:22.543Z 6 120-139 149 149 149 272 272 272 F F F
160622024-02-28T22:19:22.543Z 7 140-159 148 148 148 268 268 268 F F F
160632024-02-28T22:19:22.543Z 8 160-179 135 135 135 249 249 249 F F F
160642024-02-28T22:19:22.543Z 9 180-199 148 148 148 268 268 268 F F F
160652024-02-28T22:19:22.543Z 10 200-219 146 146 146 266 266 266 F F F
160662024-02-28T22:19:22.544Z 11 220-239 146 146 146 266 266 266 F F F
160672024-02-28T22:19:22.544Z 12 240-259 141 141 141 256 256 256 F F F
160682024-02-28T22:19:22.544Z 13 260-279 147 147 147 267 267 267 F F F
160692024-02-28T22:19:22.544Z 14 280-299 145 145 145 262 262 262 F F F
160702024-02-28T22:19:22.547Z 15 300-319 145 145 145 262 262 262 F F F
160712024-02-28T22:19:22.548Z 16 320-339 148 148 148 269 269 269 F F F
160722024-02-28T22:19:22.548Z 17 340-359 146 146 146 266 266 266 F F F
160732024-02-28T22:19:22.548Z 18 360-379 148 148 148 268 268 268 F F F
160742024-02-28T22:19:22.548Z 19 380-399 149 149 149 272 272 272 F F F
160752024-02-28T22:19:22.548Z 20 400-419 144 144 144 261 261 261 F F F
160762024-02-28T22:19:22.548Z 21 420-439 148 148 148 268 268 268 F F F
160772024-02-28T22:19:22.548Z 22 440-459 148 148 148 269 269 269 F F F
160782024-02-28T22:19:22.548Z 23 460-479 137 137 137 253 253 253 F F F
160792024-02-28T22:19:22.548Z 24 480-499 148 148 148 271 271 271 F F F
160802024-02-28T22:19:22.548Z 25 500-519 148 148 148 268 268 268 F F F
160812024-02-28T22:19:22.548Z 26 520-539 143 143 143 260 260 260 F F F
160822024-02-28T22:19:22.549Z 27 540-559 141 141 141 256 256 256 F F F
160832024-02-28T22:19:22.549Z 28 560-579 142 142 142 259 259 259 F F F
160842024-02-28T22:19:22.549Z 29 580-599 142 142 142 259 259 259 F F F
160852024-02-28T22:19:22.549ZMax gen: 149, Max flush: 272
160862024-02-28T22:19:22.549Z
160872024-02-28T22:19:22.549ZBegin loop 75
160882024-02-28T22:19:22.549ZDownstairs to restart: 2
160892024-02-28T22:19:22.565Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.44741938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
160902024-02-28T22:19:22.565Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
160912024-02-28T22:19:22.565Z,"time":"2024-02-28T22:19:21.44777907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
160922024-02-28T22:19:22.574Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.447815539Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
160932024-02-28T22:19:22.574Z{"msg":"Crucible 0a2ff54e-e455-4986-b659-3f26ce8f3168 has session id: 819d83e2-6343-485f-bfb3-7b8262060e98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.447862327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
160942024-02-28T22:19:22.574Z{"msg":"Upstairs opts: Upstairs UUID: 0a2ff54e-e455-4986-b659-3f26ce8f3168, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.447920076Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
160952024-02-28T22:19:22.574Z{"msg":"Crucible stats registered with UUID: 0a2ff54e-e455-4986-b659-3f26ce8f3168","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.447963955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
160962024-02-28T22:19:22.574Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448050032Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
160972024-02-28T22:19:22.574Z{"msg":"0a2ff54e-e455-4986-b659-3f26ce8f3168 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448095271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
160982024-02-28T22:19:22.574Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.44812445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
160992024-02-28T22:19:22.575Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.44815681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161002024-02-28T22:19:22.575Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448196178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161012024-02-28T22:19:22.575Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448235517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161022024-02-28T22:19:22.575Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448277056Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161032024-02-28T22:19:22.575Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448309195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161042024-02-28T22:19:22.575Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448343714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161052024-02-28T22:19:22.575Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448394783Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161062024-02-28T22:19:22.576Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448421772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161072024-02-28T22:19:22.576Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448446911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161082024-02-28T22:19:22.576Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.448470621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161092024-02-28T22:19:22.576Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.44851353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161102024-02-28T22:19:22.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449134272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161112024-02-28T22:19:22.576Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449172902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161122024-02-28T22:19:22.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449189511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161132024-02-28T22:19:22.576Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449208141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161142024-02-28T22:19:22.576Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449252769Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161152024-02-28T22:19:22.576Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449274969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161162024-02-28T22:19:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449767755Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161172024-02-28T22:19:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.449819594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161182024-02-28T22:19:22.576Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450032258Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161192024-02-28T22:19:22.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450330019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161202024-02-28T22:19:22.576Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450368709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161212024-02-28T22:19:22.576Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450392698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161222024-02-28T22:19:22.576Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450407297Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161232024-02-28T22:19:22.576Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450430767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161242024-02-28T22:19:22.577Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450445556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161252024-02-28T22:19:22.577Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450516394Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161262024-02-28T22:19:22.577Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450546554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161272024-02-28T22:19:22.577Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 272, 268, 249, 268, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450562763Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161282024-02-28T22:19:22.577Z{"msg":"[0]R generation[0..12]: [148, 148, 145, 146, 148, 148, 149, 148, 135, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450585203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161292024-02-28T22:19:22.577Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450600422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161302024-02-28T22:19:22.577Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 272, 268, 249, 268, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450622131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161312024-02-28T22:19:22.577Z{"msg":"[1]R generation[0..12]: [148, 148, 145, 146, 148, 148, 149, 148, 135, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450636581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161322024-02-28T22:19:22.577Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450658081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161332024-02-28T22:19:22.577Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 272, 268, 249, 268, 266, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.45067239Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161342024-02-28T22:19:22.577ZWait for a query_work_queue command to finish before sending IO
161352024-02-28T22:19:22.577Z{"msg":"[2]R generation[0..12]: [148, 148, 145, 146, 148, 148, 149, 148, 135, 148, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.45069942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161362024-02-28T22:19:22.577Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450715709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161372024-02-28T22:19:22.577Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450739888Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161382024-02-28T22:19:22.577Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450753668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161392024-02-28T22:19:22.577Z{"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450778927Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161402024-02-28T22:19:22.577Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450792507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"":"downstairs","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161412024-02-28T22:19:22.577Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450818286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161422024-02-28T22:19:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450831656Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"0","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161432024-02-28T22:19:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450851865Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"1","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161442024-02-28T22:19:22.577Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450865505Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"client":"2","":"downstairs","session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161452024-02-28T22:19:22.577Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450894034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161462024-02-28T22:19:22.577Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450909604Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161472024-02-28T22:19:22.578Z{"msg":"0a2ff54e-e455-4986-b659-3f26ce8f3168 is now active with session: 819d83e2-6343-485f-bfb3-7b8262060e98","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450934413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161482024-02-28T22:19:22.578Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450948542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735,"session_id":"819d83e2-6343-485f-bfb3-7b8262060e98"}
161492024-02-28T22:19:22.578Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:21.450969572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
161502024-02-28T22:19:22.578ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161512024-02-28T22:19:22.578ZLoading write count information from file "/tmp/test_repair_verify.data"
161522024-02-28T22:19:22.578ZRun Repair workload
161532024-02-28T22:19:22.578Z01/30 Write block 323 len 32768 data: 10 13 13 15 15 17 17 17
161542024-02-28T22:19:22.584Z02/30 Read block 359 len 20480
161552024-02-28T22:19:22.584Z03/30 Write block 261 len 20480 data: 13 17 15 14 14
161562024-02-28T22:19:22.584Z04/30 Write block 212 len 24576 data: 12 11 10 8 8 9
161572024-02-28T22:19:22.584Z05/30 Read block 243 len 12288
161582024-02-28T22:19:22.585Z06/30 Read block 42 len 40960
161592024-02-28T22:19:22.585Z07/30 Write block 567 len 28672 data: 19 16 13 12 9 9 10
161602024-02-28T22:19:22.585Z08/30 Read block 218 len 4096
161612024-02-28T22:19:22.585Z09/30 Flush
161622024-02-28T22:19:22.585Z10/30 Write block 385 len 12288 data: 17 19 16
161632024-02-28T22:19:22.585Z11/30 Read block 96 len 36864
161642024-02-28T22:19:22.586Z12/30 Read block 141 len 32768
161652024-02-28T22:19:22.602Z13/30 Read block 49 len 20480
161662024-02-28T22:19:22.602Z14/30 Write block 349 len 28672 data: 9 10 11 12 13 13 14
161672024-02-28T22:19:22.602Z15/30 Read block 339 len 12288
161682024-02-28T22:19:22.603Z16/30 Write block 125 len 24576 data: 14 16 19 22 18 17
161692024-02-28T22:19:22.603Z17/30 Write block 247 len 24576 data: 15 17 13 13 13 11
161702024-02-28T22:19:22.603Z18/30 Read block 361 len 24576
161712024-02-28T22:19:22.603Z19/30 Read block 526 len 40960
161722024-02-28T22:19:22.603Z20/30 Write block 146 len 8192 data: 14 14
161732024-02-28T22:19:22.603Z21/30 Read block 273 len 8192
161742024-02-28T22:19:22.609Z22/30 Read block 283 len 24576
161752024-02-28T22:19:22.609Z23/30 Read block 162 len 32768
161762024-02-28T22:19:22.609Z24/30 Write block 144 len 12288 data: 13 14 15
161772024-02-28T22:19:22.609Z25/30 Write block 386 len 28672 data: 20 17 14 13 15 16 13
161782024-02-28T22:19:22.609Z26/30 Write block 328 len 8192 data: 18 18
161792024-02-28T22:19:22.610Z27/30 Write block 523 len 12288 data: 11 10 12
161802024-02-28T22:19:22.610Z28/30 Write block 148 len 20480 data: 15 13 11 9 9
161812024-02-28T22:19:22.610Z29/30 Read block 160 len 32768
161822024-02-28T22:19:22.610Z30/30 Write block 327 len 28672 data: 16 19 19 18 16 11 12
161832024-02-28T22:19:22.610Z----------------------------------------------------------------
161842024-02-28T22:19:22.610Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
161852024-02-28T22:19:22.610ZStates: Active Active Active
161862024-02-28T22:19:22.610ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161872024-02-28T22:19:22.611Z 1 Acked 1000 Write 8 Done Done Sent false
161882024-02-28T22:19:22.611Z 2 Acked 1001 Read 5 Done Done Sent false
161892024-02-28T22:19:22.611Z 3 Acked 1002 Write 5 Done Done Sent false
161902024-02-28T22:19:22.611Z 4 Acked 1003 Write 6 Done Done Sent false
161912024-02-28T22:19:22.611Z 5 Acked 1004 Read 3 Done Done Sent false
161922024-02-28T22:19:22.611Z 6 Acked 1005 Read 10 Done Done Sent false
161932024-02-28T22:19:22.611Z 7 Acked 1006 Write 7 Done Done Sent false
161942024-02-28T22:19:22.612Z 8 Acked 1007 Read 1 Done Done Sent false
161952024-02-28T22:19:22.612Z 9 Acked 1008 Flush 0 Done Done Sent false
161962024-02-28T22:19:22.612Z 10 Acked 1009 Write 3 Done Done Sent false
161972024-02-28T22:19:22.612Z 11 Acked 1010 Read 9 Done Done Sent false
161982024-02-28T22:19:22.612Z 12 Acked 1011 Read 8 Done Done Sent false
161992024-02-28T22:19:22.612Z 13 Acked 1012 Read 5 Done Done Sent false
162002024-02-28T22:19:22.613Z 14 Acked 1013 Write 7 Done Done Sent false
162012024-02-28T22:19:22.613Z 15 Acked 1014 Read 3 Done Done Sent false
162022024-02-28T22:19:22.613Z 16 Acked 1015 Write 6 Done Done Sent false
162032024-02-28T22:19:22.613Z 17 Acked 1016 Write 6 Done Done Sent false
162042024-02-28T22:19:22.613Z 18 Acked 1017 Read 6 Done Done Sent false
162052024-02-28T22:19:22.613Z 19 Acked 1018 Read 10 Done Done Sent false
162062024-02-28T22:19:22.613Z 20 Acked 1019 Write 2 Done Done Sent false
162072024-02-28T22:19:22.613Z 21 Acked 1020 Read 2 Done Done Sent false
162082024-02-28T22:19:22.613Z 22 Acked 1021 Read 6 Done Done Sent false
162092024-02-28T22:19:22.614Z 23 Acked 1022 Read 8 Done Done Sent false
162102024-02-28T22:19:22.614Z 24 Acked 1023 Write 3 Done Done Sent false
162112024-02-28T22:19:22.614Z 25 Acked 1024 Write 7 Done Done Sent false
162122024-02-28T22:19:22.614Z 26 Acked 1025 Write 2 Done Done Sent false
162132024-02-28T22:19:22.614Z 27 Acked 1026 Write 3 Done Done Sent false
162142024-02-28T22:19:22.614Z 28 Acked 1027 Write 5 Done Done Sent false
162152024-02-28T22:19:22.614Z 29 Acked 1028 Read 8 Done Done Sent false
162162024-02-28T22:19:22.614Z 30 Acked 1029 Write 7 Sent Sent Sent false
162172024-02-28T22:19:22.614Z STATES DS:0 DS:1 DS:2 TOTAL
162182024-02-28T22:19:22.614Z New 0 0 0 0
162192024-02-28T22:19:22.615Z Sent 1 1 30 32
162202024-02-28T22:19:22.615Z Done 29 29 0 58
162212024-02-28T22:19:22.615Z Skipped 0 0 0 0
162222024-02-28T22:19:22.615Z Error 0 0 0 0
162232024-02-28T22:19:22.615ZLast Flush: 1008 1008 0
162242024-02-28T22:19:22.615ZDownstairs last five completed:
162252024-02-28T22:19:22.615ZUpstairs last five completed: 30 29 28 27 26
162262024-02-28T22:19:22.615Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:21.479596163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1735}
162272024-02-28T22:19:22.616ZWrote out file "/tmp/test_repair_verify.data"
162282024-02-28T22:19:22.616Z
162292024-02-28T22:19:22.616ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162302024-02-28T22:19:22.622ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162312024-02-28T22:19:22.622Z 0 000-019 148 148 148 268 268 268 F F F
162322024-02-28T22:19:22.622Z 1 020-039 148 148 148 269 269 269 F F F
162332024-02-28T22:19:22.622Z 2 040-059 145 145 145 262 262 262 F F F
162342024-02-28T22:19:22.622Z 3 060-079 146 146 146 266 266 266 F F F
162352024-02-28T22:19:22.622Z 4 080-099 148 148 148 270 270 270 F F F
162362024-02-28T22:19:22.622Z 5 100-119 148 148 148 268 268 268 F F F
162372024-02-28T22:19:22.626Z 6 120-139 149 149 149 272 272 272 T T F <---
162382024-02-28T22:19:22.626Z 7 140-159 148 148 148 268 268 268 T T F <---
162392024-02-28T22:19:22.627Z 8 160-179 135 135 135 249 249 249 F F F
162402024-02-28T22:19:22.627Z 9 180-199 148 148 148 268 268 268 F F F
162412024-02-28T22:19:22.627Z 10 200-219 150 150 146 273 273 266 F F F <---
162422024-02-28T22:19:22.627Z 11 220-239 146 146 146 266 266 266 F F F
162432024-02-28T22:19:22.627Z 12 240-259 141 141 141 256 256 256 T T F <---
162442024-02-28T22:19:22.628Z 13 260-279 150 150 147 273 273 267 F F F <---
162452024-02-28T22:19:22.628Z 14 280-299 145 145 145 262 262 262 F F F
162462024-02-28T22:19:22.628Z 15 300-319 145 145 145 262 262 262 F F F
162472024-02-28T22:19:22.628Z 16 320-339 150 150 148 273 273 269 T T F <---
162482024-02-28T22:19:22.628Z 17 340-359 146 146 146 266 266 266 T T F <---
162492024-02-28T22:19:22.628Z 18 360-379 148 148 148 268 268 268 F F F
162502024-02-28T22:19:22.628Z 19 380-399 149 149 149 272 272 272 T T F <---
162512024-02-28T22:19:22.628Z 20 400-419 144 144 144 261 261 261 F F F
162522024-02-28T22:19:22.629Z 21 420-439 148 148 148 268 268 268 F F F
162532024-02-28T22:19:22.629Z 22 440-459 148 148 148 269 269 269 F F F
162542024-02-28T22:19:22.629Z 23 460-479 137 137 137 253 253 253 F F F
162552024-02-28T22:19:22.629Z 24 480-499 148 148 148 271 271 271 F F F
162562024-02-28T22:19:22.629Z 25 500-519 148 148 148 268 268 268 F F F
162572024-02-28T22:19:22.629Z 26 520-539 143 143 143 260 260 260 T T F <---
162582024-02-28T22:19:22.629Z 27 540-559 141 141 141 256 256 256 F F F
162592024-02-28T22:19:22.630Z 28 560-579 150 150 142 273 273 259 F F F <---
162602024-02-28T22:19:22.630Z 29 580-599 142 142 142 259 259 259 F F F
162612024-02-28T22:19:22.630ZMax gen: 150, Max flush: 273
162622024-02-28T22:19:22.630ZError: Difference in extent metadata found!
162632024-02-28T22:19:22.630ZA Difference in extent metadata is expected here
162642024-02-28T22:19:22.630ZOn loop 75
162652024-02-28T22:19:22.630Z
162662024-02-28T22:19:22.635ZVerifying data now
162672024-02-28T22:19:23.161ZNo change after verify
162682024-02-28T22:19:23.161ZLoop: 75 Downstairs dump after verify (and repair):
162692024-02-28T22:19:23.176ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162702024-02-28T22:19:23.176Z 0 000-019 148 148 148 268 268 268 F F F
162712024-02-28T22:19:23.176Z 1 020-039 148 148 148 269 269 269 F F F
162722024-02-28T22:19:23.176Z 2 040-059 145 145 145 262 262 262 F F F
162732024-02-28T22:19:23.176Z 3 060-079 146 146 146 266 266 266 F F F
162742024-02-28T22:19:23.176Z 4 080-099 148 148 148 270 270 270 F F F
162752024-02-28T22:19:23.176Z 5 100-119 148 148 148 268 268 268 F F F
162762024-02-28T22:19:23.176Z 6 120-139 151 151 151 274 274 274 F F F
162772024-02-28T22:19:23.177Z 7 140-159 151 151 151 274 274 274 F F F
162782024-02-28T22:19:23.177Z 8 160-179 135 135 135 249 249 249 F F F
162792024-02-28T22:19:23.180Z 9 180-199 148 148 148 268 268 268 F F F
162802024-02-28T22:19:23.180Z 10 200-219 150 150 150 273 273 273 F F F
162812024-02-28T22:19:23.181Z 11 220-239 146 146 146 266 266 266 F F F
162822024-02-28T22:19:23.181Z 12 240-259 151 151 151 274 274 274 F F F
162832024-02-28T22:19:23.181Z 13 260-279 150 150 150 273 273 273 F F F
162842024-02-28T22:19:23.181Z 14 280-299 145 145 145 262 262 262 F F F
162852024-02-28T22:19:23.181Z 15 300-319 145 145 145 262 262 262 F F F
162862024-02-28T22:19:23.181Z 16 320-339 151 151 151 274 274 274 F F F
162872024-02-28T22:19:23.181Z 17 340-359 151 151 151 274 274 274 F F F
162882024-02-28T22:19:23.181Z 18 360-379 148 148 148 268 268 268 F F F
162892024-02-28T22:19:23.181Z 19 380-399 151 151 151 274 274 274 F F F
162902024-02-28T22:19:23.182Z 20 400-419 144 144 144 261 261 261 F F F
162912024-02-28T22:19:23.182Z 21 420-439 148 148 148 268 268 268 F F F
162922024-02-28T22:19:23.182Z 22 440-459 148 148 148 269 269 269 F F F
162932024-02-28T22:19:23.182Z 23 460-479 137 137 137 253 253 253 F F F
162942024-02-28T22:19:23.182Z 24 480-499 148 148 148 271 271 271 F F F
162952024-02-28T22:19:23.182Z 25 500-519 148 148 148 268 268 268 F F F
162962024-02-28T22:19:23.182Z 26 520-539 151 151 151 274 274 274 F F F
162972024-02-28T22:19:23.182Z 27 540-559 141 141 141 256 256 256 F F F
162982024-02-28T22:19:23.182Z 28 560-579 150 150 150 273 273 273 F F F
162992024-02-28T22:19:23.182Z 29 580-599 142 142 142 259 259 259 F F F
163002024-02-28T22:19:23.183ZMax gen: 151, Max flush: 274
163012024-02-28T22:19:23.183Z
163022024-02-28T22:19:23.183ZBegin loop 76
163032024-02-28T22:19:23.183ZDownstairs to restart: 1
163042024-02-28T22:19:23.199Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.081743384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743}
163052024-02-28T22:19:23.206Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082050666Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743Crucible runtime is spawned
163062024-02-28T22:19:23.206Z}
163072024-02-28T22:19:23.207Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082108684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743}
163082024-02-28T22:19:23.207Z{"msg":"Crucible f2b9462e-df8e-4bb7-b4af-106795156f52 has session id: 33439e84-7c9d-4ce9-8e15-ac2af9b9110e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082139403Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163092024-02-28T22:19:23.207Z{"msg":"Upstairs opts: Upstairs UUID: f2b9462e-df8e-4bb7-b4af-106795156f52, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082195432Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163102024-02-28T22:19:23.207Z{"msg":"Crucible stats registered with UUID: f2b9462e-df8e-4bb7-b4af-106795156f52","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08224113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163112024-02-28T22:19:23.207Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082285869Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743}
163122024-02-28T22:19:23.207Z{"msg":"f2b9462e-df8e-4bb7-b4af-106795156f52 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082324848Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163132024-02-28T22:19:23.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082369597Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163142024-02-28T22:19:23.207Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082416476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163152024-02-28T22:19:23.207Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082456275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163162024-02-28T22:19:23.208Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082499674Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163172024-02-28T22:19:23.208Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082534993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163182024-02-28T22:19:23.208Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08260948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163192024-02-28T22:19:23.208Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082658139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163202024-02-28T22:19:23.208Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082697778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163212024-02-28T22:19:23.208Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082722407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163222024-02-28T22:19:23.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082756866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163232024-02-28T22:19:23.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082782806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163242024-02-28T22:19:23.208Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.082831414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163252024-02-28T22:19:23.208Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08335338Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163262024-02-28T22:19:23.208Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.083389729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163272024-02-28T22:19:23.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.083406618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163282024-02-28T22:19:23.209Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.083421238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163292024-02-28T22:19:23.209Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.083444717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163302024-02-28T22:19:23.209Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.083464937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163312024-02-28T22:19:23.209Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084004172Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163322024-02-28T22:19:23.209Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084061231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163332024-02-28T22:19:23.209Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08409253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163342024-02-28T22:19:23.209Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084489539Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163352024-02-28T22:19:23.209Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084536377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163362024-02-28T22:19:23.209Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084564606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163372024-02-28T22:19:23.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084596736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163382024-02-28T22:19:23.210Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084621325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163392024-02-28T22:19:23.210Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084654784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163402024-02-28T22:19:23.210Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084678534Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163412024-02-28T22:19:23.210Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084703143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163422024-02-28T22:19:23.210Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 274, 274, 249, 268, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084732372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163432024-02-28T22:19:23.210Z{"msg":"[0]R generation[0..12]: [148, 148, 145, 146, 148, 148, 151, 151, 135, 148, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084756471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163442024-02-28T22:19:23.210Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084771551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163452024-02-28T22:19:23.210Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 274, 274, 249, 268, 273, 266]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08479426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163462024-02-28T22:19:23.210Z{"msg":"[1]R generation[0..12]: [148, 148, 145, 146, 148, 148, 151, 151, 135, 148, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.0848097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163472024-02-28T22:19:23.210Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084831979Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163482024-02-28T22:19:23.210Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 262, 266, 270, 268, 274, 274, 249, 268, 273, 266]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
163492024-02-28T22:19:23.210Z,"time":"2024-02-28T22:19:22.084868268Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163502024-02-28T22:19:23.211Z{"msg":"[2]R generation[0..12]: [148, 148, 145, 146, 148, 148, 151, 151, 135, 148, 150, 146]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084901097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163512024-02-28T22:19:23.211Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084929047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163522024-02-28T22:19:23.211Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084962436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163532024-02-28T22:19:23.211Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.084996605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163542024-02-28T22:19:23.211Z{"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085012084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163552024-02-28T22:19:23.211Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085044553Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"":"downstairs","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163562024-02-28T22:19:23.211Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085087782Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163572024-02-28T22:19:23.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085106572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"0","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163582024-02-28T22:19:23.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085129711Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"1","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163592024-02-28T22:19:23.211Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.08515327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"client":"2","":"downstairs","session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163602024-02-28T22:19:23.211Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085189939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163612024-02-28T22:19:23.211Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085216139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163622024-02-28T22:19:23.212Z{"msg":"f2b9462e-df8e-4bb7-b4af-106795156f52 is now active with session: 33439e84-7c9d-4ce9-8e15-ac2af9b9110e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085234178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163632024-02-28T22:19:23.212Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085267377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743,"session_id":"33439e84-7c9d-4ce9-8e15-ac2af9b9110e"}
163642024-02-28T22:19:23.212Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:22.085297796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743}
163652024-02-28T22:19:23.212ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163662024-02-28T22:19:23.217ZLoading write count information from file "/tmp/test_repair_verify.data"
163672024-02-28T22:19:23.217ZRun Repair workload
163682024-02-28T22:19:23.217Z01/30 Write block 66 len 16384 data: 13 14 14 10
163692024-02-28T22:19:23.217Z02/30 Read block 443 len 32768
163702024-02-28T22:19:23.217Z03/30 Write block 57 len 24576 data: 11 10 13 14 13 14
163712024-02-28T22:19:23.217Z04/30 Write block 154 len 4096 data: 6
163722024-02-28T22:19:23.217Z05/30 Write block 391 len 32768 data: 17 14 10 9 11 9 10 9
163732024-02-28T22:19:23.217Z06/30 Write block 449 len 40960 data: 14 15 16 15 17 15 15 14 12 12
163742024-02-28T22:19:23.217Z07/30 Write block 398 len 20480 data: 10 8 10 10 10
163752024-02-28T22:19:23.217Z08/30 Read block 105 len 32768
163762024-02-28T22:19:23.218Z09/30 Write block 496 len 12288 data: 15 14 11
163772024-02-28T22:19:23.218Z10/30 Write block 419 len 12288 data: 13 15 16
163782024-02-28T22:19:23.218Z11/30 Read block 100 len 28672
163792024-02-28T22:19:23.218Z12/30 Read block 325 len 28672
163802024-02-28T22:19:23.218Z13/30 Write block 263 len 16384 data: 16 15 15 16
163812024-02-28T22:19:23.222Z14/30 Write block 398 len 4096 data: 11
163822024-02-28T22:19:23.222Z15/30 Read block 229 len 36864
163832024-02-28T22:19:23.222Z16/30 Flush
163842024-02-28T22:19:23.226Z17/30 Write block 111 len 32768 data: 16 16 14 15 14 14 13 11
163852024-02-28T22:19:23.226Z18/30 Write block 216 len 40960 data: 9 10 8 8 9 10 8 7 8 11
163862024-02-28T22:19:23.230Z19/30 Flush
163872024-02-28T22:19:23.230Z20/30 Write block 163 len 8192 data: 9 11
163882024-02-28T22:19:23.234Z21/30 Flush
163892024-02-28T22:19:23.234Z22/30 Write block 395 len 4096 data: 12
163902024-02-28T22:19:23.234Z23/30 Read block 205 len 16384
163912024-02-28T22:19:23.234Z24/30 Read block 170 len 36864
163922024-02-28T22:19:23.238Z25/30 Read block 327 len 36864
163932024-02-28T22:19:23.238Z26/30 Read block 182 len 28672
163942024-02-28T22:19:23.238Z27/30 Write block 264 len 16384 data: 16 16 17 13
163952024-02-28T22:19:23.238Z28/30 Read block 86 len 16384
163962024-02-28T22:19:23.242Z29/30 Write block 407 len 28672 data: 15 13 11 10 9 10 11
163972024-02-28T22:19:23.243Z30/30 Write block 434 len 20480 data: 14 15 17 16 17
163982024-02-28T22:19:23.243Z----------------------------------------------------------------
163992024-02-28T22:19:23.243Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
164002024-02-28T22:19:23.243ZStates: Active Active Active
164012024-02-28T22:19:23.243ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164022024-02-28T22:19:23.243Z 1 Acked 1000 Write 4 Done Done Done false
164032024-02-28T22:19:23.243Z 2 Acked 1001 Read 8 Done Done Done false
164042024-02-28T22:19:23.243Z 3 Acked 1002 Write 6 Done Done Done false
164052024-02-28T22:19:23.243Z 4 Acked 1003 Write 1 Done Done Done false
164062024-02-28T22:19:23.244Z 5 Acked 1004 Write 8 Done Done Done false
164072024-02-28T22:19:23.244Z 6 Acked 1005 Write 10 Done Done Done false
164082024-02-28T22:19:23.244Z 7 Acked 1006 Write 5 Done Done Done false
164092024-02-28T22:19:23.244Z 8 Acked 1007 Read 8 Done Done Done false
164102024-02-28T22:19:23.244Z 9 Acked 1008 Write 3 Done Sent Done false
164112024-02-28T22:19:23.244Z 10 Acked 1009 Write 3 Done Sent Done false
164122024-02-28T22:19:23.244Z 11 Acked 1010 Read 7 Done Sent Done false
164132024-02-28T22:19:23.244Z 12 Acked 1011 Read 7 Done Sent Done false
164142024-02-28T22:19:23.244Z 13 Acked 1012 Write 4 Done Sent Done false
164152024-02-28T22:19:23.244Z 14 Acked 1013 Write 1 Done Sent Done false
164162024-02-28T22:19:23.245Z 15 Acked 1014 Read 9 Done Sent Done false
164172024-02-28T22:19:23.245Z 16 Acked 1015 Flush 0 Done Sent Done false
164182024-02-28T22:19:23.245Z 17 Acked 1016 Write 8 Done Sent Done false
164192024-02-28T22:19:23.245Z 18 Acked 1017 Write 10 Done Sent Done false
164202024-02-28T22:19:23.245Z 19 Acked 1018 Flush 0 Done Sent Done false
164212024-02-28T22:19:23.245Z 20 Acked 1019 Write 2 Done Sent Done false
164222024-02-28T22:19:23.245Z 21 Acked 1020 Flush 0 Done Sent Done false
164232024-02-28T22:19:23.245Z 22 Acked 1021 Write 1 Done Sent Done false
164242024-02-28T22:19:23.245Z 23 Acked 1022 Read 4 Done Sent Done false
164252024-02-28T22:19:23.246Z 24 Acked 1023 Read 9 Done Sent Done false
164262024-02-28T22:19:23.246Z 25 Acked 1024 Read 9 Done Sent Done false
164272024-02-28T22:19:23.246Z 26 Acked 1025 Read 7 Done Sent Done false
164282024-02-28T22:19:23.246Z 27 Acked 1026 Write 4 Done Sent Done false
164292024-02-28T22:19:23.246Z 28 Acked 1027 Read 4 Done Sent Done false
164302024-02-28T22:19:23.246Z 29 Acked 1028 Write 7 Done Sent Done false
164312024-02-28T22:19:23.246Z 30 Acked 1029 Write 5 Sent Sent Sent false
164322024-02-28T22:19:23.246Z STATES DS:0 DS:1 DS:2 TOTAL
164332024-02-28T22:19:23.246Z New 0 0 0 0
164342024-02-28T22:19:23.247Z Sent 1 22 1 24
164352024-02-28T22:19:23.247Z Done 29 8 29 66
164362024-02-28T22:19:23.247Z Skipped 0 0 0 0
164372024-02-28T22:19:23.247Z Error 0 0 0 0
164382024-02-28T22:19:23.247ZLast Flush: 1020 0 1020
164392024-02-28T22:19:23.247ZDownstairs last five completed:
164402024-02-28T22:19:23.247ZUpstairs last five completed: 30 29 28 27 26
164412024-02-28T22:19:23.247Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:22.122768274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1743}
164422024-02-28T22:19:23.248ZWrote out file "/tmp/test_repair_verify.data"
164432024-02-28T22:19:23.252Z
164442024-02-28T22:19:23.252ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164452024-02-28T22:19:23.264ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164462024-02-28T22:19:23.264Z 0 000-019 148 148 148 268 268 268 F F F
164472024-02-28T22:19:23.264Z 1 020-039 148 148 148 269 269 269 F F F
164482024-02-28T22:19:23.264Z 2 040-059 152 145 152 275 262 275 F T F <---
164492024-02-28T22:19:23.264Z 3 060-079 152 146 152 275 266 275 F T F <---
164502024-02-28T22:19:23.264Z 4 080-099 148 148 148 270 270 270 F F F
164512024-02-28T22:19:23.264Z 5 100-119 152 148 152 276 268 276 F F F <---
164522024-02-28T22:19:23.264Z 6 120-139 151 151 151 274 274 274 F F F
164532024-02-28T22:19:23.264Z 7 140-159 152 151 152 275 274 275 F T F <---
164542024-02-28T22:19:23.268Z 8 160-179 152 135 152 277 249 277 F F F <---
164552024-02-28T22:19:23.268Z 9 180-199 148 148 148 268 268 268 F F F
164562024-02-28T22:19:23.268Z 10 200-219 152 150 152 276 273 276 F F F <---
164572024-02-28T22:19:23.268Z 11 220-239 152 146 152 276 266 276 F F F <---
164582024-02-28T22:19:23.269Z 12 240-259 151 151 151 274 274 274 F F F
164592024-02-28T22:19:23.269Z 13 260-279 152 150 152 275 273 275 T F T <---
164602024-02-28T22:19:23.269Z 14 280-299 145 145 145 262 262 262 F F F
164612024-02-28T22:19:23.269Z 15 300-319 145 145 145 262 262 262 F F F
164622024-02-28T22:19:23.269Z 16 320-339 151 151 151 274 274 274 F F F
164632024-02-28T22:19:23.269Z 17 340-359 151 151 151 274 274 274 F F F
164642024-02-28T22:19:23.269Z 18 360-379 148 148 148 268 268 268 F F F
164652024-02-28T22:19:23.269Z 19 380-399 152 151 152 275 274 275 T T T <---
164662024-02-28T22:19:23.269Z 20 400-419 152 144 152 275 261 275 T T T <---
164672024-02-28T22:19:23.270Z 21 420-439 152 148 152 275 268 275 T F T <---
164682024-02-28T22:19:23.270Z 22 440-459 152 148 152 275 269 275 F T F <---
164692024-02-28T22:19:23.270Z 23 460-479 137 137 137 253 253 253 F F F
164702024-02-28T22:19:23.270Z 24 480-499 152 148 152 275 271 275 F F F <---
164712024-02-28T22:19:23.270Z 25 500-519 148 148 148 268 268 268 F F F
164722024-02-28T22:19:23.270Z 26 520-539 151 151 151 274 274 274 F F F
164732024-02-28T22:19:23.270Z 27 540-559 141 141 141 256 256 256 F F F
164742024-02-28T22:19:23.270Z 28 560-579 150 150 150 273 273 273 F F F
164752024-02-28T22:19:23.270Z 29 580-599 142 142 142 259 259 259 F F F
164762024-02-28T22:19:23.271ZMax gen: 152, Max flush: 277
164772024-02-28T22:19:23.271ZError: Difference in extent metadata found!
164782024-02-28T22:19:23.271ZA Difference in extent metadata is expected here
164792024-02-28T22:19:23.271ZOn loop 76
164802024-02-28T22:19:23.271Z
164812024-02-28T22:19:23.275ZVerifying data now
164822024-02-28T22:19:28.941ZNo change after verify
164832024-02-28T22:19:28.941ZLoop: 76 Downstairs dump after verify (and repair):
164842024-02-28T22:19:28.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164852024-02-28T22:19:28.952Z 0 000-019 148 148 148 268 268 268 F F F
164862024-02-28T22:19:28.952Z 1 020-039 148 148 148 269 269 269 F F F
164872024-02-28T22:19:28.952Z 2 040-059 152 152 152 275 275 275 F F F
164882024-02-28T22:19:28.952Z 3 060-079 152 152 152 275 275 275 F F F
164892024-02-28T22:19:28.952Z 4 080-099 148 148 148 270 270 270 F F F
164902024-02-28T22:19:28.952Z 5 100-119 152 152 152 276 276 276 F F F
164912024-02-28T22:19:28.953Z 6 120-139 151 151 151 274 274 274 F F F
164922024-02-28T22:19:28.953Z 7 140-159 152 152 152 275 275 275 F F F
164932024-02-28T22:19:28.953Z 8 160-179 152 152 152 277 277 277 F F F
164942024-02-28T22:19:28.958Z 9 180-199 148 148 148 268 268 268 F F F
164952024-02-28T22:19:28.958Z 10 200-219 152 152 152 276 276 276 F F F
164962024-02-28T22:19:28.958Z 11 220-239 152 152 152 276 276 276 F F F
164972024-02-28T22:19:28.958Z 12 240-259 151 151 151 274 274 274 F F F
164982024-02-28T22:19:28.958Z 13 260-279 153 153 153 278 278 278 F F F
164992024-02-28T22:19:28.958Z 14 280-299 145 145 145 262 262 262 F F F
165002024-02-28T22:19:28.958Z 15 300-319 145 145 145 262 262 262 F F F
165012024-02-28T22:19:28.959Z 16 320-339 151 151 151 274 274 274 F F F
165022024-02-28T22:19:28.959Z 17 340-359 151 151 151 274 274 274 F F F
165032024-02-28T22:19:28.959Z 18 360-379 148 148 148 268 268 268 F F F
165042024-02-28T22:19:28.959Z 19 380-399 153 153 153 278 278 278 F F F
165052024-02-28T22:19:28.959Z 20 400-419 153 153 153 278 278 278 F F F
165062024-02-28T22:19:28.959Z 21 420-439 153 153 153 278 278 278 F F F
165072024-02-28T22:19:28.959Z 22 440-459 152 152 152 275 275 275 F F F
165082024-02-28T22:19:28.959Z 23 460-479 137 137 137 253 253 253 F F F
165092024-02-28T22:19:28.959Z 24 480-499 152 152 152 275 275 275 F F F
165102024-02-28T22:19:28.960Z 25 500-519 148 148 148 268 268 268 F F F
165112024-02-28T22:19:28.960Z 26 520-539 151 151 151 274 274 274 F F F
165122024-02-28T22:19:28.960Z 27 540-559 141 141 141 256 256 256 F F F
165132024-02-28T22:19:28.960Z 28 560-579 150 150 150 273 273 273 F F F
165142024-02-28T22:19:28.960Z 29 580-599 142 142 142 259 259 259 F F F
165152024-02-28T22:19:28.960ZMax gen: 153, Max flush: 278
165162024-02-28T22:19:28.960Z
165172024-02-28T22:19:28.960ZBegin loop 77
165182024-02-28T22:19:28.960ZDownstairs to restart: 1
165192024-02-28T22:19:28.968Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850403563Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
165202024-02-28T22:19:28.968Z{"msg":"Crucible Version: BuildInfo {\nCrucible runtime is spawned
165212024-02-28T22:19:29.011Z version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850762364Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
165222024-02-28T22:19:29.012Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850810712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
165232024-02-28T22:19:29.012Z{"msg":"Crucible 3c817a30-16ba-40ce-a8d2-371f1931bfe1 has session id: 177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850840342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165242024-02-28T22:19:29.012Z{"msg":"Upstairs opts: Upstairs UUID: 3c817a30-16ba-40ce-a8d2-371f1931bfe1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.85090367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165252024-02-28T22:19:29.012Z{"msg":"Crucible stats registered with UUID: 3c817a30-16ba-40ce-a8d2-371f1931bfe1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850958608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165262024-02-28T22:19:29.012Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.850989147Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
165272024-02-28T22:19:29.012Z{"msg":"3c817a30-16ba-40ce-a8d2-371f1931bfe1 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851015517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165282024-02-28T22:19:29.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851063305Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165292024-02-28T22:19:29.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851097414Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165302024-02-28T22:19:29.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851142653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165312024-02-28T22:19:29.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851189472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165322024-02-28T22:19:29.012Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.85125488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165332024-02-28T22:19:29.012Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851305089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165342024-02-28T22:19:29.012Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851332068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165352024-02-28T22:19:29.013Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851365307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165362024-02-28T22:19:29.013Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851398796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165372024-02-28T22:19:29.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851435735Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165382024-02-28T22:19:29.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851473784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165392024-02-28T22:19:29.013Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.851502903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165402024-02-28T22:19:29.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852099837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165412024-02-28T22:19:29.013Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852137346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165422024-02-28T22:19:29.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852153195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165432024-02-28T22:19:29.013Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852167935Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165442024-02-28T22:19:29.013Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852189504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165452024-02-28T22:19:29.013Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852204774Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165462024-02-28T22:19:29.013Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852774278Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165472024-02-28T22:19:29.013Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852854006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165482024-02-28T22:19:29.013Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.852879665Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165492024-02-28T22:19:29.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853311543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165502024-02-28T22:19:29.014Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853350842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165512024-02-28T22:19:29.014Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853366152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165522024-02-28T22:19:29.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853461739Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165532024-02-28T22:19:29.014Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853489598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165542024-02-28T22:19:29.014Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853505228Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165552024-02-28T22:19:29.014Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853518048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165562024-02-28T22:19:29.014Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853531617Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165572024-02-28T22:19:29.014Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 275, 275, 270, 276, 274, 275, 277, 268, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853554377Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165582024-02-28T22:19:29.014Z{"msg":"[0]R generation[0..12]: [148, 148, 152, 152, 148, 152, 151, 152, 152, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853569766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165592024-02-28T22:19:29.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853584426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165602024-02-28T22:19:29.014Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 275, 275, 270, 276, 274, 275, 277, 268, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853606055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165612024-02-28T22:19:29.015Z{"msg":"[1]R generation[0..12]: [148, 148, 152, 152, 148, 152, 151, 152, 152, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853620925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165622024-02-28T22:19:29.015Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853635234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165632024-02-28T22:19:29.015Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 275, 275, 270, 276, 274, 275, 277, 268, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853656494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165642024-02-28T22:19:29.015Z{"msg":"[2]R generation[0..12]: [148, 148, 152, 152, 148, 152, 151, 152, 152, 148, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853670884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165652024-02-28T22:19:29.015Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853685723Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165662024-02-28T22:19:29.015Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853706772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165672024-02-28T22:19:29.015Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853722012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165682024-02-28T22:19:29.015Z{"msg":"Next flush: 279","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853735272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165692024-02-28T22:19:29.016Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853748591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"":"downstairs","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165702024-02-28T22:19:29.016Z{"msg":"Wait for a query_work_queue command to finish before sending IO
165712024-02-28T22:19:29.016ZNo downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853773931Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165722024-02-28T22:19:29.016Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.85378653Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"0","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165732024-02-28T22:19:29.016Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.85379995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"1","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165742024-02-28T22:19:29.016Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853813189Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"client":"2","":"downstairs","session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165752024-02-28T22:19:29.016Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853833599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165762024-02-28T22:19:29.016Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853846199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165772024-02-28T22:19:29.017Z{"msg":"3c817a30-16ba-40ce-a8d2-371f1931bfe1 is now active with session: 177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853859248Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165782024-02-28T22:19:29.017Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853871298Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751,"session_id":"177d60d1-dcda-4f2d-bca4-7f8fd3e2e6fb"}
165792024-02-28T22:19:29.017Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:27.853883648Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
165802024-02-28T22:19:29.017ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165812024-02-28T22:19:29.017ZLoading write count information from file "/tmp/test_repair_verify.data"
165822024-02-28T22:19:29.017ZRun Repair workload
165832024-02-28T22:19:29.017Z01/30 Write block 544 len 16384 data: 11 12 12 11
165842024-02-28T22:19:29.017Z02/30 Read block 65 len 16384
165852024-02-28T22:19:29.017Z03/30 Write block 446 len 36864 data: 14 14 14 15 16 17 16 18 16
165862024-02-28T22:19:29.017Z04/30 Read block 311 len 32768
165872024-02-28T22:19:29.017Z05/30 Write block 431 len 40960 data: 8 11 12 15 16 18 17 18 17 16
165882024-02-28T22:19:29.017Z06/30 Read block 379 len 4096
165892024-02-28T22:19:29.017Z07/30 Read block 387 len 8192
165902024-02-28T22:19:29.017Z08/30 Write block 67 len 36864 data: 15 15 11 12 10 13 12 12 9
165912024-02-28T22:19:29.017Z09/30 Read block 277 len 8192
165922024-02-28T22:19:29.017Z10/30 Read block 238 len 28672
165932024-02-28T22:19:29.017Z11/30 Read block 21 len 32768
165942024-02-28T22:19:29.017Z12/30 Flush
165952024-02-28T22:19:29.017Z13/30 Write block 253 len 36864 data: 12 11 10 13 15 15 14 13 14
165962024-02-28T22:19:29.017Z14/30 Flush
165972024-02-28T22:19:29.017Z15/30 Flush
165982024-02-28T22:19:29.018Z16/30 Write block 178 len 4096 data: 8
165992024-02-28T22:19:29.018Z17/30 Read block 515 len 32768
166002024-02-28T22:19:29.018Z18/30 Read block 555 len 24576
166012024-02-28T22:19:29.018Z19/30 Read block 368 len 32768
166022024-02-28T22:19:29.018Z20/30 Read block 401 len 12288
166032024-02-28T22:19:29.018Z21/30 Write block 159 len 4096 data: 8
166042024-02-28T22:19:29.018Z22/30 Read block 185 len 40960
166052024-02-28T22:19:29.018Z23/30 Write block 319 len 8192 data: 8 8
166062024-02-28T22:19:29.018Z24/30 Read block 269 len 4096
166072024-02-28T22:19:29.018Z25/30 Write block 191 len 4096 data: 10
166082024-02-28T22:19:29.018Z26/30 Read block 106 len 28672
166092024-02-28T22:19:29.018Z27/30 Read block 152 len 8192
166102024-02-28T22:19:29.018Z28/30 Read block 126 len 36864
166112024-02-28T22:19:29.018Z29/30 Write block 563 len 20480 data: 13 14 15 17 20
166122024-02-28T22:19:29.018Z30/30 Write block 504 len 16384 data: 16 15 17 15
166132024-02-28T22:19:29.018Z----------------------------------------------------------------
166142024-02-28T22:19:29.018Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166152024-02-28T22:19:29.018ZStates: Active Active Active
166162024-02-28T22:19:29.018ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166172024-02-28T22:19:29.018Z 1 Acked 1000 Write 4 Done Done Done false
166182024-02-28T22:19:29.018Z 2 Acked 1001 Read 4 Done Done Done false
166192024-02-28T22:19:29.018Z 3 Acked 1002 Write 9 Done Done Done false
166202024-02-28T22:19:29.019Z 4 Acked 1003 Read 8 Done Done Done false
166212024-02-28T22:19:29.024Z 5 Acked 1004 Write 10 Done Sent Done false
166222024-02-28T22:19:29.024Z 6 Acked 1005 Read 1 Done Sent Done false
166232024-02-28T22:19:29.025Z 7 Acked 1006 Read 2 Done Sent Done false
166242024-02-28T22:19:29.025Z 8 Acked 1007 Write 9 Done Sent Done false
166252024-02-28T22:19:29.025Z 9 Acked 1008 Read 2 Done Sent Done false
166262024-02-28T22:19:29.025Z 10 Acked 1009 Read 7 Done Sent Done false
166272024-02-28T22:19:29.025Z 11 Acked 1010 Read 8 Done Sent Done false
166282024-02-28T22:19:29.025Z 12 Acked 1011 Flush 0 Done Sent Done false
166292024-02-28T22:19:29.026Z 13 Acked 1012 Write 9 Done Sent Done false
166302024-02-28T22:19:29.026Z 14 Acked 1013 Flush 0 Done Sent Done false
166312024-02-28T22:19:29.026Z 15 Acked 1014 Flush 0 Done Sent Done false
166322024-02-28T22:19:29.026Z 16 Acked 1015 Write 1 Done Sent Done false
166332024-02-28T22:19:29.026Z 17 Acked 1016 Read 8 Done Sent Done false
166342024-02-28T22:19:29.026Z 18 Acked 1017 Read 6 Done Sent Done false
166352024-02-28T22:19:29.026Z 19 Acked 1018 Read 8 Done Sent Done false
166362024-02-28T22:19:29.027Z 20 Acked 1019 Read 3 Done Sent Done false
166372024-02-28T22:19:29.027Z 21 Acked 1020 Write 1 Done Sent Done false
166382024-02-28T22:19:29.027Z 22 Acked 1021 Read 10 Done Sent Done false
166392024-02-28T22:19:29.027Z 23 Acked 1022 Write 2 Done Sent Done false
166402024-02-28T22:19:29.027Z 24 Acked 1023 Read 1 Done Sent Done false
166412024-02-28T22:19:29.027Z 25 Acked 1024 Write 1 Done Sent Done false
166422024-02-28T22:19:29.028Z 26 Acked 1025 Read 7 Done Sent Done false
166432024-02-28T22:19:29.028Z 27 Acked 1026 Read 2 Done Sent Done false
166442024-02-28T22:19:29.028Z 28 Acked 1027 Read 9 Done Sent Done false
166452024-02-28T22:19:29.028Z 29 Acked 1028 Write 5 Done Sent Done false
166462024-02-28T22:19:29.028Z 30 Acked 1029 Write 4 Sent Sent Sent false
166472024-02-28T22:19:29.028Z STATES DS:0 DS:1 DS:2 TOTAL
166482024-02-28T22:19:29.028Z New 0 0 0 0
166492024-02-28T22:19:29.028Z Sent 1 26 1 28
166502024-02-28T22:19:29.028Z Done 29 4 29 62
166512024-02-28T22:19:29.029Z Skipped 0 0 0 0
166522024-02-28T22:19:29.029Z Error 0 0 0 0
166532024-02-28T22:19:29.029ZLast Flush: 1014 0 1014
166542024-02-28T22:19:29.029ZDownstairs last five completed:
166552024-02-28T22:19:29.029ZUpstairs last five completed: 30 29 28 27 26
166562024-02-28T22:19:29.029Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:27.884265671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1751}
166572024-02-28T22:19:29.029ZWrote out file "/tmp/test_repair_verify.data"
166582024-02-28T22:19:29.029Z
166592024-02-28T22:19:29.029ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166602024-02-28T22:19:29.033ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166612024-02-28T22:19:29.034Z 0 000-019 148 148 148 268 268 268 F F F
166622024-02-28T22:19:29.034Z 1 020-039 148 148 148 269 269 269 F F F
166632024-02-28T22:19:29.034Z 2 040-059 152 152 152 275 275 275 F F F
166642024-02-28T22:19:29.034Z 3 060-079 154 152 154 279 275 279 F F F <---
166652024-02-28T22:19:29.034Z 4 080-099 148 148 148 270 270 270 F F F
166662024-02-28T22:19:29.034Z 5 100-119 152 152 152 276 276 276 F F F
166672024-02-28T22:19:29.034Z 6 120-139 151 151 151 274 274 274 F F F
166682024-02-28T22:19:29.034Z 7 140-159 152 152 152 275 275 275 T F T <---
166692024-02-28T22:19:29.034Z 8 160-179 152 152 152 277 277 277 T F T <---
166702024-02-28T22:19:29.035Z 9 180-199 148 148 148 268 268 268 T F T <---
166712024-02-28T22:19:29.035Z 10 200-219 152 152 152 276 276 276 F F F
166722024-02-28T22:19:29.035Z 11 220-239 152 152 152 276 276 276 F F F
166732024-02-28T22:19:29.035Z 12 240-259 154 151 154 280 274 280 F F F <---
166742024-02-28T22:19:29.035Z 13 260-279 154 153 154 280 278 280 F F F <---
166752024-02-28T22:19:29.035Z 14 280-299 145 145 145 262 262 262 F F F
166762024-02-28T22:19:29.035Z 15 300-319 145 145 145 262 262 262 T F T <---
166772024-02-28T22:19:29.036Z 16 320-339 151 151 151 274 274 274 T F T <---
166782024-02-28T22:19:29.036Z 17 340-359 151 151 151 274 274 274 F F F
166792024-02-28T22:19:29.036Z 18 360-379 148 148 148 268 268 268 F F F
166802024-02-28T22:19:29.036Z 19 380-399 153 153 153 278 278 278 F F F
166812024-02-28T22:19:29.036Z 20 400-419 153 153 153 278 278 278 F F F
166822024-02-28T22:19:29.036Z 21 420-439 154 153 154 279 278 279 F F F <---
166832024-02-28T22:19:29.036Z 22 440-459 154 152 154 279 275 279 F T F <---
166842024-02-28T22:19:29.036Z 23 460-479 137 137 137 253 253 253 F F F
166852024-02-28T22:19:29.036Z 24 480-499 152 152 152 275 275 275 F F F
166862024-02-28T22:19:29.037Z 25 500-519 148 148 148 268 268 268 T F T <---
166872024-02-28T22:19:29.037Z 26 520-539 151 151 151 274 274 274 F F F
166882024-02-28T22:19:29.037Z 27 540-559 154 141 154 279 256 279 F T F <---
166892024-02-28T22:19:29.037Z 28 560-579 150 150 150 273 273 273 T F T <---
166902024-02-28T22:19:29.037Z 29 580-599 142 142 142 259 259 259 F F F
166912024-02-28T22:19:29.037ZMax gen: 154, Max flush: 280
166922024-02-28T22:19:29.037ZError: Difference in extent metadata found!
166932024-02-28T22:19:29.037ZA Difference in extent metadata is expected here
166942024-02-28T22:19:29.038ZOn loop 77
166952024-02-28T22:19:29.038Z
166962024-02-28T22:19:29.038ZVerifying data now
166972024-02-28T22:19:33.691ZNo change after verify
166982024-02-28T22:19:33.691ZLoop: 77 Downstairs dump after verify (and repair):
166992024-02-28T22:19:33.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167002024-02-28T22:19:33.705Z 0 000-019 148 148 148 268 268 268 F F F
167012024-02-28T22:19:33.705Z 1 020-039 148 148 148 269 269 269 F F F
167022024-02-28T22:19:33.705Z 2 040-059 152 152 152 275 275 275 F F F
167032024-02-28T22:19:33.706Z 3 060-079 154 154 154 279 279 279 F F F
167042024-02-28T22:19:33.706Z 4 080-099 148 148 148 270 270 270 F F F
167052024-02-28T22:19:33.706Z 5 100-119 152 152 152 276 276 276 F F F
167062024-02-28T22:19:33.706Z 6 120-139 151 151 151 274 274 274 F F F
167072024-02-28T22:19:33.706Z 7 140-159 155 155 155 281 281 281 F F F
167082024-02-28T22:19:33.715Z 8 160-179 155 155 155 281 281 281 F F F
167092024-02-28T22:19:33.715Z 9 180-199 155 155 155 281 281 281 F F F
167102024-02-28T22:19:33.715Z 10 200-219 152 152 152 276 276 276 F F F
167112024-02-28T22:19:33.715Z 11 220-239 152 152 152 276 276 276 F F F
167122024-02-28T22:19:33.715Z 12 240-259 154 154 154 280 280 280 F F F
167132024-02-28T22:19:33.715Z 13 260-279 154 154 154 280 280 280 F F F
167142024-02-28T22:19:33.715Z 14 280-299 145 145 145 262 262 262 F F F
167152024-02-28T22:19:33.715Z 15 300-319 155 155 155 281 281 281 F F F
167162024-02-28T22:19:33.715Z 16 320-339 155 155 155 281 281 281 F F F
167172024-02-28T22:19:33.715Z 17 340-359 151 151 151 274 274 274 F F F
167182024-02-28T22:19:33.715Z 18 360-379 148 148 148 268 268 268 F F F
167192024-02-28T22:19:33.715Z 19 380-399 153 153 153 278 278 278 F F F
167202024-02-28T22:19:33.715Z 20 400-419 153 153 153 278 278 278 F F F
167212024-02-28T22:19:33.715Z 21 420-439 154 154 154 279 279 279 F F F
167222024-02-28T22:19:33.715Z 22 440-459 154 154 154 279 279 279 F F F
167232024-02-28T22:19:33.715Z 23 460-479 137 137 137 253 253 253 F F F
167242024-02-28T22:19:33.715Z 24 480-499 152 152 152 275 275 275 F F F
167252024-02-28T22:19:33.716Z 25 500-519 155 155 155 281 281 281 F F F
167262024-02-28T22:19:33.716Z 26 520-539 151 151 151 274 274 274 F F F
167272024-02-28T22:19:33.716Z 27 540-559 154 154 154 279 279 279 F F F
167282024-02-28T22:19:33.716Z 28 560-579 155 155 155 281 281 281 F F F
167292024-02-28T22:19:33.716Z 29 580-599 142 142 142 259 259 259 F F F
167302024-02-28T22:19:33.716ZMax gen: 155, Max flush: 281
167312024-02-28T22:19:33.716Z
167322024-02-28T22:19:33.716ZBegin loop 78
167332024-02-28T22:19:33.716ZDownstairs to restart: 2
167342024-02-28T22:19:33.728Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610129608Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
167352024-02-28T22:19:33.728ZCrucible runtime is spawned
167362024-02-28T22:19:33.734Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610465908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
167372024-02-28T22:19:33.734Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610509687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
167382024-02-28T22:19:33.734Z{"msg":"Crucible 6bfeafe3-631f-45b9-a897-5223ccdd46c4 has session id: 82a80fb3-3a97-4a5e-9968-6896aabbf714","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610537026Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167392024-02-28T22:19:33.734Z{"msg":"Upstairs opts: Upstairs UUID: 6bfeafe3-631f-45b9-a897-5223ccdd46c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610584985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167402024-02-28T22:19:33.734Z{"msg":"Crucible stats registered with UUID: 6bfeafe3-631f-45b9-a897-5223ccdd46c4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610629354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167412024-02-28T22:19:33.734Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610661573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
167422024-02-28T22:19:33.734Z{"msg":"6bfeafe3-631f-45b9-a897-5223ccdd46c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610699542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167432024-02-28T22:19:33.734Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610741071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167442024-02-28T22:19:33.734Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610786759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167452024-02-28T22:19:33.734Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610865577Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167462024-02-28T22:19:33.734Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610914826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167472024-02-28T22:19:33.735Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610943625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167482024-02-28T22:19:33.735Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.610984854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167492024-02-28T22:19:33.735Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611032173Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167502024-02-28T22:19:33.735Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611070682Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167512024-02-28T22:19:33.735Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611099791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167522024-02-28T22:19:33.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.61113778Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167532024-02-28T22:19:33.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611159639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167542024-02-28T22:19:33.735Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611194918Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167552024-02-28T22:19:33.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611714524Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167562024-02-28T22:19:33.735Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611753573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167572024-02-28T22:19:33.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611770662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167582024-02-28T22:19:33.735Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611784872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167592024-02-28T22:19:33.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611798462Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167602024-02-28T22:19:33.735Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.611821161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167612024-02-28T22:19:33.735Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612348736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167622024-02-28T22:19:33.735Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612377306Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167632024-02-28T22:19:33.735Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.61259067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167642024-02-28T22:19:33.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612821133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167652024-02-28T22:19:33.735Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612868772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167662024-02-28T22:19:33.735Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.61293444Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167672024-02-28T22:19:33.735Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612972219Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167682024-02-28T22:19:33.735Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.612999058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167692024-02-28T22:19:33.736Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613029128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167702024-02-28T22:19:33.736Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613108406Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167712024-02-28T22:19:33.736Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613139165Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167722024-02-28T22:19:33.736Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 275, 279, 270, 276, 274, 281, 281, 281, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613164554Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167732024-02-28T22:19:33.736Z{"msg":"[0]R generation[0..12]: [148, 148, 152, 154, 148, 152, 151, 155, 155, 155, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613193153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167742024-02-28T22:19:33.736Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613220322Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167752024-02-28T22:19:33.736Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 275, 279, 270, 276, 274, 281, 281, 281, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613254191Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167762024-02-28T22:19:33.736Z{"msg":"[1]R generation[0..12]: [148, 148, 152, 154, 148, 152, 151, 155, 155, 155, 152, 152]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
167772024-02-28T22:19:33.736Z,"time":"2024-02-28T22:19:32.613279911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167782024-02-28T22:19:33.736Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.61331572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167792024-02-28T22:19:33.736Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 275, 279, 270, 276, 274, 281, 281, 281, 276, 276]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613350239Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167802024-02-28T22:19:33.736Z{"msg":"[2]R generation[0..12]: [148, 148, 152, 154, 148, 152, 151, 155, 155, 155, 152, 152]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613376528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167812024-02-28T22:19:33.736Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613406977Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167822024-02-28T22:19:33.736Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613430966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167832024-02-28T22:19:33.736Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613460826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167842024-02-28T22:19:33.736Z{"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613484055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167852024-02-28T22:19:33.736Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613510764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"":"downstairs","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167862024-02-28T22:19:33.736Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613534584Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167872024-02-28T22:19:33.736ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167882024-02-28T22:19:33.736Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613563063Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"0","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167892024-02-28T22:19:33.736Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613586552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"1","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167902024-02-28T22:19:33.736Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613613101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"client":"2","":"downstairs","session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167912024-02-28T22:19:33.736Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613636551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167922024-02-28T22:19:33.737Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.61366313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167932024-02-28T22:19:33.737Z{"msg":"6bfeafe3-631f-45b9-a897-5223ccdd46c4 is now active with session: 82a80fb3-3a97-4a5e-9968-6896aabbf714","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613686479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167942024-02-28T22:19:33.737Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613711409Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759,"session_id":"82a80fb3-3a97-4a5e-9968-6896aabbf714"}
167952024-02-28T22:19:33.740Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:32.613769807Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
167962024-02-28T22:19:33.740ZLoading write count information from file "/tmp/test_repair_verify.data"
167972024-02-28T22:19:33.740ZRun Repair workload
167982024-02-28T22:19:33.740Z01/30 Write block 474 len 20480 data: 13 11 10 7 5
167992024-02-28T22:19:33.741Z02/30 Read block 308 len 32768
168002024-02-28T22:19:33.741Z03/30 Read block 71 len 12288
168012024-02-28T22:19:33.741Z04/30 Write block 56 len 32768 data: 11 12 11 14 15 14 15 13
168022024-02-28T22:19:33.741Z05/30 Write block 477 len 40960 data: 8 6 5 6 6 10 11 13 15 12
168032024-02-28T22:19:33.741Z06/30 Write block 381 len 20480 data: 13 17 19 18 18
168042024-02-28T22:19:33.741Z07/30 Read block 518 len 36864
168052024-02-28T22:19:33.741Z08/30 Read block 551 len 16384
168062024-02-28T22:19:33.744Z09/30 Write block 558 len 36864 data: 9 9 10 12 11 14 15 16 18
168072024-02-28T22:19:33.744Z10/30 Flush
168082024-02-28T22:19:33.744Z11/30 Write block 222 len 12288 data: 9 8 9
168092024-02-28T22:19:33.748Z12/30 Flush
168102024-02-28T22:19:33.748Z13/30 Write block 555 len 32768 data: 7 8 9 10 10 11 13 12
168112024-02-28T22:19:33.751Z14/30 Write block 589 len 40960 data: 9 11 12 12 13 12 12 9 9 9
168122024-02-28T22:19:33.751Z15/30 Write block 439 len 20480 data: 18 17 14 14 11
168132024-02-28T22:19:33.751Z16/30 Write block 182 len 8192 data: 8 10
168142024-02-28T22:19:33.751Z17/30 Write block 171 len 12288 data: 15 14 15
168152024-02-28T22:19:33.751Z18/30 Write block 570 len 36864 data: 13 10 10 11 9 10 10 11 12
168162024-02-28T22:19:33.751Z19/30 Read block 31 len 4096
168172024-02-28T22:19:33.754Z20/30 Read block 138 len 28672
168182024-02-28T22:19:33.754Z21/30 Read block 555 len 28672
168192024-02-28T22:19:33.754Z22/30 Write block 580 len 24576 data: 11 9 6 7 9 11
168202024-02-28T22:19:33.758Z23/30 Write block 113 len 24576 data: 15 16 15 15 14 12
168212024-02-28T22:19:33.758Z24/30 Read block 14 len 28672
168222024-02-28T22:19:33.758Z25/30 Flush
168232024-02-28T22:19:33.761Z26/30 Write block 476 len 20480 data: 11 9 7 6 7
168242024-02-28T22:19:33.761Z27/30 Write block 46 len 28672 data: 7 9 10 12 10 12 7
168252024-02-28T22:19:33.761Z28/30 Read block 55 len 28672
168262024-02-28T22:19:33.766Z29/30 Read block 48 len 36864
168272024-02-28T22:19:33.766Z30/30 Write block 470 len 20480 data: 11 13 15 12 14
168282024-02-28T22:19:33.766Z----------------------------------------------------------------
168292024-02-28T22:19:33.766Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:18
168302024-02-28T22:19:33.766ZStates: Active Active Active
168312024-02-28T22:19:33.766ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168322024-02-28T22:19:33.766Z 13 Acked 1012 Write 8 Done Done Sent false
168332024-02-28T22:19:33.766Z 14 Acked 1013 Write 10 Done Done Sent false
168342024-02-28T22:19:33.766Z 15 Acked 1014 Write 5 Done Done Sent false
168352024-02-28T22:19:33.766Z 16 Acked 1015 Write 2 Done Done Sent false
168362024-02-28T22:19:33.766Z 17 Acked 1016 Write 3 Done Done Sent false
168372024-02-28T22:19:33.766Z 18 Acked 1017 Write 9 Done Done Sent false
168382024-02-28T22:19:33.766Z 19 Acked 1018 Read 1 Done Done Sent false
168392024-02-28T22:19:33.766Z 20 Acked 1019 Read 7 Done Done Sent false
168402024-02-28T22:19:33.766Z 21 Acked 1020 Read 7 Done Done Sent false
168412024-02-28T22:19:33.766Z 22 Acked 1021 Write 6 Done Done Sent false
168422024-02-28T22:19:33.766Z 23 Acked 1022 Write 6 Done Done Sent false
168432024-02-28T22:19:33.767Z 24 Acked 1023 Read 7 Done Done Sent false
168442024-02-28T22:19:33.767Z 25 Acked 1024 Flush 0 Done Done Sent false
168452024-02-28T22:19:33.767Z 26 Acked 1025 Write 5 Done Done Sent false
168462024-02-28T22:19:33.767Z 27 Acked 1026 Write 7 Done Done Sent false
168472024-02-28T22:19:33.767Z 28 Acked 1027 Read 7 Done Done Sent false
168482024-02-28T22:19:33.767Z 29 Acked 1028 Read 9 Done Done Sent false
168492024-02-28T22:19:33.767Z 30 Acked 1029 Write 5 Sent Sent Sent false
168502024-02-28T22:19:33.767Z STATES DS:0 DS:1 DS:2 TOTAL
168512024-02-28T22:19:33.767Z New 0 0 0 0
168522024-02-28T22:19:33.767Z Sent 1 1 18 20
168532024-02-28T22:19:33.767Z Done 17 17 0 34
168542024-02-28T22:19:33.767Z Skipped 0 0 0 0
168552024-02-28T22:19:33.767Z Error 0 0 0 0
168562024-02-28T22:19:33.767ZLast Flush: 1024 1024 1011
168572024-02-28T22:19:33.767ZDownstairs last five completed: 1011 1010 1009 1008 1007
168582024-02-28T22:19:33.767ZUpstairs last five completed: 30 29 28 27 26
168592024-02-28T22:19:33.767Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:32.646792758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1759}
168602024-02-28T22:19:33.767ZWrote out file "/tmp/test_repair_verify.data"
168612024-02-28T22:19:33.772Z
168622024-02-28T22:19:33.772ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168632024-02-28T22:19:33.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168642024-02-28T22:19:33.789Z 0 000-019 148 148 148 268 268 268 F F F
168652024-02-28T22:19:33.789Z 1 020-039 148 148 148 269 269 269 F F F
168662024-02-28T22:19:33.790Z 2 040-059 156 156 156 282 282 282 T T F <---
168672024-02-28T22:19:33.790Z 3 060-079 156 156 156 282 282 282 F F F
168682024-02-28T22:19:33.790Z 4 080-099 148 148 148 270 270 270 F F F
168692024-02-28T22:19:33.790Z 5 100-119 156 156 152 284 284 276 F F F <---
168702024-02-28T22:19:33.790Z 6 120-139 151 151 151 274 274 274 F F F
168712024-02-28T22:19:33.790Z 7 140-159 155 155 155 281 281 281 F F F
168722024-02-28T22:19:33.790Z 8 160-179 156 156 155 284 284 281 F F F <---
168732024-02-28T22:19:33.790Z 9 180-199 156 156 155 284 284 281 F F F <---
168742024-02-28T22:19:33.790Z 10 200-219 152 152 152 276 276 276 F F F
168752024-02-28T22:19:33.794Z 11 220-239 156 156 156 283 283 283 F F F
168762024-02-28T22:19:33.794Z 12 240-259 154 154 154 280 280 280 F F F
168772024-02-28T22:19:33.794Z 13 260-279 154 154 154 280 280 280 F F F
168782024-02-28T22:19:33.794Z 14 280-299 145 145 145 262 262 262 F F F
168792024-02-28T22:19:33.794Z 15 300-319 155 155 155 281 281 281 F F F
168802024-02-28T22:19:33.794Z 16 320-339 155 155 155 281 281 281 F F F
168812024-02-28T22:19:33.794Z 17 340-359 151 151 151 274 274 274 F F F
168822024-02-28T22:19:33.794Z 18 360-379 148 148 148 268 268 268 F F F
168832024-02-28T22:19:33.794Z 19 380-399 156 156 156 282 282 282 F F F
168842024-02-28T22:19:33.794Z 20 400-419 153 153 153 278 278 278 F F F
168852024-02-28T22:19:33.794Z 21 420-439 156 156 154 284 284 279 F F F <---
168862024-02-28T22:19:33.794Z 22 440-459 156 156 154 284 284 279 F F F <---
168872024-02-28T22:19:33.794Z 23 460-479 156 156 156 282 282 282 T T F <---
168882024-02-28T22:19:33.794Z 24 480-499 156 156 156 282 282 282 T T F <---
168892024-02-28T22:19:33.795Z 25 500-519 155 155 155 281 281 281 F F F
168902024-02-28T22:19:33.795Z 26 520-539 151 151 151 274 274 274 F F F
168912024-02-28T22:19:33.795Z 27 540-559 156 156 156 284 284 282 F F F <---
168922024-02-28T22:19:33.795Z 28 560-579 156 156 156 284 284 282 F F F <---
168932024-02-28T22:19:33.795Z 29 580-599 156 156 142 284 284 259 F F F <---
168942024-02-28T22:19:33.795ZMax gen: 156, Max flush: 284
168952024-02-28T22:19:33.795ZError: Difference in extent metadata found!
168962024-02-28T22:19:33.795ZA Difference in extent metadata is expected here
168972024-02-28T22:19:33.795ZOn loop 78
168982024-02-28T22:19:33.795Z
168992024-02-28T22:19:33.795ZVerifying data now
169002024-02-28T22:19:38.351ZNo change after verify
169012024-02-28T22:19:38.351ZLoop: 78 Downstairs dump after verify (and repair):
169022024-02-28T22:19:38.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169032024-02-28T22:19:38.366Z 0 000-019 148 148 148 268 268 268 F F F
169042024-02-28T22:19:38.366Z 1 020-039 148 148 148 269 269 269 F F F
169052024-02-28T22:19:38.366Z 2 040-059 157 157 157 285 285 285 F F F
169062024-02-28T22:19:38.366Z 3 060-079 156 156 156 282 282 282 F F F
169072024-02-28T22:19:38.366Z 4 080-099 148 148 148 270 270 270 F F F
169082024-02-28T22:19:38.367Z 5 100-119 156 156 156 284 284 284 F F F
169092024-02-28T22:19:38.367Z 6 120-139 151 151 151 274 274 274 F F F
169102024-02-28T22:19:38.367Z 7 140-159 155 155 155 281 281 281 F F F
169112024-02-28T22:19:38.367Z 8 160-179 156 156 156 284 284 284 F F F
169122024-02-28T22:19:38.367Z 9 180-199 156 156 156 284 284 284 F F F
169132024-02-28T22:19:38.367Z 10 200-219 152 152 152 276 276 276 F F F
169142024-02-28T22:19:38.367Z 11 220-239 156 156 156 283 283 283 F F F
169152024-02-28T22:19:38.367Z 12 240-259 154 154 154 280 280 280 F F F
169162024-02-28T22:19:38.371Z 13 260-279 154 154 154 280 280 280 F F F
169172024-02-28T22:19:38.371Z 14 280-299 145 145 145 262 262 262 F F F
169182024-02-28T22:19:38.371Z 15 300-319 155 155 155 281 281 281 F F F
169192024-02-28T22:19:38.371Z 16 320-339 155 155 155 281 281 281 F F F
169202024-02-28T22:19:38.371Z 17 340-359 151 151 151 274 274 274 F F F
169212024-02-28T22:19:38.371Z 18 360-379 148 148 148 268 268 268 F F F
169222024-02-28T22:19:38.371Z 19 380-399 156 156 156 282 282 282 F F F
169232024-02-28T22:19:38.371Z 20 400-419 153 153 153 278 278 278 F F F
169242024-02-28T22:19:38.371Z 21 420-439 156 156 156 284 284 284 F F F
169252024-02-28T22:19:38.371Z 22 440-459 156 156 156 284 284 284 F F F
169262024-02-28T22:19:38.371Z 23 460-479 157 157 157 285 285 285 F F F
169272024-02-28T22:19:38.371Z 24 480-499 157 157 157 285 285 285 F F F
169282024-02-28T22:19:38.371Z 25 500-519 155 155 155 281 281 281 F F F
169292024-02-28T22:19:38.371Z 26 520-539 151 151 151 274 274 274 F F F
169302024-02-28T22:19:38.371Z 27 540-559 156 156 156 284 284 284 F F F
169312024-02-28T22:19:38.371Z 28 560-579 156 156 156 284 284 284 F F F
169322024-02-28T22:19:38.372Z 29 580-599 156 156 156 284 284 284 F F F
169332024-02-28T22:19:38.372ZMax gen: 157, Max flush: 285
169342024-02-28T22:19:38.372Z
169352024-02-28T22:19:38.372ZBegin loop 79
169362024-02-28T22:19:38.372ZDownstairs to restart: 1
169372024-02-28T22:19:38.389Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271139002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
169382024-02-28T22:19:38.389Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271440214Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
169392024-02-28T22:19:38.389Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
169402024-02-28T22:19:38.396Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271487762Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
169412024-02-28T22:19:38.396Z{"msg":"Crucible 3cd1a69d-5089-4cc4-880a-21c0a891468a has session id: 4a2fca1e-0a82-405d-be97-991b261c43c0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271517332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169422024-02-28T22:19:38.397Z{"msg":"Upstairs opts: Upstairs UUID: 3cd1a69d-5089-4cc4-880a-21c0a891468a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.2715727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169432024-02-28T22:19:38.397Z{"msg":"Crucible stats registered with UUID: 3cd1a69d-5089-4cc4-880a-21c0a891468a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271619449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169442024-02-28T22:19:38.397Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271658488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
169452024-02-28T22:19:38.397Z{"msg":"3cd1a69d-5089-4cc4-880a-21c0a891468a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271695936Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169462024-02-28T22:19:38.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271741615Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169472024-02-28T22:19:38.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271778744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169482024-02-28T22:19:38.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271814183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169492024-02-28T22:19:38.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271833593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169502024-02-28T22:19:38.397Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.271875071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169512024-02-28T22:19:38.397Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27193981Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169522024-02-28T22:19:38.397Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272013658Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169532024-02-28T22:19:38.397Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272053107Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169542024-02-28T22:19:38.398Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272091976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169552024-02-28T22:19:38.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272117765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169562024-02-28T22:19:38.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272152884Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169572024-02-28T22:19:38.398Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272176623Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169582024-02-28T22:19:38.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272905513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169592024-02-28T22:19:38.398Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272942872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169602024-02-28T22:19:38.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.272989471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169612024-02-28T22:19:38.398Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27301797Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169622024-02-28T22:19:38.398Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.273196375Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169632024-02-28T22:19:38.398Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.273258073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169642024-02-28T22:19:38.398Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.273553995Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169652024-02-28T22:19:38.398Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.273719671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169662024-02-28T22:19:38.399Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.273955284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169672024-02-28T22:19:38.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27411664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169682024-02-28T22:19:38.399Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274149229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169692024-02-28T22:19:38.399Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274166208Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169702024-02-28T22:19:38.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274198738Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169712024-02-28T22:19:38.399Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274221247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169722024-02-28T22:19:38.399Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274235177Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169732024-02-28T22:19:38.399Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27447618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169742024-02-28T22:19:38.399Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274515479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169752024-02-28T22:19:38.399Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 284, 284, 276, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274541538Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169762024-02-28T22:19:38.399Z{"msg":"[0]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 156, 156, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274565077Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169772024-02-28T22:19:38.399Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274579827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169782024-02-28T22:19:38.400Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 284, 284, 276, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274601407Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169792024-02-28T22:19:38.400Z{"msg":"[1]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 156, 156, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274615766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169802024-02-28T22:19:38.400Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274636646Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169812024-02-28T22:19:38.400Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 284, 284, 276, 283]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274652225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169822024-02-28T22:19:38.400ZWait for a query_work_queue command to finish before sending IO
169832024-02-28T22:19:38.400Z{"msg":"[2]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 156, 156, 152, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274679774Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169842024-02-28T22:19:38.400Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274695054Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169852024-02-28T22:19:38.400Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274714853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169862024-02-28T22:19:38.400Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274728933Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169872024-02-28T22:19:38.400Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274754022Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169882024-02-28T22:19:38.400Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274767792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"":"downstairs","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169892024-02-28T22:19:38.400Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274788081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169902024-02-28T22:19:38.400Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274800491Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"0","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169912024-02-28T22:19:38.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27482069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"1","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169922024-02-28T22:19:38.401Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27483434Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"client":"2","":"downstairs","session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169932024-02-28T22:19:38.401Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.27484834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169942024-02-28T22:19:38.401Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274868649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169952024-02-28T22:19:38.401Z{"msg":"3cd1a69d-5089-4cc4-880a-21c0a891468a is now active with session: 4a2fca1e-0a82-405d-be97-991b261c43c0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274881859Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169962024-02-28T22:19:38.401Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274894428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767,"session_id":"4a2fca1e-0a82-405d-be97-991b261c43c0"}
169972024-02-28T22:19:38.401Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:37.274913628Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
169982024-02-28T22:19:38.401ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169992024-02-28T22:19:38.405ZLoading write count information from file "/tmp/test_repair_verify.data"
170002024-02-28T22:19:38.405ZRun Repair workload
170012024-02-28T22:19:38.405Z01/30 Write block 508 len 20480 data: 14 17 15 11 13
170022024-02-28T22:19:38.406Z02/30 Write block 555 len 36864 data: 8 9 10 11 11 12 14 13 15
170032024-02-28T22:19:38.406Z03/30 Read block 62 len 36864
170042024-02-28T22:19:38.406Z04/30 Write block 305 len 40960 data: 12 12 15 14 13 9 14 14 12 12
170052024-02-28T22:19:38.406Z05/30 Flush
170062024-02-28T22:19:38.406Z06/30 Write block 215 len 32768 data: 9 10 11 9 9 10 11 10
170072024-02-28T22:19:38.406Z07/30 Read block 156 len 8192
170082024-02-28T22:19:38.406Z08/30 Write block 463 len 24576 data: 10 9 11 9 9 10
170092024-02-28T22:19:38.410Z09/30 Write block 173 len 32768 data: 16 15 13 11 10 9 6 6
170102024-02-28T22:19:38.410Z10/30 Flush
170112024-02-28T22:19:38.410Z11/30 Flush
170122024-02-28T22:19:38.410Z12/30 Write block 313 len 24576 data: 13 13 9 9 8 8
170132024-02-28T22:19:38.410Z13/30 Read block 227 len 16384
170142024-02-28T22:19:38.422Z14/30 Write block 519 len 8192 data: 17 17
170152024-02-28T22:19:38.422Z15/30 Read block 415 len 8192
170162024-02-28T22:19:38.423Z16/30 Read block 83 len 36864
170172024-02-28T22:19:38.423Z17/30 Write block 259 len 28672 data: 15 14 15 18 17 17 17
170182024-02-28T22:19:38.426Z18/30 Read block 580 len 28672
170192024-02-28T22:19:38.426Z19/30 Write block 484 len 4096 data: 14
170202024-02-28T22:19:38.426Z20/30 Write block 573 len 12288 data: 12 10 11
170212024-02-28T22:19:38.427Z21/30 Flush
170222024-02-28T22:19:38.427Z22/30 Write block 554 len 16384 data: 6 9 10 11
170232024-02-28T22:19:38.427Z23/30 Write block 463 len 36864 data: 11 10 12 10 10 11 10 12 14
170242024-02-28T22:19:38.427Z24/30 Read block 176 len 32768
170252024-02-28T22:19:38.427Z25/30 Read block 362 len 28672
170262024-02-28T22:19:38.427Z26/30 Read block 162 len 4096
170272024-02-28T22:19:38.427Z27/30 Read block 513 len 8192
170282024-02-28T22:19:38.427Z28/30 Write block 422 len 12288 data: 16 14 13
170292024-02-28T22:19:38.427Z29/30 Read block 329 len 32768
170302024-02-28T22:19:38.427Z30/30 Write block 514 len 36864 data: 15 16 17 19 17 18 18 16 14
170312024-02-28T22:19:38.432Z----------------------------------------------------------------
170322024-02-28T22:19:38.432Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:25
170332024-02-28T22:19:38.432ZStates: Active Active Active
170342024-02-28T22:19:38.432ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170352024-02-28T22:19:38.432Z 6 Acked 1005 Write 8 Done Sent Done false
170362024-02-28T22:19:38.432Z 7 Acked 1006 Read 2 Done Sent Done false
170372024-02-28T22:19:38.432Z 8 Acked 1007 Write 6 Done Sent Done false
170382024-02-28T22:19:38.432Z 9 Acked 1008 Write 8 Done Sent Done false
170392024-02-28T22:19:38.432Z 10 Acked 1009 Flush 0 Done Sent Done false
170402024-02-28T22:19:38.432Z 11 Acked 1010 Flush 0 Done Sent Done false
170412024-02-28T22:19:38.432Z 12 Acked 1011 Write 6 Done Sent Done false
170422024-02-28T22:19:38.432Z 13 Acked 1012 Read 4 Done Sent Done false
170432024-02-28T22:19:38.433Z 14 Acked 1013 Write 2 Done Sent Done false
170442024-02-28T22:19:38.433Z 15 Acked 1014 Read 2 Done Sent Done false
170452024-02-28T22:19:38.433Z 16 Acked 1015 Read 9 Done Sent Done false
170462024-02-28T22:19:38.433Z 17 Acked 1016 Write 7 Done Sent Done false
170472024-02-28T22:19:38.433Z 18 Acked 1017 Read 7 Done Sent Done false
170482024-02-28T22:19:38.433Z 19 Acked 1018 Write 1 Done Sent Done false
170492024-02-28T22:19:38.433Z 20 Acked 1019 Write 3 Done Sent Done false
170502024-02-28T22:19:38.433Z 21 Acked 1020 Flush 0 Done Sent Done false
170512024-02-28T22:19:38.433Z 22 Acked 1021 Write 4 Done Sent Done false
170522024-02-28T22:19:38.433Z 23 Acked 1022 Write 9 Done Sent Done false
170532024-02-28T22:19:38.433Z 24 Acked 1023 Read 8 Done Sent Done false
170542024-02-28T22:19:38.433Z 25 Acked 1024 Read 7 Done Sent Done false
170552024-02-28T22:19:38.433Z 26 Acked 1025 Read 1 Done Sent Done false
170562024-02-28T22:19:38.433Z 27 Acked 1026 Read 2 Done Sent Done false
170572024-02-28T22:19:38.433Z 28 Acked 1027 Write 3 Done Sent Done false
170582024-02-28T22:19:38.433Z 29 Acked 1028 Read 8 Done Sent Done false
170592024-02-28T22:19:38.434Z 30 Acked 1029 Write 9 Sent Sent Sent false
170602024-02-28T22:19:38.434Z STATES DS:0 DS:1 DS:2 TOTAL
170612024-02-28T22:19:38.434Z New 0 0 0 0
170622024-02-28T22:19:38.434Z Sent 1 25 1 27
170632024-02-28T22:19:38.434Z Done 24 0 24 48
170642024-02-28T22:19:38.434Z Skipped 0 0 0 0
170652024-02-28T22:19:38.434Z Error 0 0 0 0
170662024-02-28T22:19:38.434ZLast Flush: 1020 1004 1020
170672024-02-28T22:19:38.434ZDownstairs last five completed: 1004 1003 1002 1001 1000
170682024-02-28T22:19:38.434ZUpstairs last five completed: 30 29 28 27 26
170692024-02-28T22:19:38.434Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:37.308901471Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1767}
170702024-02-28T22:19:38.434ZWrote out file "/tmp/test_repair_verify.data"
170712024-02-28T22:19:38.434Z
170722024-02-28T22:19:38.440ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170732024-02-28T22:19:38.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170742024-02-28T22:19:38.451Z 0 000-019 148 148 148 268 268 268 F F F
170752024-02-28T22:19:38.451Z 1 020-039 148 148 148 269 269 269 F F F
170762024-02-28T22:19:38.451Z 2 040-059 157 157 157 285 285 285 F F F
170772024-02-28T22:19:38.451Z 3 060-079 156 156 156 282 282 282 F F F
170782024-02-28T22:19:38.451Z 4 080-099 148 148 148 270 270 270 F F F
170792024-02-28T22:19:38.451Z 5 100-119 156 156 156 284 284 284 F F F
170802024-02-28T22:19:38.451Z 6 120-139 151 151 151 274 274 274 F F F
170812024-02-28T22:19:38.452Z 7 140-159 155 155 155 281 281 281 F F F
170822024-02-28T22:19:38.452Z 8 160-179 158 156 158 287 284 287 F F F <---
170832024-02-28T22:19:38.452Z 9 180-199 158 156 158 287 284 287 F F F <---
170842024-02-28T22:19:38.452Z 10 200-219 158 152 158 287 276 287 F F F <---
170852024-02-28T22:19:38.452Z 11 220-239 158 156 158 287 283 287 F F F <---
170862024-02-28T22:19:38.452Z 12 240-259 158 154 158 289 280 289 F F F <---
170872024-02-28T22:19:38.456Z 13 260-279 158 154 158 289 280 289 F F F <---
170882024-02-28T22:19:38.456Z 14 280-299 145 145 145 262 262 262 F F F
170892024-02-28T22:19:38.456Z 15 300-319 158 158 158 289 286 289 F F F <---
170902024-02-28T22:19:38.456Z 16 320-339 155 155 155 281 281 281 F F F
170912024-02-28T22:19:38.457Z 17 340-359 151 151 151 274 274 274 F F F
170922024-02-28T22:19:38.457Z 18 360-379 148 148 148 268 268 268 F F F
170932024-02-28T22:19:38.457Z 19 380-399 156 156 156 282 282 282 F F F
170942024-02-28T22:19:38.457Z 20 400-419 153 153 153 278 278 278 F F F
170952024-02-28T22:19:38.457Z 21 420-439 156 156 156 284 284 284 T F T <---
170962024-02-28T22:19:38.457Z 22 440-459 156 156 156 284 284 284 F F F
170972024-02-28T22:19:38.457Z 23 460-479 158 157 158 287 285 287 T F T <---
170982024-02-28T22:19:38.457Z 24 480-499 158 157 158 289 285 289 F F F <---
170992024-02-28T22:19:38.457Z 25 500-519 158 158 158 289 286 289 T F T <---
171002024-02-28T22:19:38.457Z 26 520-539 158 151 158 289 274 289 T F T <---
171012024-02-28T22:19:38.457Z 27 540-559 158 158 158 286 286 286 T F T <---
171022024-02-28T22:19:38.457Z 28 560-579 158 158 158 289 286 289 F F F <---
171032024-02-28T22:19:38.457Z 29 580-599 156 156 156 284 284 284 F F F
171042024-02-28T22:19:38.458ZMax gen: 158, Max flush: 289
171052024-02-28T22:19:38.458ZError: Difference in extent metadata found!
171062024-02-28T22:19:38.458ZA Difference in extent metadata is expected here
171072024-02-28T22:19:38.458ZOn loop 79
171082024-02-28T22:19:38.458Z
171092024-02-28T22:19:38.463ZVerifying data now
171102024-02-28T22:19:43.167ZNo change after verify
171112024-02-28T22:19:43.167ZLoop: 79 Downstairs dump after verify (and repair):
171122024-02-28T22:19:43.181ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171132024-02-28T22:19:43.181Z 0 000-019 148 148 148 268 268 268 F F F
171142024-02-28T22:19:43.181Z 1 020-039 148 148 148 269 269 269 F F F
171152024-02-28T22:19:43.181Z 2 040-059 157 157 157 285 285 285 F F F
171162024-02-28T22:19:43.181Z 3 060-079 156 156 156 282 282 282 F F F
171172024-02-28T22:19:43.181Z 4 080-099 148 148 148 270 270 270 F F F
171182024-02-28T22:19:43.181Z 5 100-119 156 156 156 284 284 284 F F F
171192024-02-28T22:19:43.181Z 6 120-139 151 151 151 274 274 274 F F F
171202024-02-28T22:19:43.181Z 7 140-159 155 155 155 281 281 281 F F F
171212024-02-28T22:19:43.181Z 8 160-179 158 158 158 287 287 287 F F F
171222024-02-28T22:19:43.185Z 9 180-199 158 158 158 287 287 287 F F F
171232024-02-28T22:19:43.185Z 10 200-219 158 158 158 287 287 287 F F F
171242024-02-28T22:19:43.186Z 11 220-239 158 158 158 287 287 287 F F F
171252024-02-28T22:19:43.186Z 12 240-259 158 158 158 289 289 289 F F F
171262024-02-28T22:19:43.186Z 13 260-279 158 158 158 289 289 289 F F F
171272024-02-28T22:19:43.186Z 14 280-299 145 145 145 262 262 262 F F F
171282024-02-28T22:19:43.186Z 15 300-319 158 158 158 289 289 289 F F F
171292024-02-28T22:19:43.186Z 16 320-339 155 155 155 281 281 281 F F F
171302024-02-28T22:19:43.186Z 17 340-359 151 151 151 274 274 274 F F F
171312024-02-28T22:19:43.186Z 18 360-379 148 148 148 268 268 268 F F F
171322024-02-28T22:19:43.186Z 19 380-399 156 156 156 282 282 282 F F F
171332024-02-28T22:19:43.186Z 20 400-419 153 153 153 278 278 278 F F F
171342024-02-28T22:19:43.186Z 21 420-439 159 159 159 290 290 290 F F F
171352024-02-28T22:19:43.186Z 22 440-459 156 156 156 284 284 284 F F F
171362024-02-28T22:19:43.187Z 23 460-479 159 159 159 290 290 290 F F F
171372024-02-28T22:19:43.187Z 24 480-499 158 158 158 289 289 289 F F F
171382024-02-28T22:19:43.187Z 25 500-519 159 159 159 290 290 290 F F F
171392024-02-28T22:19:43.187Z 26 520-539 159 159 159 290 290 290 F F F
171402024-02-28T22:19:43.187Z 27 540-559 159 159 159 290 290 290 F F F
171412024-02-28T22:19:43.187Z 28 560-579 158 158 158 289 289 289 F F F
171422024-02-28T22:19:43.187Z 29 580-599 156 156 156 284 284 284 F F F
171432024-02-28T22:19:43.187ZMax gen: 159, Max flush: 290
171442024-02-28T22:19:43.187Z
171452024-02-28T22:19:43.187ZBegin loop 80
171462024-02-28T22:19:43.187ZDownstairs to restart: 1
171472024-02-28T22:19:43.210Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084131349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
171482024-02-28T22:19:43.210Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\nCrucible runtime is spawned
171492024-02-28T22:19:43.211Z opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08444198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
171502024-02-28T22:19:43.211Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084497428Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
171512024-02-28T22:19:43.211Z{"msg":"Crucible 8f86733a-0211-4ddc-b2a7-657ee430f6d6 has session id: 4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084518948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171522024-02-28T22:19:43.211Z{"msg":"Upstairs opts: Upstairs UUID: 8f86733a-0211-4ddc-b2a7-657ee430f6d6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084554867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171532024-02-28T22:19:43.211Z{"msg":"Crucible stats registered with UUID: 8f86733a-0211-4ddc-b2a7-657ee430f6d6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084573446Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171542024-02-28T22:19:43.211Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084597806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
171552024-02-28T22:19:43.211Z{"msg":"8f86733a-0211-4ddc-b2a7-657ee430f6d6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084646974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171562024-02-28T22:19:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084676264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171572024-02-28T22:19:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084711663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171582024-02-28T22:19:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084757701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171592024-02-28T22:19:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08481579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171602024-02-28T22:19:43.211Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084859969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171612024-02-28T22:19:43.211Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084941426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171622024-02-28T22:19:43.211Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.084983785Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171632024-02-28T22:19:43.211Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085069953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171642024-02-28T22:19:43.211Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085147271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171652024-02-28T22:19:43.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08519112Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171662024-02-28T22:19:43.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085250638Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171672024-02-28T22:19:43.211Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085292247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171682024-02-28T22:19:43.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085860651Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171692024-02-28T22:19:43.211Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08590816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171702024-02-28T22:19:43.211Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085939829Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171712024-02-28T22:19:43.212Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.085987678Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171722024-02-28T22:19:43.212Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.086030786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171732024-02-28T22:19:43.212Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.086067955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171742024-02-28T22:19:43.212Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.086438725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171752024-02-28T22:19:43.212Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.086524503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171762024-02-28T22:19:43.212Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.086645319Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171772024-02-28T22:19:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087125746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171782024-02-28T22:19:43.212Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087168775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171792024-02-28T22:19:43.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087193274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171802024-02-28T22:19:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087244983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171812024-02-28T22:19:43.212Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087270762Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171822024-02-28T22:19:43.212Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087285722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171832024-02-28T22:19:43.212Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087306421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171842024-02-28T22:19:43.212Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087321611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171852024-02-28T22:19:43.212Z{"msg":"[0]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 287, 287, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08734323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171862024-02-28T22:19:43.212Z{"msg":"[0]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 158, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.08735803Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171872024-02-28T22:19:43.212Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087379619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171882024-02-28T22:19:43.212Z{"msg":"[1]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 287, 287, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087395139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171892024-02-28T22:19:43.212Z{"msg":"[1]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 158, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087416118Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171902024-02-28T22:19:43.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087431438Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171912024-02-28T22:19:43.212Z{"msg":"[2]R flush_numbers[0..12]: [268, 269, 285, 282, 270, 284, 274, 281, 287, 287, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087445947Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171922024-02-28T22:19:43.212Z{"msg":"[2]R generation[0..12]: [148, 148, 157, 156, 148, 156, 151, 155, 158, 158, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087467567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171932024-02-28T22:19:43.212Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
171942024-02-28T22:19:43.213Z,"time":"2024-02-28T22:19:42.087482136Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171952024-02-28T22:19:43.213Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087497436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171962024-02-28T22:19:43.213Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087511385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171972024-02-28T22:19:43.213Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087525695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171982024-02-28T22:19:43.213Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087539125Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"":"downstairs","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
171992024-02-28T22:19:43.213Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087553074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172002024-02-28T22:19:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087565814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"0","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172012024-02-28T22:19:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087579714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"1","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172022024-02-28T22:19:43.213Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087593023Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"client":"2","":"downstairs","session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172032024-02-28T22:19:43.213Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087614843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172042024-02-28T22:19:43.213Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087628472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172052024-02-28T22:19:43.213Z{"msg":"8f86733a-0211-4ddc-b2a7-657ee430f6d6 is now active with session: 4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087641522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172062024-02-28T22:19:43.213Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087661151Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775,"session_id":"4b9b8efd-4cfb-48e7-8ac8-cf8ae1a6d5f5"}
172072024-02-28T22:19:43.213Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.087674061Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
172082024-02-28T22:19:43.213ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172092024-02-28T22:19:43.213ZLoading write count information from file "/tmp/test_repair_verify.data"
172102024-02-28T22:19:43.213ZRun Repair workload
172112024-02-28T22:19:43.217Z01/30 Write block 242 len 40960 data: 8 11 11 13 9 16 18 14 14 14
172122024-02-28T22:19:43.217Z02/30 Write block 20 len 8192 data: 9 8
172132024-02-28T22:19:43.217Z03/30 Read block 543 len 28672
172142024-02-28T22:19:43.217Z04/30 Write block 439 len 8192 data: 19 18
172152024-02-28T22:19:43.217Z05/30 Read block 322 len 32768
172162024-02-28T22:19:43.217Z06/30 Read block 69 len 4096
172172024-02-28T22:19:43.217Z07/30 Write block 328 len 20480 data: 20 20 19 17 12
172182024-02-28T22:19:43.217Z08/30 Write block 187 len 12288 data: 10 10 12
172192024-02-28T22:19:43.217Z09/30 Read block 59 len 24576
172202024-02-28T22:19:43.217Z10/30 Read block 519 len 16384
172212024-02-28T22:19:43.217Z11/30 Write block 165 len 24576 data: 13 13 17 16 16 17
172222024-02-28T22:19:43.217Z12/30 Read block 1 len 28672
172232024-02-28T22:19:43.218Z13/30 Read block 180 len 32768
172242024-02-28T22:19:43.221Z14/30 Write block 431 len 36864 data: 9 12 13 16 17 19 18 19 20
172252024-02-28T22:19:43.221Z15/30 Read block 446 len 20480
172262024-02-28T22:19:43.221Z16/30 Write block 444 len 8192 data: 13 13
172272024-02-28T22:19:43.221Z17/30 Write block 348 len 28672 data: 11 10 11 12 13 14 14
172282024-02-28T22:19:43.224Z18/30 Read block 471 len 36864
172292024-02-28T22:19:43.224Z19/30 Read block 167 len 8192
172302024-02-28T22:19:43.224Z20/30 Read block 432 len 36864
172312024-02-28T22:19:43.227Z21/30 Flush
172322024-02-28T22:19:43.230Z22/30 Write block 126 len 28672 data: 17 20 23 19 18 18 17
172332024-02-28T22:19:43.230Z23/30 Write block 323 len 4096 data: 11
172342024-02-28T22:19:43.230Z24/30 Flush
172352024-02-28T22:19:43.233Z25/30 Write block 96 len 8192 data: 11 12
172362024-02-28T22:19:43.233Z26/30 Read block 206 len 8192
172372024-02-28T22:19:43.233Z27/30 Write block 180 len 20480 data: 7 7 9 11 12
172382024-02-28T22:19:43.237Z28/30 Read block 455 len 12288
172392024-02-28T22:19:43.237Z29/30 Read block 575 len 32768
172402024-02-28T22:19:43.237Z30/30 Read block 115 len 12288
172412024-02-28T22:19:43.237Z----------------------------------------------------------------
172422024-02-28T22:19:43.237Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172432024-02-28T22:19:43.237ZStates: Active Active Active
172442024-02-28T22:19:43.237ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172452024-02-28T22:19:43.237Z 1 Acked 1000 Write 10 Done Done Done false
172462024-02-28T22:19:43.238Z 2 Acked 1001 Write 2 Done Done Done false
172472024-02-28T22:19:43.238Z 3 Acked 1002 Read 7 Done Done Done false
172482024-02-28T22:19:43.238Z 4 Acked 1003 Write 2 Done Sent Done false
172492024-02-28T22:19:43.238Z 5 Acked 1004 Read 8 Done Sent Done false
172502024-02-28T22:19:43.238Z 6 Acked 1005 Read 1 Done Sent Done false
172512024-02-28T22:19:43.238Z 7 Acked 1006 Write 5 Done Sent Done false
172522024-02-28T22:19:43.238Z 8 Acked 1007 Write 3 Done Sent Done false
172532024-02-28T22:19:43.238Z 9 Acked 1008 Read 6 Done Sent Done false
172542024-02-28T22:19:43.238Z 10 Acked 1009 Read 4 Done Sent Done false
172552024-02-28T22:19:43.239Z 11 Acked 1010 Write 6 Done Sent Done false
172562024-02-28T22:19:43.239Z 12 Acked 1011 Read 7 Done Sent Done false
172572024-02-28T22:19:43.239Z 13 Acked 1012 Read 8 Done Sent Done false
172582024-02-28T22:19:43.239Z 14 Acked 1013 Write 9 Done Sent Done false
172592024-02-28T22:19:43.239Z 15 Acked 1014 Read 5 Done Sent Done false
172602024-02-28T22:19:43.239Z 16 Acked 1015 Write 2 Done Sent Done false
172612024-02-28T22:19:43.239Z 17 Acked 1016 Write 7 Done Sent Done false
172622024-02-28T22:19:43.239Z 18 Acked 1017 Read 9 Done Sent Done false
172632024-02-28T22:19:43.239Z 19 Acked 1018 Read 2 Done Sent Done false
172642024-02-28T22:19:43.239Z 20 Acked 1019 Read 9 Done Sent Done false
172652024-02-28T22:19:43.239Z 21 Acked 1020 Flush 0 Done Sent Done false
172662024-02-28T22:19:43.240Z 22 Acked 1021 Write 7 Done Sent Done false
172672024-02-28T22:19:43.240Z 23 Acked 1022 Write 1 Done Sent Done false
172682024-02-28T22:19:43.240Z 24 Acked 1023 Flush 0 Done Sent Done false
172692024-02-28T22:19:43.240Z 25 Acked 1024 Write 2 Done Sent Done false
172702024-02-28T22:19:43.240Z 26 Acked 1025 Read 2 Done Sent Done false
172712024-02-28T22:19:43.240Z 27 Acked 1026 Write 5 Done Sent Done false
172722024-02-28T22:19:43.240Z 28 Acked 1027 Read 3 Done Sent Done false
172732024-02-28T22:19:43.240Z 29 Acked 1028 Read 8 Done Sent Done false
172742024-02-28T22:19:43.240Z 30 Acked 1029 Read 3 Done Sent Done false
172752024-02-28T22:19:43.240Z STATES DS:0 DS:1 DS:2 TOTAL
172762024-02-28T22:19:43.240Z New 0 0 0 0
172772024-02-28T22:19:43.240Z Sent 0 27 0 27
172782024-02-28T22:19:43.240Z Done 30 3 30 63
172792024-02-28T22:19:43.241Z Skipped 0 0 0 0
172802024-02-28T22:19:43.241Z Error 0 0 0 0
172812024-02-28T22:19:43.241ZLast Flush: 1023 0 1023
172822024-02-28T22:19:43.241ZDownstairs last five completed:
172832024-02-28T22:19:43.241ZUpstairs last five completed: 30 29 28 27 26
172842024-02-28T22:19:43.244Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:42.118503292Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1775}
172852024-02-28T22:19:43.245ZWrote out file "/tmp/test_repair_verify.data"
172862024-02-28T22:19:43.245Z
172872024-02-28T22:19:43.245ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172882024-02-28T22:19:43.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172892024-02-28T22:19:43.260Z 0 000-019 148 148 148 268 268 268 F F F
172902024-02-28T22:19:43.260Z 1 020-039 160 148 160 291 269 291 F T F <---
172912024-02-28T22:19:43.260Z 2 040-059 157 157 157 285 285 285 F F F
172922024-02-28T22:19:43.260Z 3 060-079 156 156 156 282 282 282 F F F
172932024-02-28T22:19:43.260Z 4 080-099 148 148 148 270 270 270 T F T <---
172942024-02-28T22:19:43.260Z 5 100-119 156 156 156 284 284 284 F F F
172952024-02-28T22:19:43.260Z 6 120-139 160 151 160 292 274 292 F F F <---
172962024-02-28T22:19:43.260Z 7 140-159 155 155 155 281 281 281 F F F
172972024-02-28T22:19:43.260Z 8 160-179 160 158 160 291 287 291 F F F <---
172982024-02-28T22:19:43.260Z 9 180-199 160 158 160 291 287 291 T F T <---
172992024-02-28T22:19:43.260Z 10 200-219 158 158 158 287 287 287 F F F
173002024-02-28T22:19:43.260Z 11 220-239 158 158 158 287 287 287 F F F
173012024-02-28T22:19:43.265Z 12 240-259 160 158 160 291 289 291 F T F <---
173022024-02-28T22:19:43.265Z 13 260-279 158 158 158 289 289 289 F F F
173032024-02-28T22:19:43.265Z 14 280-299 145 145 145 262 262 262 F F F
173042024-02-28T22:19:43.266Z 15 300-319 158 158 158 289 289 289 F F F
173052024-02-28T22:19:43.266Z 16 320-339 160 155 160 292 281 292 F F F <---
173062024-02-28T22:19:43.266Z 17 340-359 160 151 160 291 274 291 F F F <---
173072024-02-28T22:19:43.266Z 18 360-379 148 148 148 268 268 268 F F F
173082024-02-28T22:19:43.266Z 19 380-399 156 156 156 282 282 282 F F F
173092024-02-28T22:19:43.266Z 20 400-419 153 153 153 278 278 278 F F F
173102024-02-28T22:19:43.266Z 21 420-439 160 159 160 291 290 291 F F F <---
173112024-02-28T22:19:43.266Z 22 440-459 160 156 160 291 284 291 F F F <---
173122024-02-28T22:19:43.266Z 23 460-479 159 159 159 290 290 290 F F F
173132024-02-28T22:19:43.266Z 24 480-499 158 158 158 289 289 289 F F F
173142024-02-28T22:19:43.266Z 25 500-519 159 159 159 290 290 290 F F F
173152024-02-28T22:19:43.266Z 26 520-539 159 159 159 290 290 290 F F F
173162024-02-28T22:19:43.266Z 27 540-559 159 159 159 290 290 290 F F F
173172024-02-28T22:19:43.266Z 28 560-579 158 158 158 289 289 289 F F F
173182024-02-28T22:19:43.266Z 29 580-599 156 156 156 284 284 284 F F F
173192024-02-28T22:19:43.266ZMax gen: 160, Max flush: 292
173202024-02-28T22:19:43.266ZError: Difference in extent metadata found!
173212024-02-28T22:19:43.266ZA Difference in extent metadata is expected here
173222024-02-28T22:19:43.267ZOn loop 80
173232024-02-28T22:19:43.267Z
173242024-02-28T22:19:43.282ZVerifying data now
173252024-02-28T22:19:43.797ZNo change after verify
173262024-02-28T22:19:43.797ZLoop: 80 Downstairs dump after verify (and repair):
173272024-02-28T22:19:43.811ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173282024-02-28T22:19:43.811Z 0 000-019 148 148 148 268 268 268 F F F
173292024-02-28T22:19:43.811Z 1 020-039 160 160 160 291 291 291 F F F
173302024-02-28T22:19:43.812Z 2 040-059 157 157 157 285 285 285 F F F
173312024-02-28T22:19:43.812Z 3 060-079 156 156 156 282 282 282 F F F
173322024-02-28T22:19:43.812Z 4 080-099 161 161 161 293 293 293 F F F
173332024-02-28T22:19:43.812Z 5 100-119 156 156 156 284 284 284 F F F
173342024-02-28T22:19:43.812Z 6 120-139 160 160 160 292 292 292 F F F
173352024-02-28T22:19:43.812Z 7 140-159 155 155 155 281 281 281 F F F
173362024-02-28T22:19:43.812Z 8 160-179 160 160 160 291 291 291 F F F
173372024-02-28T22:19:43.812Z 9 180-199 161 161 161 293 293 293 F F F
173382024-02-28T22:19:43.812Z 10 200-219 158 158 158 287 287 287 F F F
173392024-02-28T22:19:43.815Z 11 220-239 158 158 158 287 287 287 F F F
173402024-02-28T22:19:43.815Z 12 240-259 160 160 160 291 291 291 F F F
173412024-02-28T22:19:43.815Z 13 260-279 158 158 158 289 289 289 F F F
173422024-02-28T22:19:43.815Z 14 280-299 145 145 145 262 262 262 F F F
173432024-02-28T22:19:43.815Z 15 300-319 158 158 158 289 289 289 F F F
173442024-02-28T22:19:43.815Z 16 320-339 160 160 160 292 292 292 F F F
173452024-02-28T22:19:43.815Z 17 340-359 160 160 160 291 291 291 F F F
173462024-02-28T22:19:43.815Z 18 360-379 148 148 148 268 268 268 F F F
173472024-02-28T22:19:43.816Z 19 380-399 156 156 156 282 282 282 F F F
173482024-02-28T22:19:43.816Z 20 400-419 153 153 153 278 278 278 F F F
173492024-02-28T22:19:43.816Z 21 420-439 160 160 160 291 291 291 F F F
173502024-02-28T22:19:43.816Z 22 440-459 160 160 160 291 291 291 F F F
173512024-02-28T22:19:43.816Z 23 460-479 159 159 159 290 290 290 F F F
173522024-02-28T22:19:43.816Z 24 480-499 158 158 158 289 289 289 F F F
173532024-02-28T22:19:43.816Z 25 500-519 159 159 159 290 290 290 F F F
173542024-02-28T22:19:43.816Z 26 520-539 159 159 159 290 290 290 F F F
173552024-02-28T22:19:43.816Z 27 540-559 159 159 159 290 290 290 F F F
173562024-02-28T22:19:43.816Z 28 560-579 158 158 158 289 289 289 F F F
173572024-02-28T22:19:43.816Z 29 580-599 156 156 156 284 284 284 F F F
173582024-02-28T22:19:43.816ZMax gen: 161, Max flush: 293
173592024-02-28T22:19:43.816Z
173602024-02-28T22:19:43.816ZBegin loop 81
173612024-02-28T22:19:43.816ZDownstairs to restart: 0
173622024-02-28T22:19:43.834Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716352871Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
173632024-02-28T22:19:43.834Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716625484Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
173642024-02-28T22:19:43.840Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716657193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
173652024-02-28T22:19:43.840ZCrucible runtime is spawned
173662024-02-28T22:19:43.840Z{"msg":"Crucible f7f5e71a-f9ca-4885-ba48-c0adf846794b has session id: 80dcc5a1-de94-4aad-bcca-cf8965ea835c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716732041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173672024-02-28T22:19:43.840Z{"msg":"Upstairs opts: Upstairs UUID: f7f5e71a-f9ca-4885-ba48-c0adf846794b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716780969Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173682024-02-28T22:19:43.840Z{"msg":"Crucible stats registered with UUID: f7f5e71a-f9ca-4885-ba48-c0adf846794b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716815598Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173692024-02-28T22:19:43.840Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716838438Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
173702024-02-28T22:19:43.840Z{"msg":"f7f5e71a-f9ca-4885-ba48-c0adf846794b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.716967954Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173712024-02-28T22:19:43.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717006253Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173722024-02-28T22:19:43.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717026323Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173732024-02-28T22:19:43.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717050722Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173742024-02-28T22:19:43.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717071441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173752024-02-28T22:19:43.840Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.7171021Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173762024-02-28T22:19:43.840Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717145679Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173772024-02-28T22:19:43.840Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717184128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173782024-02-28T22:19:43.840Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717223457Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173792024-02-28T22:19:43.840Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717283455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173802024-02-28T22:19:43.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717337864Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173812024-02-28T22:19:43.840Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717370433Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173822024-02-28T22:19:43.841Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.717507759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173832024-02-28T22:19:43.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718078974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173842024-02-28T22:19:43.841Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718125342Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173852024-02-28T22:19:43.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718152362Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173862024-02-28T22:19:43.841Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718184181Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173872024-02-28T22:19:43.841Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718679427Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173882024-02-28T22:19:43.841Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718861882Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173892024-02-28T22:19:43.841Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718951489Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173902024-02-28T22:19:43.841Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.718987599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173912024-02-28T22:19:43.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719171814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173922024-02-28T22:19:43.841Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719229942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173932024-02-28T22:19:43.841Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719267141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173942024-02-28T22:19:43.841Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719388768Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173952024-02-28T22:19:43.841Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719430556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173962024-02-28T22:19:43.841Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.719446746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173972024-02-28T22:19:43.841Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.71965859Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173982024-02-28T22:19:43.846Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720159636Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
173992024-02-28T22:19:43.846Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720207785Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174002024-02-28T22:19:43.846Z{"msg":"[0]R flush_numbers[0..12]: [268, 291, 285, 282, 293, 284, 292, 281, 291, 293, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720226634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174012024-02-28T22:19:43.846Z{"msg":"[0]R generation[0..12]: [148, 160, 157, 156, 161, 156, 160, 155, 160, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720248944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174022024-02-28T22:19:43.846Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720264373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174032024-02-28T22:19:43.846Z{"msg":"[1]R flush_numbers[0..12]: [268, 291, 285, 282, 293, 284, 292, 281, 291, 293, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720285093Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174042024-02-28T22:19:43.846Z{"msg":"[1]R generation[0..12]: [148, 160, 157, 156, 161, 156, 160, 155, 160, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720299823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174052024-02-28T22:19:43.846Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720320862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174062024-02-28T22:19:43.846Z{"msg":"[2]R flush_numbers[0..12]: [268, 291, 285, 282, 293, 284, 292, 281, 291, 293, 287, 287]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
174072024-02-28T22:19:43.846Z,"time":"2024-02-28T22:19:42.720335122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174082024-02-28T22:19:43.846Z{"msg":"[2]R generation[0..12]: [148, 160, 157, 156, 161, 156, 160, 155, 160, 161, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720355311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174092024-02-28T22:19:43.846Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720369821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174102024-02-28T22:19:43.847Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.72039139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174112024-02-28T22:19:43.847Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720411349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174122024-02-28T22:19:43.847Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720444908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174132024-02-28T22:19:43.847Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720460718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"":"downstairs","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174142024-02-28T22:19:43.847Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720493557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174152024-02-28T22:19:43.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720513957Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"0","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174162024-02-28T22:19:43.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720546516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"1","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174172024-02-28T22:19:43.847Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720580245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"client":"2","":"downstairs","session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174182024-02-28T22:19:43.847Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720604744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174192024-02-28T22:19:43.847Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720638203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174202024-02-28T22:19:43.847Z{"msg":"f7f5e71a-f9ca-4885-ba48-c0adf846794b is now active with session: 80dcc5a1-de94-4aad-bcca-cf8965ea835c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720674222Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174212024-02-28T22:19:43.847Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174222024-02-28T22:19:43.847Z,"time":"2024-02-28T22:19:42.720697001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783,"session_id":"80dcc5a1-de94-4aad-bcca-cf8965ea835c"}
174232024-02-28T22:19:43.847Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:42.720724301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
174242024-02-28T22:19:43.847ZLoading write count information from file "/tmp/test_repair_verify.data"
174252024-02-28T22:19:43.847ZRun Repair workload
174262024-02-28T22:19:43.847Z01/30 Write block 301 len 16384 data: 7 9 12 12
174272024-02-28T22:19:43.847Z02/30 Flush
174282024-02-28T22:19:43.847Z03/30 Write block 343 len 12288 data: 11 11 10
174292024-02-28T22:19:43.847Z04/30 Flush
174302024-02-28T22:19:43.853Z05/30 Write block 145 len 24576 data: 15 16 15 16 14 12
174312024-02-28T22:19:43.853Z06/30 Write block 392 len 28672 data: 15 11 10 13 10 11 12
174322024-02-28T22:19:43.853Z07/30 Write block 145 len 20480 data: 16 17 16 17 15
174332024-02-28T22:19:43.853Z08/30 Read block 332 len 36864
174342024-02-28T22:19:43.853Z09/30 Read block 440 len 24576
174352024-02-28T22:19:43.853Z10/30 Read block 119 len 24576
174362024-02-28T22:19:43.853Z11/30 Read block 278 len 40960
174372024-02-28T22:19:43.853Z12/30 Read block 15 len 8192
174382024-02-28T22:19:43.854Z13/30 Read block 196 len 16384
174392024-02-28T22:19:43.857Z14/30 Write block 179 len 20480 data: 7 8 8 10 12
174402024-02-28T22:19:43.857Z15/30 Read block 345 len 40960
174412024-02-28T22:19:43.857Z16/30 Read block 365 len 20480
174422024-02-28T22:19:43.857Z17/30 Write block 453 len 28672 data: 19 17 16 15 13 13 10
174432024-02-28T22:19:43.857Z18/30 Flush
174442024-02-28T22:19:43.861Z19/30 Write block 46 len 28672 data: 8 10 11 13 11 13 8
174452024-02-28T22:19:43.864Z20/30 Write block 277 len 36864 data: 7 8 9 10 9 10 11 13 12
174462024-02-28T22:19:43.864Z21/30 Write block 354 len 28672 data: 15 15 12 13 11 9 8
174472024-02-28T22:19:43.864Z22/30 Read block 585 len 36864
174482024-02-28T22:19:43.867Z23/30 Read block 307 len 20480
174492024-02-28T22:19:43.867Z24/30 Read block 430 len 24576
174502024-02-28T22:19:43.867Z25/30 Write block 105 len 20480 data: 9 10 11 15 19
174512024-02-28T22:19:43.867Z26/30 Read block 246 len 4096
174522024-02-28T22:19:43.870Z27/30 Read block 150 len 24576
174532024-02-28T22:19:43.870Z28/30 Write block 527 len 36864 data: 9 7 9 8 8 9 11 14 11
174542024-02-28T22:19:43.870Z29/30 Read block 8 len 28672
174552024-02-28T22:19:43.874Z30/30 Read block 535 len 28672
174562024-02-28T22:19:43.874Z----------------------------------------------------------------
174572024-02-28T22:19:43.874Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:26
174582024-02-28T22:19:43.874ZStates: Active Active Active
174592024-02-28T22:19:43.874ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174602024-02-28T22:19:43.874Z 5 Acked 1004 Write 6 Done Done Done false
174612024-02-28T22:19:43.874Z 6 Acked 1005 Write 7 Done Done Done false
174622024-02-28T22:19:43.874Z 7 Acked 1006 Write 5 Sent Done Done false
174632024-02-28T22:19:43.874Z 8 Acked 1007 Read 9 Sent Done Done false
174642024-02-28T22:19:43.874Z 9 Acked 1008 Read 6 Sent Done Done false
174652024-02-28T22:19:43.874Z 10 Acked 1009 Read 6 Sent Done Done false
174662024-02-28T22:19:43.875Z 11 Acked 1010 Read 10 Sent Done Done false
174672024-02-28T22:19:43.875Z 12 Acked 1011 Read 2 Sent Done Done false
174682024-02-28T22:19:43.875Z 13 Acked 1012 Read 4 Sent Done Done false
174692024-02-28T22:19:43.875Z 14 Acked 1013 Write 5 Sent Done Done false
174702024-02-28T22:19:43.875Z 15 Acked 1014 Read 10 Sent Done Done false
174712024-02-28T22:19:43.875Z 16 Acked 1015 Read 5 Sent Done Done false
174722024-02-28T22:19:43.875Z 17 Acked 1016 Write 7 Sent Done Done false
174732024-02-28T22:19:43.875Z 18 Acked 1017 Flush 0 Sent Done Done false
174742024-02-28T22:19:43.875Z 19 Acked 1018 Write 7 Sent Done Done false
174752024-02-28T22:19:43.875Z 20 Acked 1019 Write 9 Sent Done Done false
174762024-02-28T22:19:43.875Z 21 Acked 1020 Write 7 Sent Done Done false
174772024-02-28T22:19:43.875Z 22 Acked 1021 Read 9 Sent Done Done false
174782024-02-28T22:19:43.875Z 23 Acked 1022 Read 5 Sent Done Done false
174792024-02-28T22:19:43.875Z 24 Acked 1023 Read 6 Sent Done Done false
174802024-02-28T22:19:43.875Z 25 Acked 1024 Write 5 Sent Done Done false
174812024-02-28T22:19:43.875Z 26 Acked 1025 Read 1 Sent Done Done false
174822024-02-28T22:19:43.875Z 27 Acked 1026 Read 6 Sent Done Done false
174832024-02-28T22:19:43.875Z 28 Acked 1027 Write 9 Sent Done Done false
174842024-02-28T22:19:43.875Z 29 Acked 1028 Read 7 Sent Done Done false
174852024-02-28T22:19:43.875Z 30 Acked 1029 Read 7 Sent Done Done false
174862024-02-28T22:19:43.875Z STATES DS:0 DS:1 DS:2 TOTAL
174872024-02-28T22:19:43.875Z New 0 0 0 0
174882024-02-28T22:19:43.876Z Sent 24 0 0 24
174892024-02-28T22:19:43.876Z Done 2 26 26 54
174902024-02-28T22:19:43.876Z Skipped 0 0 0 0
174912024-02-28T22:19:43.876Z Error 0 0 0 0
174922024-02-28T22:19:43.876ZLast Flush: 1003 1017 1017
174932024-02-28T22:19:43.876ZDownstairs last five completed: 1003 1002 1001 1000
174942024-02-28T22:19:43.876ZUpstairs last five completed: 30 29 28 27 26
174952024-02-28T22:19:43.876Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:42.75452642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1783}
174962024-02-28T22:19:43.876ZWrote out file "/tmp/test_repair_verify.data"
174972024-02-28T22:19:43.879Z
174982024-02-28T22:19:43.882ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
174992024-02-28T22:19:43.898ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175002024-02-28T22:19:43.899Z 0 000-019 148 148 148 268 268 268 F F F
175012024-02-28T22:19:43.899Z 1 020-039 160 160 160 291 291 291 F F F
175022024-02-28T22:19:43.899Z 2 040-059 157 157 157 285 285 285 F T T <---
175032024-02-28T22:19:43.899Z 3 060-079 156 156 156 282 282 282 F F F
175042024-02-28T22:19:43.899Z 4 080-099 161 161 161 293 293 293 F F F
175052024-02-28T22:19:43.899Z 5 100-119 156 156 156 284 284 284 F T T <---
175062024-02-28T22:19:43.899Z 6 120-139 160 160 160 292 292 292 F F F
175072024-02-28T22:19:43.899Z 7 140-159 155 162 162 281 296 296 T F F <---
175082024-02-28T22:19:43.899Z 8 160-179 160 162 162 291 296 296 F F F <---
175092024-02-28T22:19:43.899Z 9 180-199 161 162 162 293 296 296 F F F <---
175102024-02-28T22:19:43.902Z 10 200-219 158 158 158 287 287 287 F F F
175112024-02-28T22:19:43.902Z 11 220-239 158 158 158 287 287 287 F F F
175122024-02-28T22:19:43.902Z 12 240-259 160 160 160 291 291 291 F F F
175132024-02-28T22:19:43.903Z 13 260-279 158 158 158 289 289 289 F T T <---
175142024-02-28T22:19:43.903Z 14 280-299 145 145 145 262 262 262 F T T <---
175152024-02-28T22:19:43.903Z 15 300-319 162 162 162 294 294 294 F F F
175162024-02-28T22:19:43.903Z 16 320-339 160 160 160 292 292 292 F F F
175172024-02-28T22:19:43.903Z 17 340-359 162 162 162 295 295 295 F T T <---
175182024-02-28T22:19:43.903Z 18 360-379 148 148 148 268 268 268 F T T <---
175192024-02-28T22:19:43.903Z 19 380-399 156 162 162 282 296 296 T F F <---
175202024-02-28T22:19:43.903Z 20 400-419 153 153 153 278 278 278 F F F
175212024-02-28T22:19:43.903Z 21 420-439 160 160 160 291 291 291 F F F
175222024-02-28T22:19:43.903Z 22 440-459 160 162 162 291 296 296 F F F <---
175232024-02-28T22:19:43.903Z 23 460-479 159 159 159 290 290 290 F F F
175242024-02-28T22:19:43.903Z 24 480-499 158 158 158 289 289 289 F F F
175252024-02-28T22:19:43.903Z 25 500-519 159 159 159 290 290 290 F F F
175262024-02-28T22:19:43.903Z 26 520-539 159 159 159 290 290 290 F T T <---
175272024-02-28T22:19:43.903Z 27 540-559 159 159 159 290 290 290 F F F
175282024-02-28T22:19:43.903Z 28 560-579 158 158 158 289 289 289 F F F
175292024-02-28T22:19:43.903Z 29 580-599 156 156 156 284 284 284 F F F
175302024-02-28T22:19:43.903ZMax gen: 162, Max flush: 296
175312024-02-28T22:19:43.903ZError: Difference in extent metadata found!
175322024-02-28T22:19:43.903ZA Difference in extent metadata is expected here
175332024-02-28T22:19:43.903ZOn loop 81
175342024-02-28T22:19:43.903Z
175352024-02-28T22:19:43.909ZVerifying data now
175362024-02-28T22:19:49.521ZNo change after verify
175372024-02-28T22:19:49.521ZLoop: 81 Downstairs dump after verify (and repair):
175382024-02-28T22:19:49.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175392024-02-28T22:19:49.535Z 0 000-019 148 148 148 268 268 268 F F F
175402024-02-28T22:19:49.535Z 1 020-039 160 160 160 291 291 291 F F F
175412024-02-28T22:19:49.535Z 2 040-059 163 163 163 297 297 297 F F F
175422024-02-28T22:19:49.535Z 3 060-079 156 156 156 282 282 282 F F F
175432024-02-28T22:19:49.535Z 4 080-099 161 161 161 293 293 293 F F F
175442024-02-28T22:19:49.535Z 5 100-119 163 163 163 297 297 297 F F F
175452024-02-28T22:19:49.535Z 6 120-139 160 160 160 292 292 292 F F F
175462024-02-28T22:19:49.535Z 7 140-159 162 162 162 296 296 296 F F F
175472024-02-28T22:19:49.535Z 8 160-179 162 162 162 296 296 296 F F F
175482024-02-28T22:19:49.535Z 9 180-199 162 162 162 296 296 296 F F F
175492024-02-28T22:19:49.535Z 10 200-219 158 158 158 287 287 287 F F F
175502024-02-28T22:19:49.536Z 11 220-239 158 158 158 287 287 287 F F F
175512024-02-28T22:19:49.536Z 12 240-259 160 160 160 291 291 291 F F F
175522024-02-28T22:19:49.536Z 13 260-279 163 163 163 297 297 297 F F F
175532024-02-28T22:19:49.536Z 14 280-299 163 163 163 297 297 297 F F F
175542024-02-28T22:19:49.539Z 15 300-319 162 162 162 294 294 294 F F F
175552024-02-28T22:19:49.539Z 16 320-339 160 160 160 292 292 292 F F F
175562024-02-28T22:19:49.539Z 17 340-359 163 163 163 297 297 297 F F F
175572024-02-28T22:19:49.540Z 18 360-379 163 163 163 297 297 297 F F F
175582024-02-28T22:19:49.540Z 19 380-399 162 162 162 296 296 296 F F F
175592024-02-28T22:19:49.540Z 20 400-419 153 153 153 278 278 278 F F F
175602024-02-28T22:19:49.540Z 21 420-439 160 160 160 291 291 291 F F F
175612024-02-28T22:19:49.540Z 22 440-459 162 162 162 296 296 296 F F F
175622024-02-28T22:19:49.540Z 23 460-479 159 159 159 290 290 290 F F F
175632024-02-28T22:19:49.540Z 24 480-499 158 158 158 289 289 289 F F F
175642024-02-28T22:19:49.540Z 25 500-519 159 159 159 290 290 290 F F F
175652024-02-28T22:19:49.540Z 26 520-539 163 163 163 297 297 297 F F F
175662024-02-28T22:19:49.540Z 27 540-559 159 159 159 290 290 290 F F F
175672024-02-28T22:19:49.540Z 28 560-579 158 158 158 289 289 289 F F F
175682024-02-28T22:19:49.540Z 29 580-599 156 156 156 284 284 284 F F F
175692024-02-28T22:19:49.540ZMax gen: 163, Max flush: 297
175702024-02-28T22:19:49.540Z
175712024-02-28T22:19:49.540ZBegin loop 82
175722024-02-28T22:19:49.540ZDownstairs to restart: 2
175732024-02-28T22:19:49.557Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.43820605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
175742024-02-28T22:19:49.557Z{"Crucible runtime is spawned
175752024-02-28T22:19:49.564Zmsg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.43857256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
175762024-02-28T22:19:49.564Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438620079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
175772024-02-28T22:19:49.564Z{"msg":"Crucible 7a8320fe-1d65-4812-841a-7d86cb58092b has session id: 1e20059a-0b02-474c-a228-343d59b1a146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438697667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175782024-02-28T22:19:49.564Z{"msg":"Upstairs opts: Upstairs UUID: 7a8320fe-1d65-4812-841a-7d86cb58092b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438743675Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175792024-02-28T22:19:49.564Z{"msg":"Crucible stats registered with UUID: 7a8320fe-1d65-4812-841a-7d86cb58092b","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438783345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175802024-02-28T22:19:49.564Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438812094Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
175812024-02-28T22:19:49.564Z{"msg":"7a8320fe-1d65-4812-841a-7d86cb58092b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438837773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175822024-02-28T22:19:49.564Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438875582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175832024-02-28T22:19:49.564Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438911661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175842024-02-28T22:19:49.564Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.43895249Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175852024-02-28T22:19:49.564Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.438980529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175862024-02-28T22:19:49.564Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439015938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175872024-02-28T22:19:49.564Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439058707Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175882024-02-28T22:19:49.564Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439092256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175892024-02-28T22:19:49.564Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439148044Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175902024-02-28T22:19:49.564Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439176194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175912024-02-28T22:19:49.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439210843Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175922024-02-28T22:19:49.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439245212Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175932024-02-28T22:19:49.564Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439269341Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175942024-02-28T22:19:49.564Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439944002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175952024-02-28T22:19:49.565Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.439982581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175962024-02-28T22:19:49.565Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.440000891Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175972024-02-28T22:19:49.565Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.44001576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175982024-02-28T22:19:49.565Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.44004445Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
175992024-02-28T22:19:49.565Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.440070249Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176002024-02-28T22:19:49.565Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.440461888Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176012024-02-28T22:19:49.565Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.440539836Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176022024-02-28T22:19:49.565Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.440822308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176032024-02-28T22:19:49.565Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441025313Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176042024-02-28T22:19:49.565Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.44110855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176052024-02-28T22:19:49.565Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441187998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176062024-02-28T22:19:49.565Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441224657Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176072024-02-28T22:19:49.565Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441246197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176082024-02-28T22:19:49.565Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441276216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176092024-02-28T22:19:49.565Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441333834Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176102024-02-28T22:19:49.565Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441376413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176112024-02-28T22:19:49.565Z{"msg":"[0]R flush_numbers[0..12]: [268, 291, 297, 282, 293, 297, 292, 296, 296, 296, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441396162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176122024-02-28T22:19:49.565Z{"msg":"[0]R generation[0..12]: [148, 160, 163, 156, 161, 163, 160, 162, 162, 162, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441423522Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176132024-02-28T22:19:49.565Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441438911Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176142024-02-28T22:19:49.565Z{"msg":"[1]R flush_numbers[0..12]: [268, 291, 297, 282, 293, 297, 292, 296, 296, 296, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441460691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176152024-02-28T22:19:49.565Z{"msg":"[1]R generation[0..12]: [148, 160, 163, 156, 161, 163, 160, 162, 162, 162, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.44147576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176162024-02-28T22:19:49.565Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.44149093Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176172024-02-28T22:19:49.565Z{"msg":"[2]R flush_numbers[0..12]: [268, 291, 297, 282, 293, 297, 292, 296, 296, 296, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441505539Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176182024-02-28T22:19:49.566Z{"msg":"[2]R generation[0..12]: [148, 160, 163, 156, 161, 163, 160, 162, 162, 162, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441523169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176192024-02-28T22:19:49.566ZWait for a query_work_queue command to finish before sending IO
176202024-02-28T22:19:49.566Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441553418Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176212024-02-28T22:19:49.566Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441575837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176222024-02-28T22:19:49.566Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441590057Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176232024-02-28T22:19:49.566Z{"msg":"Next flush: 298","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441603717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176242024-02-28T22:19:49.566Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441617826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"":"downstairs","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176252024-02-28T22:19:49.566Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441631246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176262024-02-28T22:19:49.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441659585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"0","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176272024-02-28T22:19:49.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441681325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"1","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176282024-02-28T22:19:49.566Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441713314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"client":"2","":"downstairs","session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176292024-02-28T22:19:49.566Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441731673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176302024-02-28T22:19:49.566Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441763182Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176312024-02-28T22:19:49.566Z{"msg":"7a8320fe-1d65-4812-841a-7d86cb58092b is now active with session: 1e20059a-0b02-474c-a228-343d59b1a146","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441785002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176322024-02-28T22:19:49.566Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441806691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791,"session_id":"1e20059a-0b02-474c-a228-343d59b1a146"}
176332024-02-28T22:19:49.566Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:48.441826741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
176342024-02-28T22:19:49.570ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176352024-02-28T22:19:49.570ZLoading write count information from file "/tmp/test_repair_verify.data"
176362024-02-28T22:19:49.570ZRun Repair workload
176372024-02-28T22:19:49.570Z01/30 Write block 253 len 4096 data: 13
176382024-02-28T22:19:49.570Z02/30 Write block 55 len 12288 data: 9 12 13
176392024-02-28T22:19:49.570Z03/30 Write block 192 len 4096 data: 13
176402024-02-28T22:19:49.570Z04/30 Write block 14 len 12288 data: 11 11 12
176412024-02-28T22:19:49.570Z05/30 Read block 477 len 20480
176422024-02-28T22:19:49.570Z06/30 Write block 187 len 16384 data: 11 11 13 10
176432024-02-28T22:19:49.570Z07/30 Read block 53 len 28672
176442024-02-28T22:19:49.570Z08/30 Write block 394 len 16384 data: 11 14 11 12
176452024-02-28T22:19:49.570Z09/30 Write block 571 len 28672 data: 11 11 13 11 12 11 12
176462024-02-28T22:19:49.570Z10/30 Read block 427 len 24576
176472024-02-28T22:19:49.571Z11/30 Flush
176482024-02-28T22:19:49.574Z12/30 Write block 324 len 8192 data: 14 14
176492024-02-28T22:19:49.574Z13/30 Flush
176502024-02-28T22:19:49.580Z14/30 Flush
176512024-02-28T22:19:49.580Z15/30 Write block 341 len 32768 data: 11 12 12 12 11 9 10 12
176522024-02-28T22:19:49.580Z16/30 Read block 315 len 32768
176532024-02-28T22:19:49.583Z17/30 Write block 58 len 20480 data: 12 15 16 15 16
176542024-02-28T22:19:49.583Z18/30 Read block 508 len 32768
176552024-02-28T22:19:49.583Z19/30 Read block 493 len 28672
176562024-02-28T22:19:49.583Z20/30 Write block 114 len 20480 data: 17 16 16 15 13
176572024-02-28T22:19:49.583Z21/30 Write block 499 len 16384 data: 12 13 14 15
176582024-02-28T22:19:49.583Z22/30 Write block 110 len 20480 data: 17 17 17 16 18
176592024-02-28T22:19:49.583Z23/30 Write block 115 len 32768 data: 17 17 16 14 11 11 13 14
176602024-02-28T22:19:49.583Z24/30 Read block 115 len 20480
176612024-02-28T22:19:49.586Z25/30 Read block 242 len 8192
176622024-02-28T22:19:49.586Z26/30 Read block 81 len 24576
176632024-02-28T22:19:49.586Z27/30 Read block 146 len 28672
176642024-02-28T22:19:49.589Z28/30 Read block 183 len 16384
176652024-02-28T22:19:49.589Z29/30 Write block 351 len 16384 data: 13 14 15 16
176662024-02-28T22:19:49.589Z30/30 Write block 365 len 12288 data: 9 10 10
176672024-02-28T22:19:49.589Z----------------------------------------------------------------
176682024-02-28T22:19:49.589Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
176692024-02-28T22:19:49.590ZStates: Active Active Active
176702024-02-28T22:19:49.590ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
176712024-02-28T22:19:49.590Z 1 Acked 1000 Write 1 Done Done Done false
176722024-02-28T22:19:49.590Z 2 Acked 1001 Write 3 Done Done Sent false
176732024-02-28T22:19:49.590Z 3 Acked 1002 Write 1 Done Done Sent false
176742024-02-28T22:19:49.590Z 4 Acked 1003 Write 3 Done Done Sent false
176752024-02-28T22:19:49.590Z 5 Acked 1004 Read 5 Done Done Sent false
176762024-02-28T22:19:49.590Z 6 Acked 1005 Write 4 Done Done Sent false
176772024-02-28T22:19:49.590Z 7 Acked 1006 Read 7 Done Done Sent false
176782024-02-28T22:19:49.590Z 8 Acked 1007 Write 4 Done Done Sent false
176792024-02-28T22:19:49.590Z 9 Acked 1008 Write 7 Done Done Sent false
176802024-02-28T22:19:49.590Z 10 Acked 1009 Read 6 Done Done Sent false
176812024-02-28T22:19:49.590Z 11 Acked 1010 Flush 0 Done Done Sent false
176822024-02-28T22:19:49.590Z 12 Acked 1011 Write 2 Done Done Sent false
176832024-02-28T22:19:49.590Z 13 Acked 1012 Flush 0 Done Done Sent false
176842024-02-28T22:19:49.590Z 14 Acked 1013 Flush 0 Done Done Sent false
176852024-02-28T22:19:49.590Z 15 Acked 1014 Write 8 Done Done Sent false
176862024-02-28T22:19:49.590Z 16 Acked 1015 Read 8 Done Done Sent false
176872024-02-28T22:19:49.590Z 17 Acked 1016 Write 5 Done Done Sent false
176882024-02-28T22:19:49.590Z 18 Acked 1017 Read 8 Done Done Sent false
176892024-02-28T22:19:49.590Z 19 Acked 1018 Read 7 Done Done Sent false
176902024-02-28T22:19:49.590Z 20 Acked 1019 Write 5 Done Done Sent false
176912024-02-28T22:19:49.590Z 21 Acked 1020 Write 4 Done Done Sent false
176922024-02-28T22:19:49.591Z 22 Acked 1021 Write 5 Done Done Sent false
176932024-02-28T22:19:49.591Z 23 Acked 1022 Write 8 Done Done Sent false
176942024-02-28T22:19:49.591Z 24 Acked 1023 Read 5 Done Done Sent false
176952024-02-28T22:19:49.591Z 25 Acked 1024 Read 2 Done Done Sent false
176962024-02-28T22:19:49.591Z 26 Acked 1025 Read 6 Done Done Sent false
176972024-02-28T22:19:49.591Z 27 Acked 1026 Read 7 Done Done Sent false
176982024-02-28T22:19:49.591Z 28 Acked 1027 Read 4 Done Done Sent false
176992024-02-28T22:19:49.591Z 29 Acked 1028 Write 4 Done Sent Sent false
177002024-02-28T22:19:49.591Z 30 Acked 1029 Write 3 Sent Sent Sent false
177012024-02-28T22:19:49.591Z STATES DS:0 DS:1 DS:2 TOTAL
177022024-02-28T22:19:49.591Z New 0 0 0 0
177032024-02-28T22:19:49.591Z Sent 1 2 29 32
177042024-02-28T22:19:49.591Z Done 29 28 1 58
177052024-02-28T22:19:49.591Z Skipped 0 0 0 0
177062024-02-28T22:19:49.591Z Error 0 0 0 0
177072024-02-28T22:19:49.591ZLast Flush: 1013 1013 0
177082024-02-28T22:19:49.591ZDownstairs last five completed:
177092024-02-28T22:19:49.591ZUpstairs last five completed: 30 29 28 27 26
177102024-02-28T22:19:49.591Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:48.470211159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1791}
177112024-02-28T22:19:49.591ZWrote out file "/tmp/test_repair_verify.data"
177122024-02-28T22:19:49.595Z
177132024-02-28T22:19:49.598ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177142024-02-28T22:19:49.614ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177152024-02-28T22:19:49.614Z 0 000-019 164 164 148 298 298 268 F F F <---
177162024-02-28T22:19:49.614Z 1 020-039 160 160 160 291 291 291 F F F
177172024-02-28T22:19:49.614Z 2 040-059 164 164 163 298 298 297 T T F <---
177182024-02-28T22:19:49.614Z 3 060-079 156 156 156 282 282 282 T T F <---
177192024-02-28T22:19:49.614Z 4 080-099 161 161 161 293 293 293 F F F
177202024-02-28T22:19:49.614Z 5 100-119 163 163 163 297 297 297 T T F <---
177212024-02-28T22:19:49.614Z 6 120-139 160 160 160 292 292 292 T T F <---
177222024-02-28T22:19:49.614Z 7 140-159 162 162 162 296 296 296 F F F
177232024-02-28T22:19:49.614Z 8 160-179 162 162 162 296 296 296 F F F
177242024-02-28T22:19:49.614Z 9 180-199 164 164 162 298 298 296 F F F <---
177252024-02-28T22:19:49.614Z 10 200-219 158 158 158 287 287 287 F F F
177262024-02-28T22:19:49.614Z 11 220-239 158 158 158 287 287 287 F F F
177272024-02-28T22:19:49.614Z 12 240-259 164 164 160 298 298 291 F F T <---
177282024-02-28T22:19:49.617Z 13 260-279 163 163 163 297 297 297 F F F
177292024-02-28T22:19:49.617Z 14 280-299 163 163 163 297 297 297 F F F
177302024-02-28T22:19:49.618Z 15 300-319 162 162 162 294 294 294 F F F
177312024-02-28T22:19:49.618Z 16 320-339 164 164 160 299 299 292 F F F <---
177322024-02-28T22:19:49.618Z 17 340-359 163 163 163 297 297 297 T T F <---
177332024-02-28T22:19:49.618Z 18 360-379 163 163 163 297 297 297 T T F <---
177342024-02-28T22:19:49.618Z 19 380-399 164 164 162 298 298 296 F F F <---
177352024-02-28T22:19:49.618Z 20 400-419 153 153 153 278 278 278 F F F
177362024-02-28T22:19:49.618Z 21 420-439 160 160 160 291 291 291 F F F
177372024-02-28T22:19:49.618Z 22 440-459 162 162 162 296 296 296 F F F
177382024-02-28T22:19:49.618Z 23 460-479 159 159 159 290 290 290 F F F
177392024-02-28T22:19:49.618Z 24 480-499 158 158 158 289 289 289 T T F <---
177402024-02-28T22:19:49.618Z 25 500-519 159 159 159 290 290 290 T T F <---
177412024-02-28T22:19:49.618Z 26 520-539 163 163 163 297 297 297 F F F
177422024-02-28T22:19:49.618Z 27 540-559 159 159 159 290 290 290 F F F
177432024-02-28T22:19:49.618Z 28 560-579 164 164 158 298 298 289 F F F <---
177442024-02-28T22:19:49.618Z 29 580-599 156 156 156 284 284 284 F F F
177452024-02-28T22:19:49.618ZMax gen: 164, Max flush: 299
177462024-02-28T22:19:49.618ZError: Difference in extent metadata found!
177472024-02-28T22:19:49.618ZA Difference in extent metadata is expected here
177482024-02-28T22:19:49.618ZOn loop 82
177492024-02-28T22:19:49.618Z
177502024-02-28T22:19:49.622ZVerifying data now
177512024-02-28T22:19:54.344ZNo change after verify
177522024-02-28T22:19:54.344ZLoop: 82 Downstairs dump after verify (and repair):
177532024-02-28T22:19:54.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177542024-02-28T22:19:54.359Z 0 000-019 164 164 164 298 298 298 F F F
177552024-02-28T22:19:54.359Z 1 020-039 160 160 160 291 291 291 F F F
177562024-02-28T22:19:54.359Z 2 040-059 165 165 165 300 300 300 F F F
177572024-02-28T22:19:54.359Z 3 060-079 165 165 165 300 300 300 F F F
177582024-02-28T22:19:54.359Z 4 080-099 161 161 161 293 293 293 F F F
177592024-02-28T22:19:54.359Z 5 100-119 165 165 165 300 300 300 F F F
177602024-02-28T22:19:54.360Z 6 120-139 165 165 165 300 300 300 F F F
177612024-02-28T22:19:54.360Z 7 140-159 162 162 162 296 296 296 F F F
177622024-02-28T22:19:54.360Z 8 160-179 162 162 162 296 296 296 F F F
177632024-02-28T22:19:54.360Z 9 180-199 164 164 164 298 298 298 F F F
177642024-02-28T22:19:54.360Z 10 200-219 158 158 158 287 287 287 F F F
177652024-02-28T22:19:54.363Z 11 220-239 158 158 158 287 287 287 F F F
177662024-02-28T22:19:54.363Z 12 240-259 164 164 164 298 298 298 F F F
177672024-02-28T22:19:54.363Z 13 260-279 163 163 163 297 297 297 F F F
177682024-02-28T22:19:54.363Z 14 280-299 163 163 163 297 297 297 F F F
177692024-02-28T22:19:54.363Z 15 300-319 162 162 162 294 294 294 F F F
177702024-02-28T22:19:54.363Z 16 320-339 164 164 164 299 299 299 F F F
177712024-02-28T22:19:54.363Z 17 340-359 165 165 165 300 300 300 F F F
177722024-02-28T22:19:54.363Z 18 360-379 165 165 165 300 300 300 F F F
177732024-02-28T22:19:54.363Z 19 380-399 164 164 164 298 298 298 F F F
177742024-02-28T22:19:54.363Z 20 400-419 153 153 153 278 278 278 F F F
177752024-02-28T22:19:54.364Z 21 420-439 160 160 160 291 291 291 F F F
177762024-02-28T22:19:54.364Z 22 440-459 162 162 162 296 296 296 F F F
177772024-02-28T22:19:54.364Z 23 460-479 159 159 159 290 290 290 F F F
177782024-02-28T22:19:54.364Z 24 480-499 165 165 165 300 300 300 F F F
177792024-02-28T22:19:54.364Z 25 500-519 165 165 165 300 300 300 F F F
177802024-02-28T22:19:54.364Z 26 520-539 163 163 163 297 297 297 F F F
177812024-02-28T22:19:54.364Z 27 540-559 159 159 159 290 290 290 F F F
177822024-02-28T22:19:54.364Z 28 560-579 164 164 164 298 298 298 F F F
177832024-02-28T22:19:54.364Z 29 580-599 156 156 156 284 284 284 F F F
177842024-02-28T22:19:54.364ZMax gen: 165, Max flush: 300
177852024-02-28T22:19:54.364Z
177862024-02-28T22:19:54.364ZBegin loop 83
177872024-02-28T22:19:54.364ZDownstairs to restart: 2
177882024-02-28T22:19:54.382Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.262751139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
177892024-02-28T22:19:54.382Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
177902024-02-28T22:19:54.382Z,"time":"2024-02-28T22:19:53.26308758Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
177912024-02-28T22:19:54.390Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263135178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
177922024-02-28T22:19:54.390Z{"msg":"Crucible 2522f112-fd54-44d2-9030-8c5779dbc355 has session id: 0b360105-7ace-4ab0-b713-b1fda61d60db","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263171837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177932024-02-28T22:19:54.390Z{"msg":"Upstairs opts: Upstairs UUID: 2522f112-fd54-44d2-9030-8c5779dbc355, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263213126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177942024-02-28T22:19:54.390Z{"msg":"Crucible stats registered with UUID: 2522f112-fd54-44d2-9030-8c5779dbc355","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263246795Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177952024-02-28T22:19:54.390Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263264455Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
177962024-02-28T22:19:54.390Z{"msg":"2522f112-fd54-44d2-9030-8c5779dbc355 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263300284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177972024-02-28T22:19:54.390Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263354892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177982024-02-28T22:19:54.390Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263385401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
177992024-02-28T22:19:54.390Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.26342959Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178002024-02-28T22:19:54.391Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263470579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178012024-02-28T22:19:54.391Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263530128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178022024-02-28T22:19:54.391Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263565687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178032024-02-28T22:19:54.391Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263591566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178042024-02-28T22:19:54.391Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263635385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178052024-02-28T22:19:54.391Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263675713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178062024-02-28T22:19:54.391Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263708163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178072024-02-28T22:19:54.391Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263734762Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178082024-02-28T22:19:54.391Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.263771221Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178092024-02-28T22:19:54.391Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.264466792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178102024-02-28T22:19:54.391Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.264504051Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178112024-02-28T22:19:54.391Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.26454001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178122024-02-28T22:19:54.391Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.264575299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178132024-02-28T22:19:54.391Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.264651826Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178142024-02-28T22:19:54.391Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.264695735Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178152024-02-28T22:19:54.391Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265097474Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178162024-02-28T22:19:54.391Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265140573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178172024-02-28T22:19:54.391Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.26524691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178182024-02-28T22:19:54.391Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265700528Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178192024-02-28T22:19:54.391Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265758906Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178202024-02-28T22:19:54.391Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265791495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178212024-02-28T22:19:54.391Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265812625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178222024-02-28T22:19:54.391Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265837934Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178232024-02-28T22:19:54.392Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265852224Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178242024-02-28T22:19:54.392Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265865383Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178252024-02-28T22:19:54.392Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265885952Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178262024-02-28T22:19:54.392Z{"msg":"[0]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 300, 296, 296, 298, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265900422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178272024-02-28T22:19:54.392Z{"msg":"[0]R generation[0..12]: [164, 160, 165, 165, 161, 165, 165, 162, 162, 164, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265921242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178282024-02-28T22:19:54.392Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265936451Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178292024-02-28T22:19:54.392Z{"msg":"[1]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 300, 296, 296, 298, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.265950841Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178302024-02-28T22:19:54.392Z{"msg":"[1]R generation[0..12]: [164, 160, 165, 165, 161, 165, 165, 162, 162, 164, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.26597907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178312024-02-28T22:19:54.392ZWait for a query_work_queue command to finish before sending IO
178322024-02-28T22:19:54.392Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266006089Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178332024-02-28T22:19:54.392Z{"msg":"[2]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 300, 296, 296, 298, 287, 287]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266033448Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178342024-02-28T22:19:54.392Z{"msg":"[2]R generation[0..12]: [164, 160, 165, 165, 161, 165, 165, 162, 162, 164, 158, 158]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266048348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178352024-02-28T22:19:54.392Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266063388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178362024-02-28T22:19:54.392Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266083457Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178372024-02-28T22:19:54.392Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266097257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178382024-02-28T22:19:54.392Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266110896Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178392024-02-28T22:19:54.392Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266124576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"":"downstairs","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178402024-02-28T22:19:54.392Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266138276Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178412024-02-28T22:19:54.392Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266150175Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"0","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178422024-02-28T22:19:54.392Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266171145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"1","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178432024-02-28T22:19:54.392Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266184704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"client":"2","":"downstairs","session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178442024-02-28T22:19:54.392Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266198384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178452024-02-28T22:19:54.392Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266217894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178462024-02-28T22:19:54.393Z{"msg":"2522f112-fd54-44d2-9030-8c5779dbc355 is now active with session: 0b360105-7ace-4ab0-b713-b1fda61d60db","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266231183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178472024-02-28T22:19:54.393Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266243213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799,"session_id":"0b360105-7ace-4ab0-b713-b1fda61d60db"}
178482024-02-28T22:19:54.393Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:53.266262992Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
178492024-02-28T22:19:54.393ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178502024-02-28T22:19:54.393ZLoading write count information from file "/tmp/test_repair_verify.data"
178512024-02-28T22:19:54.393ZRun Repair workload
178522024-02-28T22:19:54.393Z01/30 Write block 374 len 12288 data: 15 14 14
178532024-02-28T22:19:54.399Z02/30 Read block 280 len 40960
178542024-02-28T22:19:54.399Z03/30 Write block 233 len 28672 data: 15 16 12 11 12 11 7
178552024-02-28T22:19:54.399Z04/30 Read block 413 len 12288
178562024-02-28T22:19:54.399Z05/30 Write block 160 len 16384 data: 6 7 8 10
178572024-02-28T22:19:54.399Z06/30 Read block 431 len 4096
178582024-02-28T22:19:54.399Z07/30 Write block 549 len 8192 data: 9 8
178592024-02-28T22:19:54.399Z08/30 Flush
178602024-02-28T22:19:54.399Z09/30 Write block 431 len 28672 data: 10 13 14 17 18 20 19
178612024-02-28T22:19:54.400Z10/30 Read block 76 len 12288
178622024-02-28T22:19:54.400Z11/30 Read block 413 len 36864
178632024-02-28T22:19:54.405Z12/30 Write block 450 len 20480 data: 17 18 17 20 18
178642024-02-28T22:19:54.405Z13/30 Flush
178652024-02-28T22:19:54.405Z14/30 Write block 168 len 4096 data: 17
178662024-02-28T22:19:54.405Z15/30 Read block 341 len 20480
178672024-02-28T22:19:54.405Z16/30 Flush
178682024-02-28T22:19:54.405Z17/30 Write block 335 len 40960 data: 13 13 11 11 11 11 12 13 13 13
178692024-02-28T22:19:54.405Z18/30 Flush
178702024-02-28T22:19:54.409Z19/30 Write block 282 len 8192 data: 11 12
178712024-02-28T22:19:54.409Z20/30 Read block 176 len 28672
178722024-02-28T22:19:54.413Z21/30 Write block 390 len 4096 data: 16
178732024-02-28T22:19:54.413Z22/30 Read block 501 len 4096
178742024-02-28T22:19:54.413Z23/30 Flush
178752024-02-28T22:19:54.413Z24/30 Write block 432 len 32768 data: 14 15 18 19 21 20 20 21
178762024-02-28T22:19:54.417Z25/30 Write block 127 len 24576 data: 21 24 20 19 19 18
178772024-02-28T22:19:54.417Z26/30 Read block 280 len 12288
178782024-02-28T22:19:54.418Z27/30 Write block 576 len 12288 data: 12 13 13
178792024-02-28T22:19:54.418Z28/30 Write block 442 len 36864 data: 15 12 14 14 15 15 15 16 18
178802024-02-28T22:19:54.418Z29/30 Write block 195 len 40960 data: 12 12 12 12 9 8 10 10 9 11
178812024-02-28T22:19:54.424Z30/30 Write block 444 len 20480 data: 15 15 16 16 16
178822024-02-28T22:19:54.424Z----------------------------------------------------------------
178832024-02-28T22:19:54.424Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
178842024-02-28T22:19:54.424ZStates: Active Active Active
178852024-02-28T22:19:54.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
178862024-02-28T22:19:54.424Z 1 Acked 1000 Write 3 Done Done Sent false
178872024-02-28T22:19:54.424Z 2 Acked 1001 Read 10 Done Done Sent false
178882024-02-28T22:19:54.424Z 3 Acked 1002 Write 7 Done Done Sent false
178892024-02-28T22:19:54.424Z 4 Acked 1003 Read 3 Done Done Sent false
178902024-02-28T22:19:54.424Z 5 Acked 1004 Write 4 Done Done Sent false
178912024-02-28T22:19:54.424Z 6 Acked 1005 Read 1 Done Done Sent false
178922024-02-28T22:19:54.424Z 7 Acked 1006 Write 2 Done Done Sent false
178932024-02-28T22:19:54.424Z 8 Acked 1007 Flush 0 Done Done Sent false
178942024-02-28T22:19:54.424Z 9 Acked 1008 Write 7 Done Done Sent false
178952024-02-28T22:19:54.424Z 10 Acked 1009 Read 3 Done Done Sent false
178962024-02-28T22:19:54.424Z 11 Acked 1010 Read 9 Done Done Sent false
178972024-02-28T22:19:54.425Z 12 Acked 1011 Write 5 Done Done Sent false
178982024-02-28T22:19:54.425Z 13 Acked 1012 Flush 0 Done Done Sent false
178992024-02-28T22:19:54.425Z 14 Acked 1013 Write 1 Done Done Sent false
179002024-02-28T22:19:54.425Z 15 Acked 1014 Read 5 Done Done Sent false
179012024-02-28T22:19:54.425Z 16 Acked 1015 Flush 0 Done Done Sent false
179022024-02-28T22:19:54.425Z 17 Acked 1016 Write 10 Done Done Sent false
179032024-02-28T22:19:54.425Z 18 Acked 1017 Flush 0 Done Done Sent false
179042024-02-28T22:19:54.425Z 19 Acked 1018 Write 2 Done Done Sent false
179052024-02-28T22:19:54.425Z 20 Acked 1019 Read 7 Done Done Sent false
179062024-02-28T22:19:54.425Z 21 Acked 1020 Write 1 Done Done Sent false
179072024-02-28T22:19:54.425Z 22 Acked 1021 Read 1 Done Done Sent false
179082024-02-28T22:19:54.425Z 23 Acked 1022 Flush 0 Done Done Sent false
179092024-02-28T22:19:54.425Z 24 Acked 1023 Write 8 Done Done Sent false
179102024-02-28T22:19:54.425Z 25 Acked 1024 Write 6 Done Done Sent false
179112024-02-28T22:19:54.425Z 26 Acked 1025 Read 3 Done Done Sent false
179122024-02-28T22:19:54.425Z 27 Acked 1026 Write 3 Done Done Sent false
179132024-02-28T22:19:54.425Z 28 Acked 1027 Write 9 Done Done Sent false
179142024-02-28T22:19:54.425Z 29 Acked 1028 Write 10 Sent Sent Sent false
179152024-02-28T22:19:54.425Z 30 Acked 1029 Write 5 Sent Sent Sent false
179162024-02-28T22:19:54.425Z STATES DS:0 DS:1 DS:2 TOTAL
179172024-02-28T22:19:54.425Z New 0 0 0 0
179182024-02-28T22:19:54.425Z Sent 2 2 30 34
179192024-02-28T22:19:54.425Z Done 28 28 0 56
179202024-02-28T22:19:54.426Z Skipped 0 0 0 0
179212024-02-28T22:19:54.426Z Error 0 0 0 0
179222024-02-28T22:19:54.426ZLast Flush: 1022 1022 0
179232024-02-28T22:19:54.426ZDownstairs last five completed:
179242024-02-28T22:19:54.426ZUpstairs last five completed: 30 29 28 27 26
179252024-02-28T22:19:54.426Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:53.299782479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1799}
179262024-02-28T22:19:54.426ZWrote out file "/tmp/test_repair_verify.data"
179272024-02-28T22:19:54.431Z
179282024-02-28T22:19:54.431ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179292024-02-28T22:19:54.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179302024-02-28T22:19:54.470Z 0 000-019 164 164 164 298 298 298 F F F
179312024-02-28T22:19:54.471Z 1 020-039 160 160 160 291 291 291 F F F
179322024-02-28T22:19:54.471Z 2 040-059 165 165 165 300 300 300 F F F
179332024-02-28T22:19:54.471Z 3 060-079 165 165 165 300 300 300 F F F
179342024-02-28T22:19:54.471Z 4 080-099 161 161 161 293 293 293 F F F
179352024-02-28T22:19:54.471Z 5 100-119 165 165 165 300 300 300 F F F
179362024-02-28T22:19:54.471Z 6 120-139 165 165 165 300 300 300 T T F <---
179372024-02-28T22:19:54.471Z 7 140-159 162 162 162 296 296 296 F F F
179382024-02-28T22:19:54.471Z 8 160-179 166 166 162 303 303 296 F F F <---
179392024-02-28T22:19:54.471Z 9 180-199 164 164 164 298 298 298 T T F <---
179402024-02-28T22:19:54.471Z 10 200-219 158 158 158 287 287 287 T T F <---
179412024-02-28T22:19:54.472Z 11 220-239 166 166 158 301 301 287 F F F <---
179422024-02-28T22:19:54.477Z 12 240-259 164 164 164 298 298 298 F F F
179432024-02-28T22:19:54.477Z 13 260-279 163 163 163 297 297 297 F F F
179442024-02-28T22:19:54.477Z 14 280-299 166 166 163 305 305 297 F F F <---
179452024-02-28T22:19:54.477Z 15 300-319 162 162 162 294 294 294 F F F
179462024-02-28T22:19:54.478Z 16 320-339 166 166 164 304 304 299 F F F <---
179472024-02-28T22:19:54.478Z 17 340-359 166 166 165 304 304 300 F F F <---
179482024-02-28T22:19:54.478Z 18 360-379 166 166 165 301 301 300 F F F <---
179492024-02-28T22:19:54.478Z 19 380-399 166 166 164 305 305 298 F F F <---
179502024-02-28T22:19:54.478Z 20 400-419 153 153 153 278 278 278 F F F
179512024-02-28T22:19:54.478Z 21 420-439 166 166 160 302 302 291 T T F <---
179522024-02-28T22:19:54.478Z 22 440-459 166 166 162 302 302 296 T T F <---
179532024-02-28T22:19:54.479Z 23 460-479 159 159 159 290 290 290 F F F
179542024-02-28T22:19:54.479Z 24 480-499 165 165 165 300 300 300 F F F
179552024-02-28T22:19:54.479Z 25 500-519 165 165 165 300 300 300 F F F
179562024-02-28T22:19:54.479Z 26 520-539 163 163 163 297 297 297 F F F
179572024-02-28T22:19:54.479Z 27 540-559 166 166 159 301 301 290 F F F <---
179582024-02-28T22:19:54.479Z 28 560-579 164 164 164 298 298 298 T T F <---
179592024-02-28T22:19:54.479Z 29 580-599 156 156 156 284 284 284 F F F
179602024-02-28T22:19:54.479ZMax gen: 166, Max flush: 305
179612024-02-28T22:19:54.479ZError: Difference in extent metadata found!
179622024-02-28T22:19:54.479ZA Difference in extent metadata is expected here
179632024-02-28T22:19:54.479ZOn loop 83
179642024-02-28T22:19:54.479Z
179652024-02-28T22:19:54.479ZVerifying data now
179662024-02-28T22:19:59.148ZNo change after verify
179672024-02-28T22:19:59.148ZLoop: 83 Downstairs dump after verify (and repair):
179682024-02-28T22:19:59.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179692024-02-28T22:19:59.173Z 0 000-019 164 164 164 298 298 298 F F F
179702024-02-28T22:19:59.173Z 1 020-039 160 160 160 291 291 291 F F F
179712024-02-28T22:19:59.173Z 2 040-059 165 165 165 300 300 300 F F F
179722024-02-28T22:19:59.174Z 3 060-079 165 165 165 300 300 300 F F F
179732024-02-28T22:19:59.174Z 4 080-099 161 161 161 293 293 293 F F F
179742024-02-28T22:19:59.174Z 5 100-119 165 165 165 300 300 300 F F F
179752024-02-28T22:19:59.174Z 6 120-139 167 167 167 306 306 306 F F F
179762024-02-28T22:19:59.174Z 7 140-159 162 162 162 296 296 296 F F F
179772024-02-28T22:19:59.174Z 8 160-179 166 166 166 303 303 303 F F F
179782024-02-28T22:19:59.174Z 9 180-199 167 167 167 306 306 306 F F F
179792024-02-28T22:19:59.175Z 10 200-219 167 167 167 306 306 306 F F F
179802024-02-28T22:19:59.194Z 11 220-239 166 166 166 301 301 301 F F F
179812024-02-28T22:19:59.194Z 12 240-259 164 164 164 298 298 298 F F F
179822024-02-28T22:19:59.194Z 13 260-279 163 163 163 297 297 297 F F F
179832024-02-28T22:19:59.194Z 14 280-299 166 166 166 305 305 305 F F F
179842024-02-28T22:19:59.194Z 15 300-319 162 162 162 294 294 294 F F F
179852024-02-28T22:19:59.194Z 16 320-339 166 166 166 304 304 304 F F F
179862024-02-28T22:19:59.194Z 17 340-359 166 166 166 304 304 304 F F F
179872024-02-28T22:19:59.194Z 18 360-379 166 166 166 301 301 301 F F F
179882024-02-28T22:19:59.194Z 19 380-399 166 166 166 305 305 305 F F F
179892024-02-28T22:19:59.194Z 20 400-419 153 153 153 278 278 278 F F F
179902024-02-28T22:19:59.194Z 21 420-439 167 167 167 306 306 306 F F F
179912024-02-28T22:19:59.194Z 22 440-459 167 167 167 306 306 306 F F F
179922024-02-28T22:19:59.194Z 23 460-479 159 159 159 290 290 290 F F F
179932024-02-28T22:19:59.194Z 24 480-499 165 165 165 300 300 300 F F F
179942024-02-28T22:19:59.194Z 25 500-519 165 165 165 300 300 300 F F F
179952024-02-28T22:19:59.194Z 26 520-539 163 163 163 297 297 297 F F F
179962024-02-28T22:19:59.195Z 27 540-559 166 166 166 301 301 301 F F F
179972024-02-28T22:19:59.195Z 28 560-579 167 167 167 306 306 306 F F F
179982024-02-28T22:19:59.195Z 29 580-599 156 156 156 284 284 284 F F F
179992024-02-28T22:19:59.195ZMax gen: 167, Max flush: 306
180002024-02-28T22:19:59.195Z
180012024-02-28T22:19:59.195ZBegin loop 84
180022024-02-28T22:19:59.195ZDownstairs to restart: 1
180032024-02-28T22:19:59.204Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065289264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
180042024-02-28T22:19:59.204Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065625045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
180052024-02-28T22:19:59.204ZCrucible runtime is spawned
180062024-02-28T22:19:59.204Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065674193Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
180072024-02-28T22:19:59.204Z{"msg":"Crucible f38cb7b4-2a60-43be-8d10-6657d8e6235c has session id: d6e1802f-f2e6-46da-a959-e8a9c4b42ead","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065715382Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180082024-02-28T22:19:59.204Z{"msg":"Upstairs opts: Upstairs UUID: f38cb7b4-2a60-43be-8d10-6657d8e6235c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065755831Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180092024-02-28T22:19:59.204Z{"msg":"Crucible stats registered with UUID: f38cb7b4-2a60-43be-8d10-6657d8e6235c","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065771441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180102024-02-28T22:19:59.204Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06579262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
180112024-02-28T22:19:59.204Z{"msg":"f38cb7b4-2a60-43be-8d10-6657d8e6235c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06580786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180122024-02-28T22:19:59.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065828749Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180132024-02-28T22:19:59.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065851178Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180142024-02-28T22:19:59.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065887117Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180152024-02-28T22:19:59.204Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065924296Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180162024-02-28T22:19:59.204Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.065962225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180172024-02-28T22:19:59.205Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066032124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180182024-02-28T22:19:59.205Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066081252Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180192024-02-28T22:19:59.205Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066129731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180202024-02-28T22:19:59.205Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06615963Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180212024-02-28T22:19:59.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066195159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180222024-02-28T22:19:59.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066231148Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180232024-02-28T22:19:59.205Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066300656Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180242024-02-28T22:19:59.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.066906059Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180252024-02-28T22:19:59.205Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067033816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180262024-02-28T22:19:59.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067087494Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180272024-02-28T22:19:59.205Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067122163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180282024-02-28T22:19:59.205Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067207671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180292024-02-28T22:19:59.205Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06724307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180302024-02-28T22:19:59.205Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06761975Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180312024-02-28T22:19:59.205Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067654329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180322024-02-28T22:19:59.205Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.067911892Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180332024-02-28T22:19:59.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068215903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180342024-02-28T22:19:59.205Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068261162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180352024-02-28T22:19:59.205Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068285991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180362024-02-28T22:19:59.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068357389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180372024-02-28T22:19:59.205Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068384399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180382024-02-28T22:19:59.205Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068424687Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180392024-02-28T22:19:59.205Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068447027Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180402024-02-28T22:19:59.206Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068479816Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180412024-02-28T22:19:59.206Z{"msg":"[0]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 306, 296, 303, 306, 306, 301]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068507495Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180422024-02-28T22:19:59.206Z{"msg":"[0]R generation[0..12]: [164, 160, 165, 165, 161, 165, 167, 162, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068537104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180432024-02-28T22:19:59.206Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068563724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180442024-02-28T22:19:59.206Z{"msg":"[1]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 306, 296, 303, 306, 306, 301]","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
180452024-02-28T22:19:59.206Z:30,"time":"2024-02-28T22:19:58.068598143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180462024-02-28T22:19:59.206Z{"msg":"[1]R generation[0..12]: [164, 160, 165, 165, 161, 165, 167, 162, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068632822Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180472024-02-28T22:19:59.206Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068667751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180482024-02-28T22:19:59.206Z{"msg":"[2]R flush_numbers[0..12]: [298, 291, 300, 300, 293, 300, 306, 296, 303, 306, 306, 301]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06870792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180492024-02-28T22:19:59.206Z{"msg":"[2]R generation[0..12]: [164, 160, 165, 165, 161, 165, 167, 162, 166, 167, 167, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068733149Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180502024-02-28T22:19:59.206Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068763858Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180512024-02-28T22:19:59.206Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068787837Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180522024-02-28T22:19:59.206Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068817007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180532024-02-28T22:19:59.206Z{"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068840786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180542024-02-28T22:19:59.206Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068867035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"":"downstairs","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180552024-02-28T22:19:59.206Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068891195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180562024-02-28T22:19:59.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068917144Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"0","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180572024-02-28T22:19:59.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068949013Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"1","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180582024-02-28T22:19:59.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.068980662Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"client":"2","":"downstairs","session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180592024-02-28T22:19:59.206ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180602024-02-28T22:19:59.206Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.069008511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180612024-02-28T22:19:59.206Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06904045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180622024-02-28T22:19:59.207Z{"msg":"f38cb7b4-2a60-43be-8d10-6657d8e6235c is now active with session: d6e1802f-f2e6-46da-a959-e8a9c4b42ead","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.06907288Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180632024-02-28T22:19:59.207Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.069102299Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807,"session_id":"d6e1802f-f2e6-46da-a959-e8a9c4b42ead"}
180642024-02-28T22:19:59.207Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:19:58.069124968Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
180652024-02-28T22:19:59.207ZLoading write count information from file "/tmp/test_repair_verify.data"
180662024-02-28T22:19:59.207ZRun Repair workload
180672024-02-28T22:19:59.207Z01/30 Write block 222 len 20480 data: 11 9 10 12 13
180682024-02-28T22:19:59.207Z02/30 Write block 70 len 8192 data: 13 11
180692024-02-28T22:19:59.207Z03/30 Flush
180702024-02-28T22:19:59.207Z04/30 Write block 82 len 24576 data: 15 13 11 14 13 13
180712024-02-28T22:19:59.207Z05/30 Read block 531 len 32768
180722024-02-28T22:19:59.207Z06/30 Flush
180732024-02-28T22:19:59.207Z07/30 Write block 586 len 8192 data: 11 9
180742024-02-28T22:19:59.207Z08/30 Read block 441 len 32768
180752024-02-28T22:19:59.211Z09/30 Flush
180762024-02-28T22:19:59.211Z10/30 Flush
180772024-02-28T22:19:59.211Z11/30 Write block 263 len 40960 data: 18 18 18 18 14 11 9 9 10 8
180782024-02-28T22:19:59.211Z12/30 Read block 135 len 40960
180792024-02-28T22:19:59.211Z13/30 Read block 102 len 24576
180802024-02-28T22:19:59.211Z14/30 Read block 107 len 20480
180812024-02-28T22:19:59.211Z15/30 Write block 548 len 8192 data: 8 10
180822024-02-28T22:19:59.211Z16/30 Read block 44 len 20480
180832024-02-28T22:19:59.211Z17/30 Write block 310 len 4096 data: 10
180842024-02-28T22:19:59.211Z18/30 Read block 58 len 24576
180852024-02-28T22:19:59.211Z19/30 Read block 563 len 8192
180862024-02-28T22:19:59.211Z20/30 Read block 208 len 4096
180872024-02-28T22:19:59.211Z21/30 Read block 19 len 4096
180882024-02-28T22:19:59.211Z22/30 Flush
180892024-02-28T22:19:59.215Z23/30 Write block 386 len 8192 data: 21 18
180902024-02-28T22:19:59.215Z24/30 Read block 207 len 28672
180912024-02-28T22:19:59.215Z25/30 Flush
180922024-02-28T22:19:59.218Z26/30 Write block 276 len 36864 data: 7 8 9 10 11 10 12 13 14
180932024-02-28T22:19:59.219Z27/30 Read block 333 len 20480
180942024-02-28T22:19:59.222Z28/30 Write block 261 len 36864 data: 16 19 19 19 19 19 15 12 10
180952024-02-28T22:19:59.222Z29/30 Read block 212 len 32768
180962024-02-28T22:19:59.222Z30/30 Write block 295 len 20480 data: 9 11 9 7 6
180972024-02-28T22:19:59.222Z----------------------------------------------------------------
180982024-02-28T22:19:59.222Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:27
180992024-02-28T22:19:59.222ZStates: Active Active Active
181002024-02-28T22:19:59.222ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181012024-02-28T22:19:59.222Z 4 Acked 1003 Write 6 Done Sent Done false
181022024-02-28T22:19:59.222Z 5 Acked 1004 Read 8 Done Sent Done false
181032024-02-28T22:19:59.222Z 6 Acked 1005 Flush 0 Done Sent Done false
181042024-02-28T22:19:59.222Z 7 Acked 1006 Write 2 Done Sent Done false
181052024-02-28T22:19:59.223Z 8 Acked 1007 Read 8 Done Sent Done false
181062024-02-28T22:19:59.223Z 9 Acked 1008 Flush 0 Done Sent Done false
181072024-02-28T22:19:59.223Z 10 Acked 1009 Flush 0 Done Sent Done false
181082024-02-28T22:19:59.223Z 11 Acked 1010 Write 10 Done Sent Done false
181092024-02-28T22:19:59.223Z 12 Acked 1011 Read 10 Done Sent Done false
181102024-02-28T22:19:59.223Z 13 Acked 1012 Read 6 Done Sent Done false
181112024-02-28T22:19:59.223Z 14 Acked 1013 Read 5 Done Sent Done false
181122024-02-28T22:19:59.223Z 15 Acked 1014 Write 2 Done Sent Done false
181132024-02-28T22:19:59.223Z 16 Acked 1015 Read 5 Done Sent Done false
181142024-02-28T22:19:59.223Z 17 Acked 1016 Write 1 Done Sent Done false
181152024-02-28T22:19:59.223Z 18 Acked 1017 Read 6 Done Sent Done false
181162024-02-28T22:19:59.223Z 19 Acked 1018 Read 2 Done Sent Done false
181172024-02-28T22:19:59.223Z 20 Acked 1019 Read 1 Done Sent Done false
181182024-02-28T22:19:59.223Z 21 Acked 1020 Read 1 Done Sent Done false
181192024-02-28T22:19:59.223Z 22 Acked 1021 Flush 0 Done Sent Done false
181202024-02-28T22:19:59.223Z 23 Acked 1022 Write 2 Done Sent Done false
181212024-02-28T22:19:59.223Z 24 Acked 1023 Read 7 Done Sent Done false
181222024-02-28T22:19:59.223Z 25 Acked 1024 Flush 0 Done Sent Done false
181232024-02-28T22:19:59.223Z 26 Acked 1025 Write 9 Done Sent Done false
181242024-02-28T22:19:59.223Z 27 Acked 1026 Read 5 Done Sent Done false
181252024-02-28T22:19:59.223Z 28 Acked 1027 Write 9 Done Sent Done false
181262024-02-28T22:19:59.223Z 29 Acked 1028 Read 8 Done Sent Done false
181272024-02-28T22:19:59.223Z 30 Acked 1029 Write 5 Sent Sent Sent false
181282024-02-28T22:19:59.224Z STATES DS:0 DS:1 DS:2 TOTAL
181292024-02-28T22:19:59.224Z New 0 0 0 0
181302024-02-28T22:19:59.224Z Sent 1 27 1 29
181312024-02-28T22:19:59.224Z Done 26 0 26 52
181322024-02-28T22:19:59.224Z Skipped 0 0 0 0
181332024-02-28T22:19:59.224Z Error 0 0 0 0
181342024-02-28T22:19:59.224ZLast Flush: 1024 1002 1024
181352024-02-28T22:19:59.224ZDownstairs last five completed: 1002 1001 1000
181362024-02-28T22:19:59.224ZUpstairs last five completed: 30 29 28 27 26
181372024-02-28T22:19:59.224Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:19:58.102324804Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1807}
181382024-02-28T22:19:59.224ZWrote out file "/tmp/test_repair_verify.data"
181392024-02-28T22:19:59.227Z
181402024-02-28T22:19:59.233ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181412024-02-28T22:19:59.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181422024-02-28T22:19:59.244Z 0 000-019 164 164 164 298 298 298 F F F
181432024-02-28T22:19:59.244Z 1 020-039 160 160 160 291 291 291 F F F
181442024-02-28T22:19:59.244Z 2 040-059 165 165 165 300 300 300 F F F
181452024-02-28T22:19:59.244Z 3 060-079 168 168 168 307 307 307 F F F
181462024-02-28T22:19:59.244Z 4 080-099 168 161 168 308 293 308 F F F <---
181472024-02-28T22:19:59.244Z 5 100-119 165 165 165 300 300 300 F F F
181482024-02-28T22:19:59.247Z 6 120-139 167 167 167 306 306 306 F F F
181492024-02-28T22:19:59.247Z 7 140-159 162 162 162 296 296 296 F F F
181502024-02-28T22:19:59.247Z 8 160-179 166 166 166 303 303 303 F F F
181512024-02-28T22:19:59.248Z 9 180-199 167 167 167 306 306 306 F F F
181522024-02-28T22:19:59.248Z 10 200-219 167 167 167 306 306 306 F F F
181532024-02-28T22:19:59.248Z 11 220-239 168 168 168 307 307 307 F F F
181542024-02-28T22:19:59.248Z 12 240-259 164 164 164 298 298 298 F F F
181552024-02-28T22:19:59.248Z 13 260-279 168 163 168 311 297 311 T F T <---
181562024-02-28T22:19:59.248Z 14 280-299 166 166 166 305 305 305 T F T <---
181572024-02-28T22:19:59.248Z 15 300-319 168 162 168 311 294 311 F F F <---
181582024-02-28T22:19:59.248Z 16 320-339 166 166 166 304 304 304 F F F
181592024-02-28T22:19:59.248Z 17 340-359 166 166 166 304 304 304 F F F
181602024-02-28T22:19:59.248Z 18 360-379 166 166 166 301 301 301 F F F
181612024-02-28T22:19:59.248Z 19 380-399 168 166 168 312 305 312 F F F <---
181622024-02-28T22:19:59.248Z 20 400-419 153 153 153 278 278 278 F F F
181632024-02-28T22:19:59.248Z 21 420-439 167 167 167 306 306 306 F F F
181642024-02-28T22:19:59.248Z 22 440-459 167 167 167 306 306 306 F F F
181652024-02-28T22:19:59.248Z 23 460-479 159 159 159 290 290 290 F F F
181662024-02-28T22:19:59.248Z 24 480-499 165 165 165 300 300 300 F F F
181672024-02-28T22:19:59.248Z 25 500-519 165 165 165 300 300 300 F F F
181682024-02-28T22:19:59.248Z 26 520-539 163 163 163 297 297 297 F F F
181692024-02-28T22:19:59.248Z 27 540-559 168 166 168 311 301 311 F F F <---
181702024-02-28T22:19:59.248Z 28 560-579 167 167 167 306 306 306 F F F
181712024-02-28T22:19:59.248Z 29 580-599 168 156 168 309 284 309 F F F <---
181722024-02-28T22:19:59.248ZMax gen: 168, Max flush: 312
181732024-02-28T22:19:59.249ZError: Difference in extent metadata found!
181742024-02-28T22:19:59.249ZA Difference in extent metadata is expected here
181752024-02-28T22:19:59.249ZOn loop 84
181762024-02-28T22:19:59.249Z
181772024-02-28T22:19:59.252ZVerifying data now
181782024-02-28T22:20:03.658ZNo change after verify
181792024-02-28T22:20:03.658ZLoop: 84 Downstairs dump after verify (and repair):
181802024-02-28T22:20:03.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181812024-02-28T22:20:03.671Z 0 000-019 164 164 164 298 298 298 F F F
181822024-02-28T22:20:03.671Z 1 020-039 160 160 160 291 291 291 F F F
181832024-02-28T22:20:03.671Z 2 040-059 165 165 165 300 300 300 F F F
181842024-02-28T22:20:03.671Z 3 060-079 168 168 168 307 307 307 F F F
181852024-02-28T22:20:03.671Z 4 080-099 168 168 168 308 308 308 F F F
181862024-02-28T22:20:03.679Z 5 100-119 165 165 165 300 300 300 F F F
181872024-02-28T22:20:03.680Z 6 120-139 167 167 167 306 306 306 F F F
181882024-02-28T22:20:03.680Z 7 140-159 162 162 162 296 296 296 F F F
181892024-02-28T22:20:03.680Z 8 160-179 166 166 166 303 303 303 F F F
181902024-02-28T22:20:03.680Z 9 180-199 167 167 167 306 306 306 F F F
181912024-02-28T22:20:03.680Z 10 200-219 167 167 167 306 306 306 F F F
181922024-02-28T22:20:03.680Z 11 220-239 168 168 168 307 307 307 F F F
181932024-02-28T22:20:03.680Z 12 240-259 164 164 164 298 298 298 F F F
181942024-02-28T22:20:03.681Z 13 260-279 169 169 169 313 313 313 F F F
181952024-02-28T22:20:03.681Z 14 280-299 169 169 169 313 313 313 F F F
181962024-02-28T22:20:03.681Z 15 300-319 168 168 168 311 311 311 F F F
181972024-02-28T22:20:03.681Z 16 320-339 166 166 166 304 304 304 F F F
181982024-02-28T22:20:03.681Z 17 340-359 166 166 166 304 304 304 F F F
181992024-02-28T22:20:03.681Z 18 360-379 166 166 166 301 301 301 F F F
182002024-02-28T22:20:03.681Z 19 380-399 168 168 168 312 312 312 F F F
182012024-02-28T22:20:03.681Z 20 400-419 153 153 153 278 278 278 F F F
182022024-02-28T22:20:03.681Z 21 420-439 167 167 167 306 306 306 F F F
182032024-02-28T22:20:03.682Z 22 440-459 167 167 167 306 306 306 F F F
182042024-02-28T22:20:03.682Z 23 460-479 159 159 159 290 290 290 F F F
182052024-02-28T22:20:03.682Z 24 480-499 165 165 165 300 300 300 F F F
182062024-02-28T22:20:03.682Z 25 500-519 165 165 165 300 300 300 F F F
182072024-02-28T22:20:03.682Z 26 520-539 163 163 163 297 297 297 F F F
182082024-02-28T22:20:03.682Z 27 540-559 168 168 168 311 311 311 F F F
182092024-02-28T22:20:03.682Z 28 560-579 167 167 167 306 306 306 F F F
182102024-02-28T22:20:03.682Z 29 580-599 168 168 168 309 309 309 F F F
182112024-02-28T22:20:03.682ZMax gen: 169, Max flush: 313
182122024-02-28T22:20:03.682Z
182132024-02-28T22:20:03.682ZBegin loop 85
182142024-02-28T22:20:03.682ZDownstairs to restart: 2
182152024-02-28T22:20:03.694Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.574601456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
182162024-02-28T22:20:03.694Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.574943477Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
182172024-02-28T22:20:03.695Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.574966856Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
182182024-02-28T22:20:03.702Z{"msg":"Crucible edeadf09-45d8-47a6-9b92-9a9a9c353de0 has session id: d33b5ab1-2683-4393-8052-a4f8fee4843f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575004535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182192024-02-28T22:20:03.702Z{"msg":"Upstairs opts: Upstairs UUID: edeadf09-45d8-47a6-9b92-9a9a9c353de0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575058984Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182202024-02-28T22:20:03.702Z{"msg":"Crucible stats registered with UUID: edeadf09-45d8-47a6-9b92-9a9a9c353de0"Crucible runtime is spawned
182212024-02-28T22:20:03.702Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575107312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182222024-02-28T22:20:03.702Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575219919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
182232024-02-28T22:20:03.702Z{"msg":"edeadf09-45d8-47a6-9b92-9a9a9c353de0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575277718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182242024-02-28T22:20:03.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575311637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182252024-02-28T22:20:03.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575358805Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182262024-02-28T22:20:03.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575399504Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182272024-02-28T22:20:03.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575424964Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182282024-02-28T22:20:03.702Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575454413Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182292024-02-28T22:20:03.702Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575473372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182302024-02-28T22:20:03.702Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.57557535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182312024-02-28T22:20:03.702Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575641388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182322024-02-28T22:20:03.703Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575681307Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182332024-02-28T22:20:03.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575708516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182342024-02-28T22:20:03.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575734135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182352024-02-28T22:20:03.703Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.575759005Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182362024-02-28T22:20:03.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576336818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182372024-02-28T22:20:03.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576373468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182382024-02-28T22:20:03.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576389627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182392024-02-28T22:20:03.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576417036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182402024-02-28T22:20:03.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576431556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182412024-02-28T22:20:03.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.576445915Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182422024-02-28T22:20:03.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.57702537Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182432024-02-28T22:20:03.703Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577173866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182442024-02-28T22:20:03.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577221664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182452024-02-28T22:20:03.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577483867Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182462024-02-28T22:20:03.703Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577533436Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182472024-02-28T22:20:03.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577559705Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182482024-02-28T22:20:03.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577638693Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182492024-02-28T22:20:03.703Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577696821Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182502024-02-28T22:20:03.703Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.57774197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182512024-02-28T22:20:03.703Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577763309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182522024-02-28T22:20:03.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577787669Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182532024-02-28T22:20:03.703Z{"msg":"[0]R flush_numbers[0..12]: [298, 291, 300, 307, 308, 300, 306, 296, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577802838Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182542024-02-28T22:20:03.703Z{"msg":"[0]R generation[0..12]: [164, 160, 165, 168, 168, 165, 167, 162, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577832807Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182552024-02-28T22:20:03.704Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577860956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182562024-02-28T22:20:03.704Z{"msg":"[1]R flush_numbers[0..12]: [298, 291, 300, 307, 308, 300, 306, 296, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577876676Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182572024-02-28T22:20:03.704Z{"msg":"[1]R generation[0..12]: [164, 160, 165, 168, 168, 165, 167, 162, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577906545Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182582024-02-28T22:20:03.704Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"Wait for a query_work_queue command to finish before sending IO
182592024-02-28T22:20:03.704Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577947144Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182602024-02-28T22:20:03.704Z{"msg":"[2]R flush_numbers[0..12]: [298, 291, 300, 307, 308, 300, 306, 296, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577973564Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182612024-02-28T22:20:03.704Z{"msg":"[2]R generation[0..12]: [164, 160, 165, 168, 168, 165, 167, 162, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.577996493Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182622024-02-28T22:20:03.704Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578018592Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182632024-02-28T22:20:03.704Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578039242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182642024-02-28T22:20:03.704Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578053511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182652024-02-28T22:20:03.704Z{"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578075401Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182662024-02-28T22:20:03.704Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.57808993Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"":"downstairs","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182672024-02-28T22:20:03.704Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.57810351Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182682024-02-28T22:20:03.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578123389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"0","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182692024-02-28T22:20:03.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578138079Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"1","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182702024-02-28T22:20:03.704Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578166218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"client":"2","":"downstairs","session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182712024-02-28T22:20:03.704Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578190187Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182722024-02-28T22:20:03.704Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578218997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182732024-02-28T22:20:03.704Z{"msg":"edeadf09-45d8-47a6-9b92-9a9a9c353de0 is now active with session: d33b5ab1-2683-4393-8052-a4f8fee4843f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578242796Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182742024-02-28T22:20:03.704Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578268355Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815,"session_id":"d33b5ab1-2683-4393-8052-a4f8fee4843f"}
182752024-02-28T22:20:03.704Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:02.578290435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
182762024-02-28T22:20:03.704ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182772024-02-28T22:20:03.704ZLoading write count information from file "/tmp/test_repair_verify.data"
182782024-02-28T22:20:03.705ZRun Repair workload
182792024-02-28T22:20:03.712Z01/30 Write block 357 len 40960 data: 14 12 10 9 8 11 12 8 10 11
182802024-02-28T22:20:03.713Z02/30 Read block 573 len 32768
182812024-02-28T22:20:03.713Z03/30 Read block 203 len 28672
182822024-02-28T22:20:03.713Z04/30 Read block 99 len 40960
182832024-02-28T22:20:03.713Z05/30 Read block 75 len 32768
182842024-02-28T22:20:03.713Z06/30 Flush
182852024-02-28T22:20:03.713Z07/30 Write block 248 len 36864 data: 19 15 15 15 12 14 12 11 14
182862024-02-28T22:20:03.713Z08/30 Read block 348 len 28672
182872024-02-28T22:20:03.716Z09/30 Write block 506 len 4096 data: 18
182882024-02-28T22:20:03.717Z10/30 Flush
182892024-02-28T22:20:03.717Z11/30 Write block 91 len 28672 data: 14 14 12 11 10 12 13
182902024-02-28T22:20:03.717Z12/30 Write block 145 len 36864 data: 17 18 17 18 16 13 10 10 6
182912024-02-28T22:20:03.717Z13/30 Write block 66 len 4096 data: 14
182922024-02-28T22:20:03.717Z14/30 Read block 330 len 24576
182932024-02-28T22:20:03.717Z15/30 Read block 354 len 24576
182942024-02-28T22:20:03.717Z16/30 Read block 330 len 20480
182952024-02-28T22:20:03.721Z17/30 Read block 583 len 12288
182962024-02-28T22:20:03.721Z18/30 Write block 295 len 20480 data: 10 12 10 8 7
182972024-02-28T22:20:03.721Z19/30 Read block 382 len 8192
182982024-02-28T22:20:03.721Z20/30 Write block 335 len 12288 data: 14 14 12
182992024-02-28T22:20:03.721Z21/30 Read block 61 len 20480
183002024-02-28T22:20:03.727Z22/30 Read block 83 len 16384
183012024-02-28T22:20:03.727Z23/30 Read block 339 len 24576
183022024-02-28T22:20:03.728Z24/30 Flush
183032024-02-28T22:20:03.732Z25/30 Write block 411 len 28672 data: 10 11 12 14 15 14 10
183042024-02-28T22:20:03.732Z26/30 Read block 171 len 24576
183052024-02-28T22:20:03.732Z27/30 Read block 504 len 8192
183062024-02-28T22:20:03.732Z28/30 Read block 392 len 20480
183072024-02-28T22:20:03.732Z29/30 Read block 128 len 24576
183082024-02-28T22:20:03.732Z30/30 Write block 477 len 40960 data: 10 8 7 8 7 11 12 15 16 13
183092024-02-28T22:20:03.732Z----------------------------------------------------------------
183102024-02-28T22:20:03.732Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183112024-02-28T22:20:03.732ZStates: Active Active Active
183122024-02-28T22:20:03.732ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183132024-02-28T22:20:03.732Z 1 Acked 1000 Write 10 Done Done Done false
183142024-02-28T22:20:03.732Z 2 Acked 1001 Read 8 Done Done Done false
183152024-02-28T22:20:03.732Z 3 Acked 1002 Read 7 Done Done Done false
183162024-02-28T22:20:03.732Z 4 Acked 1003 Read 10 Done Done Done false
183172024-02-28T22:20:03.732Z 5 Acked 1004 Read 8 Done Done Done false
183182024-02-28T22:20:03.732Z 6 Acked 1005 Flush 0 Done Done Sent false
183192024-02-28T22:20:03.733Z 7 Acked 1006 Write 9 Done Done Sent false
183202024-02-28T22:20:03.733Z 8 Acked 1007 Read 7 Done Done Sent false
183212024-02-28T22:20:03.733Z 9 Acked 1008 Write 1 Done Done Sent false
183222024-02-28T22:20:03.733Z 10 Acked 1009 Flush 0 Done Done Sent false
183232024-02-28T22:20:03.733Z 11 Acked 1010 Write 7 Done Done Sent false
183242024-02-28T22:20:03.733Z 12 Acked 1011 Write 9 Done Done Sent false
183252024-02-28T22:20:03.733Z 13 Acked 1012 Write 1 Done Done Sent false
183262024-02-28T22:20:03.733Z 14 Acked 1013 Read 6 Done Done Sent false
183272024-02-28T22:20:03.733Z 15 Acked 1014 Read 6 Done Done Sent false
183282024-02-28T22:20:03.733Z 16 Acked 1015 Read 5 Done Done Sent false
183292024-02-28T22:20:03.733Z 17 Acked 1016 Read 3 Done Done Sent false
183302024-02-28T22:20:03.733Z 18 Acked 1017 Write 5 Done Done Sent false
183312024-02-28T22:20:03.733Z 19 Acked 1018 Read 2 Done Done Sent false
183322024-02-28T22:20:03.733Z 20 Acked 1019 Write 3 Done Done Sent false
183332024-02-28T22:20:03.733Z 21 Acked 1020 Read 5 Done Done Sent false
183342024-02-28T22:20:03.733Z 22 Acked 1021 Read 4 Done Done Sent false
183352024-02-28T22:20:03.733Z 23 Acked 1022 Read 6 Done Done Sent false
183362024-02-28T22:20:03.733Z 24 Acked 1023 Flush 0 Done Done Sent false
183372024-02-28T22:20:03.733Z 25 Acked 1024 Write 7 Done Done Sent false
183382024-02-28T22:20:03.733Z 26 Acked 1025 Read 6 Done Done Sent false
183392024-02-28T22:20:03.733Z 27 Acked 1026 Read 2 Done Done Sent false
183402024-02-28T22:20:03.733Z 28 Acked 1027 Read 5 Done Done Sent false
183412024-02-28T22:20:03.733Z 29 Acked 1028 Read 6 Done Done Sent false
183422024-02-28T22:20:03.734Z 30 Acked 1029 Write 10 Sent Sent Sent false
183432024-02-28T22:20:03.734Z STATES DS:0 DS:1 DS:2 TOTAL
183442024-02-28T22:20:03.734Z New 0 0 0 0
183452024-02-28T22:20:03.734Z Sent 1 1 25 27
183462024-02-28T22:20:03.734Z Done 29 29 5 63
183472024-02-28T22:20:03.734Z Skipped 0 0 0 0
183482024-02-28T22:20:03.734Z Error 0 0 0 0
183492024-02-28T22:20:03.734ZLast Flush: 1023 1023 0
183502024-02-28T22:20:03.734ZDownstairs last five completed:
183512024-02-28T22:20:03.734ZUpstairs last five completed: 30 29 28 27 26
183522024-02-28T22:20:03.734Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:02.612046245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1815}
183532024-02-28T22:20:03.738ZWrote out file "/tmp/test_repair_verify.data"
183542024-02-28T22:20:03.738Z
183552024-02-28T22:20:03.744ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183562024-02-28T22:20:03.755ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183572024-02-28T22:20:03.755Z 0 000-019 164 164 164 298 298 298 F F F
183582024-02-28T22:20:03.755Z 1 020-039 160 160 160 291 291 291 F F F
183592024-02-28T22:20:03.755Z 2 040-059 165 165 165 300 300 300 F F F
183602024-02-28T22:20:03.755Z 3 060-079 170 170 168 316 316 307 F F F <---
183612024-02-28T22:20:03.755Z 4 080-099 170 170 168 316 316 308 F F F <---
183622024-02-28T22:20:03.755Z 5 100-119 165 165 165 300 300 300 F F F
183632024-02-28T22:20:03.755Z 6 120-139 167 167 167 306 306 306 F F F
183642024-02-28T22:20:03.755Z 7 140-159 170 170 162 316 316 296 F F F <---
183652024-02-28T22:20:03.759Z 8 160-179 166 166 166 303 303 303 F F F
183662024-02-28T22:20:03.759Z 9 180-199 167 167 167 306 306 306 F F F
183672024-02-28T22:20:03.759Z 10 200-219 167 167 167 306 306 306 F F F
183682024-02-28T22:20:03.759Z 11 220-239 168 168 168 307 307 307 F F F
183692024-02-28T22:20:03.759Z 12 240-259 170 170 164 315 315 298 F F F <---
183702024-02-28T22:20:03.759Z 13 260-279 169 169 169 313 313 313 F F F
183712024-02-28T22:20:03.759Z 14 280-299 170 170 169 316 316 313 F F F <---
183722024-02-28T22:20:03.759Z 15 300-319 168 168 168 311 311 311 F F F
183732024-02-28T22:20:03.759Z 16 320-339 170 170 166 316 316 304 F F F <---
183742024-02-28T22:20:03.759Z 17 340-359 170 170 166 314 314 304 F F T <---
183752024-02-28T22:20:03.759Z 18 360-379 170 170 166 314 314 301 F F T <---
183762024-02-28T22:20:03.759Z 19 380-399 168 168 168 312 312 312 F F F
183772024-02-28T22:20:03.759Z 20 400-419 153 153 153 278 278 278 T T F <---
183782024-02-28T22:20:03.759Z 21 420-439 167 167 167 306 306 306 F F F
183792024-02-28T22:20:03.759Z 22 440-459 167 167 167 306 306 306 F F F
183802024-02-28T22:20:03.759Z 23 460-479 159 159 159 290 290 290 T T F <---
183812024-02-28T22:20:03.759Z 24 480-499 165 165 165 300 300 300 T T F <---
183822024-02-28T22:20:03.759Z 25 500-519 170 170 165 315 315 300 F F F <---
183832024-02-28T22:20:03.759Z 26 520-539 163 163 163 297 297 297 F F F
183842024-02-28T22:20:03.760Z 27 540-559 168 168 168 311 311 311 F F F
183852024-02-28T22:20:03.760Z 28 560-579 167 167 167 306 306 306 F F F
183862024-02-28T22:20:03.760Z 29 580-599 168 168 168 309 309 309 F F F
183872024-02-28T22:20:03.760ZMax gen: 170, Max flush: 316
183882024-02-28T22:20:03.760ZError: Difference in extent metadata found!
183892024-02-28T22:20:03.760ZA Difference in extent metadata is expected here
183902024-02-28T22:20:03.760ZOn loop 85
183912024-02-28T22:20:03.760Z
183922024-02-28T22:20:03.764ZVerifying data now
183932024-02-28T22:20:09.386ZNo change after verify
183942024-02-28T22:20:09.386ZLoop: 85 Downstairs dump after verify (and repair):
183952024-02-28T22:20:09.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183962024-02-28T22:20:09.401Z 0 000-019 164 164 164 298 298 298 F F F
183972024-02-28T22:20:09.401Z 1 020-039 160 160 160 291 291 291 F F F
183982024-02-28T22:20:09.401Z 2 040-059 165 165 165 300 300 300 F F F
183992024-02-28T22:20:09.401Z 3 060-079 170 170 170 316 316 316 F F F
184002024-02-28T22:20:09.401Z 4 080-099 170 170 170 316 316 316 F F F
184012024-02-28T22:20:09.401Z 5 100-119 165 165 165 300 300 300 F F F
184022024-02-28T22:20:09.401Z 6 120-139 167 167 167 306 306 306 F F F
184032024-02-28T22:20:09.401Z 7 140-159 170 170 170 316 316 316 F F F
184042024-02-28T22:20:09.405Z 8 160-179 166 166 166 303 303 303 F F F
184052024-02-28T22:20:09.405Z 9 180-199 167 167 167 306 306 306 F F F
184062024-02-28T22:20:09.405Z 10 200-219 167 167 167 306 306 306 F F F
184072024-02-28T22:20:09.405Z 11 220-239 168 168 168 307 307 307 F F F
184082024-02-28T22:20:09.405Z 12 240-259 170 170 170 315 315 315 F F F
184092024-02-28T22:20:09.405Z 13 260-279 169 169 169 313 313 313 F F F
184102024-02-28T22:20:09.405Z 14 280-299 170 170 170 316 316 316 F F F
184112024-02-28T22:20:09.405Z 15 300-319 168 168 168 311 311 311 F F F
184122024-02-28T22:20:09.405Z 16 320-339 170 170 170 316 316 316 F F F
184132024-02-28T22:20:09.405Z 17 340-359 170 170 170 314 314 314 F F F
184142024-02-28T22:20:09.405Z 18 360-379 170 170 170 314 314 314 F F F
184152024-02-28T22:20:09.405Z 19 380-399 168 168 168 312 312 312 F F F
184162024-02-28T22:20:09.405Z 20 400-419 171 171 171 317 317 317 F F F
184172024-02-28T22:20:09.405Z 21 420-439 167 167 167 306 306 306 F F F
184182024-02-28T22:20:09.405Z 22 440-459 167 167 167 306 306 306 F F F
184192024-02-28T22:20:09.405Z 23 460-479 171 171 171 317 317 317 F F F
184202024-02-28T22:20:09.405Z 24 480-499 171 171 171 317 317 317 F F F
184212024-02-28T22:20:09.406Z 25 500-519 170 170 170 315 315 315 F F F
184222024-02-28T22:20:09.406Z 26 520-539 163 163 163 297 297 297 F F F
184232024-02-28T22:20:09.406Z 27 540-559 168 168 168 311 311 311 F F F
184242024-02-28T22:20:09.406Z 28 560-579 167 167 167 306 306 306 F F F
184252024-02-28T22:20:09.406Z 29 580-599 168 168 168 309 309 309 F F F
184262024-02-28T22:20:09.406ZMax gen: 171, Max flush: 317
184272024-02-28T22:20:09.406Z
184282024-02-28T22:20:09.406ZBegin loop 86
184292024-02-28T22:20:09.406ZDownstairs to restart: 1
184302024-02-28T22:20:09.423Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303041124Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
184312024-02-28T22:20:09.423Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
184322024-02-28T22:20:09.423Z,"time":"2024-02-28T22:20:08.303377055Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
184332024-02-28T22:20:09.430Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303446733Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
184342024-02-28T22:20:09.430Z{"msg":"Crucible 07fa0675-75bf-468a-a8a8-7a5f9f23fa36 has session id: bcf7b20b-0088-415a-bbc5-b95dcbcb9c74","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303488412Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184352024-02-28T22:20:09.430Z{"msg":"Upstairs opts: Upstairs UUID: 07fa0675-75bf-468a-a8a8-7a5f9f23fa36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.30354479Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184362024-02-28T22:20:09.430Z{"msg":"Crucible stats registered with UUID: 07fa0675-75bf-468a-a8a8-7a5f9f23fa36","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303592689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184372024-02-28T22:20:09.430Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303631848Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
184382024-02-28T22:20:09.430Z{"msg":"07fa0675-75bf-468a-a8a8-7a5f9f23fa36 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303669727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184392024-02-28T22:20:09.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303695286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184402024-02-28T22:20:09.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303732815Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184412024-02-28T22:20:09.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303772974Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184422024-02-28T22:20:09.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303812073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184432024-02-28T22:20:09.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303848802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184442024-02-28T22:20:09.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303882921Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184452024-02-28T22:20:09.430Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.30390832Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184462024-02-28T22:20:09.430Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.303940649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184472024-02-28T22:20:09.430Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304012767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184482024-02-28T22:20:09.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304093705Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184492024-02-28T22:20:09.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304133514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184502024-02-28T22:20:09.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304159123Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184512024-02-28T22:20:09.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304873024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184522024-02-28T22:20:09.430Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304913183Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184532024-02-28T22:20:09.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.304973621Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184542024-02-28T22:20:09.431Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305038139Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184552024-02-28T22:20:09.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305096997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184562024-02-28T22:20:09.431Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305114857Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184572024-02-28T22:20:09.431Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305537605Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184582024-02-28T22:20:09.431Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305575374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184592024-02-28T22:20:09.431Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.305598994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184602024-02-28T22:20:09.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.30608701Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184612024-02-28T22:20:09.431Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306144579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184622024-02-28T22:20:09.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306207547Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184632024-02-28T22:20:09.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306251146Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184642024-02-28T22:20:09.431Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306268925Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184652024-02-28T22:20:09.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306291335Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184662024-02-28T22:20:09.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306312624Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184672024-02-28T22:20:09.431Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306333754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184682024-02-28T22:20:09.431Z{"msg":"[0]R flush_numbers[0..12]: [298, 291, 300, 316, 316, 300, 306, 316, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306399002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184692024-02-28T22:20:09.431Z{"msg":"[0]R generation[0..12]: [164, 160, 165, 170, 170, 165, 167, 170, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306426041Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184702024-02-28T22:20:09.431Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306441991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184712024-02-28T22:20:09.431Z{"msg":"[1]R flush_numbers[0..12]: [298, 291, 300, 316, 316, 300, 306, 316, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.30646309Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184722024-02-28T22:20:09.431Z{"msg":"[1]R generation[0..12]: [164, 160, 165, 170, 170, 165, 167, 170, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306492939Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184732024-02-28T22:20:09.431Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306516218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184742024-02-28T22:20:09.431Z{"msg":"[2]R flush_numbers[0..12]: [298, 291, 300, 316, 316, 300, 306, 316, 303, 306, 306, 307]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306530798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184752024-02-28T22:20:09.431Z{"msg":"[2]R generation[0..12]: [164, 160, 165, 170, 170, 165, 167, 170, 166, 167, 167, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306557507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184762024-02-28T22:20:09.432Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306572357Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","Wait for a query_work_queue command to finish before sending IO
184772024-02-28T22:20:09.432Z":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184782024-02-28T22:20:09.432Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306597256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184792024-02-28T22:20:09.432Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306611756Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184802024-02-28T22:20:09.432Z{"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306632425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184812024-02-28T22:20:09.432Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306654685Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"":"downstairs","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184822024-02-28T22:20:09.432Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306686034Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184832024-02-28T22:20:09.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306700833Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"0","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184842024-02-28T22:20:09.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306722333Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"1","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184852024-02-28T22:20:09.432Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306741312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"client":"2","":"downstairs","session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184862024-02-28T22:20:09.432Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306772002Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184872024-02-28T22:20:09.432Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306786671Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184882024-02-28T22:20:09.432Z{"msg":"07fa0675-75bf-468a-a8a8-7a5f9f23fa36 is now active with session: bcf7b20b-0088-415a-bbc5-b95dcbcb9c74","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306806881Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184892024-02-28T22:20:09.432Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.30681938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823,"session_id":"bcf7b20b-0088-415a-bbc5-b95dcbcb9c74"}
184902024-02-28T22:20:09.432Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:08.306850389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
184912024-02-28T22:20:09.436ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184922024-02-28T22:20:09.437ZLoading write count information from file "/tmp/test_repair_verify.data"
184932024-02-28T22:20:09.437ZRun Repair workload
184942024-02-28T22:20:09.437Z01/30 Write block 19 len 28672 data: 12 10 9 7 6 6 6
184952024-02-28T22:20:09.437Z02/30 Write block 558 len 32768 data: 12 12 13 15 14 16 16 17
184962024-02-28T22:20:09.437Z03/30 Read block 300 len 28672
184972024-02-28T22:20:09.437Z04/30 Read block 124 len 20480
184982024-02-28T22:20:09.437Z05/30 Flush
184992024-02-28T22:20:09.437Z06/30 Write block 436 len 32768 data: 22 21 21 22 19 15 16 13
185002024-02-28T22:20:09.437Z07/30 Read block 173 len 8192
185012024-02-28T22:20:09.437Z08/30 Flush
185022024-02-28T22:20:09.440Z09/30 Write block 389 len 32768 data: 14 17 18 16 12 12 15 12
185032024-02-28T22:20:09.441Z10/30 Write block 161 len 8192 data: 8 9
185042024-02-28T22:20:09.441Z11/30 Write block 242 len 28672 data: 9 12 12 14 10 17 20
185052024-02-28T22:20:09.444Z12/30 Write block 313 len 20480 data: 14 14 10 10 9
185062024-02-28T22:20:09.444Z13/30 Read block 140 len 32768
185072024-02-28T22:20:09.444Z14/30 Read block 564 len 20480
185082024-02-28T22:20:09.444Z15/30 Write block 563 len 8192 data: 17 17
185092024-02-28T22:20:09.447Z16/30 Read block 15 len 36864
185102024-02-28T22:20:09.447Z17/30 Write block 225 len 28672 data: 13 14 11 14 17 15 19
185112024-02-28T22:20:09.447Z18/30 Read block 237 len 16384
185122024-02-28T22:20:09.448Z19/30 Write block 466 len 8192 data: 11 11
185132024-02-28T22:20:09.450Z20/30 Write block 298 len 32768 data: 9 8 5 8 10 13 13 13
185142024-02-28T22:20:09.450Z21/30 Write block 46 len 4096 data: 9
185152024-02-28T22:20:09.451Z22/30 Read block 367 len 40960
185162024-02-28T22:20:09.454Z23/30 Write block 318 len 36864 data: 9 9 9 9 11 12 15 15 16
185172024-02-28T22:20:09.454Z24/30 Flush
185182024-02-28T22:20:09.457Z25/30 Write block 16 len 36864 data: 13 13 12 13 11 10 8 7 7
185192024-02-28T22:20:09.457Z26/30 Write block 342 len 24576 data: 14 14 14 12 10 11
185202024-02-28T22:20:09.461Z27/30 Read block 489 len 8192
185212024-02-28T22:20:09.461Z28/30 Write block 374 len 4096 data: 16
185222024-02-28T22:20:09.461Z29/30 Read block 449 len 16384
185232024-02-28T22:20:09.461Z30/30 Write block 84 len 16384 data: 12 15 14 14
185242024-02-28T22:20:09.461Z----------------------------------------------------------------
185252024-02-28T22:20:09.461Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
185262024-02-28T22:20:09.461ZStates: Active Active Active
185272024-02-28T22:20:09.461ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185282024-02-28T22:20:09.461Z 1 Acked 1000 Write 7 Done Done Done false
185292024-02-28T22:20:09.461Z 2 Acked 1001 Write 8 Done Done Done false
185302024-02-28T22:20:09.461Z 3 Acked 1002 Read 7 Done Sent Done false
185312024-02-28T22:20:09.461Z 4 Acked 1003 Read 5 Done Sent Done false
185322024-02-28T22:20:09.461Z 5 Acked 1004 Flush 0 Done Sent Done false
185332024-02-28T22:20:09.461Z 6 Acked 1005 Write 8 Done Sent Done false
185342024-02-28T22:20:09.461Z 7 Acked 1006 Read 2 Done Sent Done false
185352024-02-28T22:20:09.461Z 8 Acked 1007 Flush 0 Done Sent Done false
185362024-02-28T22:20:09.461Z 9 Acked 1008 Write 8 Done Sent Done false
185372024-02-28T22:20:09.461Z 10 Acked 1009 Write 2 Done Sent Done false
185382024-02-28T22:20:09.461Z 11 Acked 1010 Write 7 Done Sent Done false
185392024-02-28T22:20:09.461Z 12 Acked 1011 Write 5 Done Sent Done false
185402024-02-28T22:20:09.461Z 13 Acked 1012 Read 8 Done Sent Done false
185412024-02-28T22:20:09.462Z 14 Acked 1013 Read 5 Done Sent Done false
185422024-02-28T22:20:09.462Z 15 Acked 1014 Write 2 Done Sent Done false
185432024-02-28T22:20:09.462Z 16 Acked 1015 Read 9 Done Sent Done false
185442024-02-28T22:20:09.462Z 17 Acked 1016 Write 7 Done Sent Done false
185452024-02-28T22:20:09.462Z 18 Acked 1017 Read 4 Done Sent Done false
185462024-02-28T22:20:09.462Z 19 Acked 1018 Write 2 Done Sent Done false
185472024-02-28T22:20:09.462Z 20 Acked 1019 Write 8 Done Sent Done false
185482024-02-28T22:20:09.462Z 21 Acked 1020 Write 1 Done Sent Done false
185492024-02-28T22:20:09.462Z 22 Acked 1021 Read 10 Done Sent Done false
185502024-02-28T22:20:09.465Z 23 Acked 1022 Write 9 Done Sent Done false
185512024-02-28T22:20:09.466Z 24 Acked 1023 Flush 0 Done Sent Done false
185522024-02-28T22:20:09.466Z 25 Acked 1024 Write 9 Done Sent Done false
185532024-02-28T22:20:09.466Z 26 Acked 1025 Write 6 Done Sent Done false
185542024-02-28T22:20:09.466Z 27 Acked 1026 Read 2 Done Sent Done false
185552024-02-28T22:20:09.466Z 28 Acked 1027 Write 1 Done Sent Done false
185562024-02-28T22:20:09.466Z 29 Acked 1028 Read 4 Done Sent Done false
185572024-02-28T22:20:09.466Z 30 Acked 1029 Write 4 Sent Sent Sent false
185582024-02-28T22:20:09.466Z STATES DS:0 DS:1 DS:2 TOTAL
185592024-02-28T22:20:09.466Z New 0 0 0 0
185602024-02-28T22:20:09.466Z Sent 1 28 1 30
185612024-02-28T22:20:09.466Z Done 29 2 29 60
185622024-02-28T22:20:09.466Z Skipped 0 0 0 0
185632024-02-28T22:20:09.466Z Error 0 0 0 0
185642024-02-28T22:20:09.466ZLast Flush: 1023 0 1023
185652024-02-28T22:20:09.466ZDownstairs last five completed:
185662024-02-28T22:20:09.466ZUpstairs last five completed: 30 29 28 27 26
185672024-02-28T22:20:09.466Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:08.341415097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1823}
185682024-02-28T22:20:09.466ZWrote out file "/tmp/test_repair_verify.data"
185692024-02-28T22:20:09.466Z
185702024-02-28T22:20:09.470ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185712024-02-28T22:20:09.485ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185722024-02-28T22:20:09.485Z 0 000-019 172 164 172 318 298 318 T T T <---
185732024-02-28T22:20:09.485Z 1 020-039 172 160 172 318 291 318 T T T <---
185742024-02-28T22:20:09.485Z 2 040-059 172 165 172 320 300 320 F F F <---
185752024-02-28T22:20:09.485Z 3 060-079 170 170 170 316 316 316 F F F
185762024-02-28T22:20:09.485Z 4 080-099 170 170 170 316 316 316 T F T <---
185772024-02-28T22:20:09.485Z 5 100-119 165 165 165 300 300 300 F F F
185782024-02-28T22:20:09.485Z 6 120-139 167 167 167 306 306 306 F F F
185792024-02-28T22:20:09.485Z 7 140-159 170 170 170 316 316 316 F F F
185802024-02-28T22:20:09.485Z 8 160-179 172 166 172 320 303 320 F F F <---
185812024-02-28T22:20:09.489Z 9 180-199 167 167 167 306 306 306 F F F
185822024-02-28T22:20:09.489Z 10 200-219 167 167 167 306 306 306 F F F
185832024-02-28T22:20:09.489Z 11 220-239 172 168 172 320 307 320 F F F <---
185842024-02-28T22:20:09.489Z 12 240-259 172 170 172 320 315 320 F F F <---
185852024-02-28T22:20:09.489Z 13 260-279 169 169 169 313 313 313 F F F
185862024-02-28T22:20:09.489Z 14 280-299 172 170 172 320 316 320 F F F <---
185872024-02-28T22:20:09.489Z 15 300-319 172 168 172 320 311 320 F F F <---
185882024-02-28T22:20:09.490Z 16 320-339 172 170 172 320 316 320 F F F <---
185892024-02-28T22:20:09.490Z 17 340-359 170 170 170 314 314 314 T F T <---
185902024-02-28T22:20:09.490Z 18 360-379 170 170 170 314 314 314 T F T <---
185912024-02-28T22:20:09.490Z 19 380-399 172 168 172 320 312 320 F F F <---
185922024-02-28T22:20:09.490Z 20 400-419 171 171 171 317 317 317 F F F
185932024-02-28T22:20:09.490Z 21 420-439 172 167 172 319 306 319 F F F <---
185942024-02-28T22:20:09.490Z 22 440-459 172 167 172 319 306 319 F F F <---
185952024-02-28T22:20:09.490Z 23 460-479 172 171 172 320 317 320 F F F <---
185962024-02-28T22:20:09.490Z 24 480-499 171 171 171 317 317 317 F F F
185972024-02-28T22:20:09.490Z 25 500-519 170 170 170 315 315 315 F F F
185982024-02-28T22:20:09.490Z 26 520-539 163 163 163 297 297 297 F F F
185992024-02-28T22:20:09.490Z 27 540-559 172 168 172 318 311 318 F T F <---
186002024-02-28T22:20:09.490Z 28 560-579 172 167 172 320 306 320 F T F <---
186012024-02-28T22:20:09.490Z 29 580-599 168 168 168 309 309 309 F F F
186022024-02-28T22:20:09.490ZMax gen: 172, Max flush: 320
186032024-02-28T22:20:09.490ZError: Difference in extent metadata found!
186042024-02-28T22:20:09.490ZA Difference in extent metadata is expected here
186052024-02-28T22:20:09.491ZOn loop 86
186062024-02-28T22:20:09.491Z
186072024-02-28T22:20:09.494ZVerifying data now
186082024-02-28T22:20:15.353ZNo change after verify
186092024-02-28T22:20:15.353ZLoop: 86 Downstairs dump after verify (and repair):
186102024-02-28T22:20:15.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186112024-02-28T22:20:15.366Z 0 000-019 173 173 173 321 321 321 F F F
186122024-02-28T22:20:15.366Z 1 020-039 173 173 173 321 321 321 F F F
186132024-02-28T22:20:15.366Z 2 040-059 172 172 172 320 320 320 F F F
186142024-02-28T22:20:15.366Z 3 060-079 170 170 170 316 316 316 F F F
186152024-02-28T22:20:15.366Z 4 080-099 173 173 173 321 321 321 F F F
186162024-02-28T22:20:15.366Z 5 100-119 165 165 165 300 300 300 F F F
186172024-02-28T22:20:15.367Z 6 120-139 167 167 167 306 306 306 F F F
186182024-02-28T22:20:15.367Z 7 140-159 170 170 170 316 316 316 F F F
186192024-02-28T22:20:15.367Z 8 160-179 172 172 172 320 320 320 F F F
186202024-02-28T22:20:15.367Z 9 180-199 167 167 167 306 306 306 F F F
186212024-02-28T22:20:15.367Z 10 200-219 167 167 167 306 306 306 F F F
186222024-02-28T22:20:15.370Z 11 220-239 172 172 172 320 320 320 F F F
186232024-02-28T22:20:15.370Z 12 240-259 172 172 172 320 320 320 F F F
186242024-02-28T22:20:15.371Z 13 260-279 169 169 169 313 313 313 F F F
186252024-02-28T22:20:15.371Z 14 280-299 172 172 172 320 320 320 F F F
186262024-02-28T22:20:15.371Z 15 300-319 172 172 172 320 320 320 F F F
186272024-02-28T22:20:15.371Z 16 320-339 172 172 172 320 320 320 F F F
186282024-02-28T22:20:15.371Z 17 340-359 173 173 173 321 321 321 F F F
186292024-02-28T22:20:15.371Z 18 360-379 173 173 173 321 321 321 F F F
186302024-02-28T22:20:15.371Z 19 380-399 172 172 172 320 320 320 F F F
186312024-02-28T22:20:15.371Z 20 400-419 171 171 171 317 317 317 F F F
186322024-02-28T22:20:15.371Z 21 420-439 172 172 172 319 319 319 F F F
186332024-02-28T22:20:15.371Z 22 440-459 172 172 172 319 319 319 F F F
186342024-02-28T22:20:15.371Z 23 460-479 172 172 172 320 320 320 F F F
186352024-02-28T22:20:15.371Z 24 480-499 171 171 171 317 317 317 F F F
186362024-02-28T22:20:15.371Z 25 500-519 170 170 170 315 315 315 F F F
186372024-02-28T22:20:15.371Z 26 520-539 163 163 163 297 297 297 F F F
186382024-02-28T22:20:15.371Z 27 540-559 172 172 172 318 318 318 F F F
186392024-02-28T22:20:15.371Z 28 560-579 172 172 172 320 320 320 F F F
186402024-02-28T22:20:15.371Z 29 580-599 168 168 168 309 309 309 F F F
186412024-02-28T22:20:15.371ZMax gen: 173, Max flush: 321
186422024-02-28T22:20:15.371Z
186432024-02-28T22:20:15.371ZBegin loop 87
186442024-02-28T22:20:15.371ZDownstairs to restart: 0
186452024-02-28T22:20:15.390Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27036901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
186462024-02-28T22:20:15.390Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270611853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
186472024-02-28T22:20:15.390Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270641963Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
186482024-02-28T22:20:15.396Z{"msg":"Crucible 8ce529ac-0093-4365-842b-1850ebd23a86 has session id: bef7e773-f5c2-4527-80ac-b1c988ca1e9d","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
186492024-02-28T22:20:15.396Z2024-02-28T22:20:14.270686611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186502024-02-28T22:20:15.396Z{"msg":"Upstairs opts: Upstairs UUID: 8ce529ac-0093-4365-842b-1850ebd23a86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27073872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186512024-02-28T22:20:15.396Z{"msg":"Crucible stats registered with UUID: 8ce529ac-0093-4365-842b-1850ebd23a86","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270774769Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186522024-02-28T22:20:15.396Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270807558Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
186532024-02-28T22:20:15.396Z{"msg":"8ce529ac-0093-4365-842b-1850ebd23a86 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270942814Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186542024-02-28T22:20:15.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.270984113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186552024-02-28T22:20:15.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271011092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186562024-02-28T22:20:15.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271056741Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186572024-02-28T22:20:15.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27108496Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186582024-02-28T22:20:15.396Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271120049Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186592024-02-28T22:20:15.396Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271157488Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186602024-02-28T22:20:15.396Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271197437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186612024-02-28T22:20:15.396Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271218977Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186622024-02-28T22:20:15.396Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271243476Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186632024-02-28T22:20:15.396Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271286745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186642024-02-28T22:20:15.396Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271324424Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186652024-02-28T22:20:15.396Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.271577937Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186662024-02-28T22:20:15.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.272151201Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186672024-02-28T22:20:15.397Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27218764Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186682024-02-28T22:20:15.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.272212589Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186692024-02-28T22:20:15.397Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.272240408Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186702024-02-28T22:20:15.397Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.272801373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186712024-02-28T22:20:15.397Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.272838712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186722024-02-28T22:20:15.397Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273231091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186732024-02-28T22:20:15.397Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.2732798Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186742024-02-28T22:20:15.397Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273389987Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186752024-02-28T22:20:15.397Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273430956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186762024-02-28T22:20:15.397Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273466665Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186772024-02-28T22:20:15.397Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273482714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186782024-02-28T22:20:15.397Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273514084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186792024-02-28T22:20:15.397Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273529833Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186802024-02-28T22:20:15.397Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.273836244Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186812024-02-28T22:20:15.402Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274284632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186822024-02-28T22:20:15.402Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274323261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186832024-02-28T22:20:15.402Z{"msg":"[0]R flush_numbers[0..12]: [321, 321, 320, 316, 321, 300, 306, 316, 320, 306, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274340101Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186842024-02-28T22:20:15.402Z{"msg":"[0]R generation[0..12]: [173, 173, 172, 170, 173, 165, 167, 170, 172, 167, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27436329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186852024-02-28T22:20:15.402Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27437862Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186862024-02-28T22:20:15.402Z{"msg":"[1]R flush_numbers[0..12]: [321, 321, 320, 316, 321, 300, 306, 316, 320, 306, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274400269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186872024-02-28T22:20:15.403Z{"msg":"[1]R generation[0..12]: [173, 173, 172, 170, 173, 165, 167, 170, 172, 167, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274415159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186882024-02-28T22:20:15.403Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274437308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186892024-02-28T22:20:15.403Z{"msg":"[2]R flush_numbers[0..12]: [321, 321, 320, 316, 321, 300, 306, 316, 320, 306, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274452038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186902024-02-28T22:20:15.403Z{"msg":"[2]R generation[0..12]: [173, 173, 172, 170, 173, 165, 167, 170, 172, 167, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274473197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":Wait for a query_work_queue command to finish before sending IO
186912024-02-28T22:20:15.403Z1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186922024-02-28T22:20:15.403Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274492666Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186932024-02-28T22:20:15.403Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274506786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186942024-02-28T22:20:15.403Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274520536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186952024-02-28T22:20:15.403Z{"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274541135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186962024-02-28T22:20:15.403Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274554855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"":"downstairs","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186972024-02-28T22:20:15.403Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274569204Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186982024-02-28T22:20:15.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274581464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"0","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
186992024-02-28T22:20:15.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274602003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"1","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187002024-02-28T22:20:15.403Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274615973Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"client":"2","":"downstairs","session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187012024-02-28T22:20:15.403Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274635942Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187022024-02-28T22:20:15.403Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274648552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187032024-02-28T22:20:15.403Z{"msg":"8ce529ac-0093-4365-842b-1850ebd23a86 is now active with session: bef7e773-f5c2-4527-80ac-b1c988ca1e9d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274674581Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187042024-02-28T22:20:15.403Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.274687991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831,"session_id":"bef7e773-f5c2-4527-80ac-b1c988ca1e9d"}
187052024-02-28T22:20:15.403Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:14.27471306Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
187062024-02-28T22:20:15.403ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187072024-02-28T22:20:15.403ZLoading write count information from file "/tmp/test_repair_verify.data"
187082024-02-28T22:20:15.403ZRun Repair workload
187092024-02-28T22:20:15.403Z01/30 Write block 168 len 8192 data: 18 17
187102024-02-28T22:20:15.404Z02/30 Write block 539 len 16384 data: 10 10 11 11
187112024-02-28T22:20:15.404Z03/30 Read block 512 len 28672
187122024-02-28T22:20:15.404Z04/30 Write block 546 len 4096 data: 13
187132024-02-28T22:20:15.404Z05/30 Read block 501 len 40960
187142024-02-28T22:20:15.404Z06/30 Flush
187152024-02-28T22:20:15.407Z07/30 Write block 431 len 36864 data: 11 15 16 19 20 23 22 22 23
187162024-02-28T22:20:15.408Z08/30 Read block 477 len 20480
187172024-02-28T22:20:15.408Z09/30 Read block 427 len 36864
187182024-02-28T22:20:15.408Z10/30 Write block 558 len 4096 data: 13
187192024-02-28T22:20:15.408Z11/30 Read block 78 len 20480
187202024-02-28T22:20:15.408Z12/30 Write block 439 len 8192 data: 24 20
187212024-02-28T22:20:15.408Z13/30 Write block 476 len 8192 data: 12 11
187222024-02-28T22:20:15.411Z14/30 Read block 291 len 36864
187232024-02-28T22:20:15.411Z15/30 Read block 112 len 16384
187242024-02-28T22:20:15.411Z16/30 Write block 6 len 40960 data: 15 18 18 12 11 13 16 14 12 12
187252024-02-28T22:20:15.412Z17/30 Write block 572 len 36864 data: 12 14 12 13 13 14 14 14 12
187262024-02-28T22:20:15.415Z18/30 Read block 251 len 16384
187272024-02-28T22:20:15.415Z19/30 Write block 470 len 16384 data: 13 15 16 13
187282024-02-28T22:20:15.415Z20/30 Write block 512 len 4096 data: 14
187292024-02-28T22:20:15.415Z21/30 Write block 117 len 16384 data: 17 15 12 12
187302024-02-28T22:20:15.415Z22/30 Read block 344 len 36864
187312024-02-28T22:20:15.418Z23/30 Flush
187322024-02-28T22:20:15.421Z24/30 Flush
187332024-02-28T22:20:15.421Z25/30 Write block 190 len 20480 data: 11 11 14 12 11
187342024-02-28T22:20:15.421Z26/30 Read block 585 len 40960
187352024-02-28T22:20:15.424Z27/30 Write block 540 len 32768 data: 11 12 12 11 12 13 14 12
187362024-02-28T22:20:15.424Z28/30 Read block 92 len 16384
187372024-02-28T22:20:15.424Z29/30 Read block 276 len 8192
187382024-02-28T22:20:15.428Z30/30 Write block 138 len 32768 data: 15 13 12 11 12 14 14 18
187392024-02-28T22:20:15.428Z----------------------------------------------------------------
187402024-02-28T22:20:15.428Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187412024-02-28T22:20:15.428ZStates: Active Active Active
187422024-02-28T22:20:15.428ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187432024-02-28T22:20:15.428Z 1 Acked 1000 Write 2 Done Done Done false
187442024-02-28T22:20:15.428Z 2 Acked 1001 Write 4 Sent Done Done false
187452024-02-28T22:20:15.428Z 3 Acked 1002 Read 7 Sent Done Done false
187462024-02-28T22:20:15.428Z 4 Acked 1003 Write 1 Sent Done Done false
187472024-02-28T22:20:15.428Z 5 Acked 1004 Read 10 Sent Done Done false
187482024-02-28T22:20:15.428Z 6 Acked 1005 Flush 0 Sent Done Done false
187492024-02-28T22:20:15.428Z 7 Acked 1006 Write 9 Sent Done Done false
187502024-02-28T22:20:15.428Z 8 Acked 1007 Read 5 Sent Done Done false
187512024-02-28T22:20:15.429Z 9 Acked 1008 Read 9 Sent Done Done false
187522024-02-28T22:20:15.429Z 10 Acked 1009 Write 1 Sent Done Done false
187532024-02-28T22:20:15.429Z 11 Acked 1010 Read 5 Sent Done Done false
187542024-02-28T22:20:15.429Z 12 Acked 1011 Write 2 Sent Done Done false
187552024-02-28T22:20:15.429Z 13 Acked 1012 Write 2 Sent Done Done false
187562024-02-28T22:20:15.429Z 14 Acked 1013 Read 9 Sent Done Done false
187572024-02-28T22:20:15.429Z 15 Acked 1014 Read 4 Sent Done Done false
187582024-02-28T22:20:15.429Z 16 Acked 1015 Write 10 Sent Done Done false
187592024-02-28T22:20:15.429Z 17 Acked 1016 Write 9 Sent Done Done false
187602024-02-28T22:20:15.429Z 18 Acked 1017 Read 4 Sent Done Done false
187612024-02-28T22:20:15.429Z 19 Acked 1018 Write 4 Sent Done Done false
187622024-02-28T22:20:15.429Z 20 Acked 1019 Write 1 Sent Done Done false
187632024-02-28T22:20:15.429Z 21 Acked 1020 Write 4 Sent Done Done false
187642024-02-28T22:20:15.429Z 22 Acked 1021 Read 9 Sent Done Done false
187652024-02-28T22:20:15.429Z 23 Acked 1022 Flush 0 Sent Done Done false
187662024-02-28T22:20:15.429Z 24 Acked 1023 Flush 0 Sent Done Done false
187672024-02-28T22:20:15.429Z 25 Acked 1024 Write 5 Sent Done Done false
187682024-02-28T22:20:15.429Z 26 Acked 1025 Read 10 Sent Done Done false
187692024-02-28T22:20:15.429Z 27 Acked 1026 Write 8 Sent Done Done false
187702024-02-28T22:20:15.429Z 28 Acked 1027 Read 4 Sent Done Done false
187712024-02-28T22:20:15.429Z 29 Acked 1028 Read 2 Sent Done Done false
187722024-02-28T22:20:15.429Z 30 Acked 1029 Write 8 Sent Sent Sent false
187732024-02-28T22:20:15.429Z STATES DS:0 DS:1 DS:2 TOTAL
187742024-02-28T22:20:15.429Z New 0 0 0 0
187752024-02-28T22:20:15.430Z Sent 29 1 1 31
187762024-02-28T22:20:15.430Z Done 1 29 29 59
187772024-02-28T22:20:15.430Z Skipped 0 0 0 0
187782024-02-28T22:20:15.430Z Error 0 0 0 0
187792024-02-28T22:20:15.430ZLast Flush: 0 1023 1023
187802024-02-28T22:20:15.430ZDownstairs last five completed:
187812024-02-28T22:20:15.430ZUpstairs last five completed: 30 29 28 27 26
187822024-02-28T22:20:15.430Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:14.306601302Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1831}
187832024-02-28T22:20:15.430ZWrote out file "/tmp/test_repair_verify.data"
187842024-02-28T22:20:15.433Z
187852024-02-28T22:20:15.436ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187862024-02-28T22:20:15.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187872024-02-28T22:20:15.451Z 0 000-019 173 174 174 321 323 323 F F F <---
187882024-02-28T22:20:15.451Z 1 020-039 173 173 173 321 321 321 F F F
187892024-02-28T22:20:15.451Z 2 040-059 172 172 172 320 320 320 F F F
187902024-02-28T22:20:15.452Z 3 060-079 170 170 170 316 316 316 F F F
187912024-02-28T22:20:15.452Z 4 080-099 173 173 173 321 321 321 F F F
187922024-02-28T22:20:15.452Z 5 100-119 165 174 174 300 323 323 F F F <---
187932024-02-28T22:20:15.452Z 6 120-139 167 174 174 306 323 323 F T T <---
187942024-02-28T22:20:15.452Z 7 140-159 170 170 170 316 316 316 F T T <---
187952024-02-28T22:20:15.452Z 8 160-179 172 174 174 320 322 322 T F F <---
187962024-02-28T22:20:15.452Z 9 180-199 167 167 167 306 306 306 F T T <---
187972024-02-28T22:20:15.452Z 10 200-219 167 167 167 306 306 306 F F F
187982024-02-28T22:20:15.452Z 11 220-239 172 172 172 320 320 320 F F F
187992024-02-28T22:20:15.452Z 12 240-259 172 172 172 320 320 320 F F F
188002024-02-28T22:20:15.452Z 13 260-279 169 169 169 313 313 313 F F F
188012024-02-28T22:20:15.452Z 14 280-299 172 172 172 320 320 320 F F F
188022024-02-28T22:20:15.452Z 15 300-319 172 172 172 320 320 320 F F F
188032024-02-28T22:20:15.452Z 16 320-339 172 172 172 320 320 320 F F F
188042024-02-28T22:20:15.452Z 17 340-359 173 173 173 321 321 321 F F F
188052024-02-28T22:20:15.452Z 18 360-379 173 173 173 321 321 321 F F F
188062024-02-28T22:20:15.452Z 19 380-399 172 172 172 320 320 320 F F F
188072024-02-28T22:20:15.456Z 20 400-419 171 171 171 317 317 317 F F F
188082024-02-28T22:20:15.456Z 21 420-439 172 174 174 319 323 323 F F F <---
188092024-02-28T22:20:15.456Z 22 440-459 172 174 174 319 323 323 F F F <---
188102024-02-28T22:20:15.456Z 23 460-479 172 174 174 320 323 323 F F F <---
188112024-02-28T22:20:15.456Z 24 480-499 171 171 171 317 317 317 F F F
188122024-02-28T22:20:15.456Z 25 500-519 170 174 174 315 323 323 F F F <---
188132024-02-28T22:20:15.456Z 26 520-539 163 174 174 297 322 322 F F F <---
188142024-02-28T22:20:15.456Z 27 540-559 172 174 174 318 323 323 F T T <---
188152024-02-28T22:20:15.456Z 28 560-579 172 174 174 320 323 323 F F F <---
188162024-02-28T22:20:15.456Z 29 580-599 168 174 174 309 323 323 F F F <---
188172024-02-28T22:20:15.456ZMax gen: 174, Max flush: 323
188182024-02-28T22:20:15.456ZError: Difference in extent metadata found!
188192024-02-28T22:20:15.456ZA Difference in extent metadata is expected here
188202024-02-28T22:20:15.456ZOn loop 87
188212024-02-28T22:20:15.456Z
188222024-02-28T22:20:15.460ZVerifying data now
188232024-02-28T22:20:20.793ZNo change after verify
188242024-02-28T22:20:20.816ZLoop: 87 Downstairs dump after verify (and repair):
188252024-02-28T22:20:20.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188262024-02-28T22:20:20.816Z 0 000-019 174 174 174 323 323 323 F F F
188272024-02-28T22:20:20.816Z 1 020-039 173 173 173 321 321 321 F F F
188282024-02-28T22:20:20.816Z 2 040-059 172 172 172 320 320 320 F F F
188292024-02-28T22:20:20.817Z 3 060-079 170 170 170 316 316 316 F F F
188302024-02-28T22:20:20.817Z 4 080-099 173 173 173 321 321 321 F F F
188312024-02-28T22:20:20.817Z 5 100-119 174 174 174 323 323 323 F F F
188322024-02-28T22:20:20.817Z 6 120-139 175 175 175 324 324 324 F F F
188332024-02-28T22:20:20.817Z 7 140-159 175 175 175 324 324 324 F F F
188342024-02-28T22:20:20.817Z 8 160-179 174 174 174 322 322 322 F F F
188352024-02-28T22:20:20.817Z 9 180-199 175 175 175 324 324 324 F F F
188362024-02-28T22:20:20.817Z 10 200-219 167 167 167 306 306 306 F F F
188372024-02-28T22:20:20.817Z 11 220-239 172 172 172 320 320 320 F F F
188382024-02-28T22:20:20.817Z 12 240-259 172 172 172 320 320 320 F F F
188392024-02-28T22:20:20.817Z 13 260-279 169 169 169 313 313 313 F F F
188402024-02-28T22:20:20.817Z 14 280-299 172 172 172 320 320 320 F F F
188412024-02-28T22:20:20.817Z 15 300-319 172 172 172 320 320 320 F F F
188422024-02-28T22:20:20.817Z 16 320-339 172 172 172 320 320 320 F F F
188432024-02-28T22:20:20.818Z 17 340-359 173 173 173 321 321 321 F F F
188442024-02-28T22:20:20.818Z 18 360-379 173 173 173 321 321 321 F F F
188452024-02-28T22:20:20.818Z 19 380-399 172 172 172 320 320 320 F F F
188462024-02-28T22:20:20.818Z 20 400-419 171 171 171 317 317 317 F F F
188472024-02-28T22:20:20.818Z 21 420-439 174 174 174 323 323 323 F F F
188482024-02-28T22:20:20.818Z 22 440-459 174 174 174 323 323 323 F F F
188492024-02-28T22:20:20.818Z 23 460-479 174 174 174 323 323 323 F F F
188502024-02-28T22:20:20.818Z 24 480-499 171 171 171 317 317 317 F F F
188512024-02-28T22:20:20.818Z 25 500-519 174 174 174 323 323 323 F F F
188522024-02-28T22:20:20.818Z 26 520-539 174 174 174 322 322 322 F F F
188532024-02-28T22:20:20.818Z 27 540-559 175 175 175 324 324 324 F F F
188542024-02-28T22:20:20.818Z 28 560-579 174 174 174 323 323 323 F F F
188552024-02-28T22:20:20.819Z 29 580-599 174 174 174 323 323 323 F F F
188562024-02-28T22:20:20.819ZMax gen: 175, Max flush: 324
188572024-02-28T22:20:20.819Z
188582024-02-28T22:20:20.819ZBegin loop 88
188592024-02-28T22:20:20.819ZDownstairs to restart: 2
188602024-02-28T22:20:20.819Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.067908232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
188612024-02-28T22:20:20.819Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
188622024-02-28T22:20:20.819Z,"time":"2024-02-28T22:20:19.068227874Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
188632024-02-28T22:20:20.819Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068261353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
188642024-02-28T22:20:20.819Z{"msg":"Crucible 4dd06b35-8e19-4839-9ae0-bc8e8f3780df has session id: a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068290562Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188652024-02-28T22:20:20.819Z{"msg":"Upstairs opts: Upstairs UUID: 4dd06b35-8e19-4839-9ae0-bc8e8f3780df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068326731Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188662024-02-28T22:20:20.819Z{"msg":"Crucible stats registered with UUID: 4dd06b35-8e19-4839-9ae0-bc8e8f3780df","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.06835279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188672024-02-28T22:20:20.819Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068380999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
188682024-02-28T22:20:20.820Z{"msg":"4dd06b35-8e19-4839-9ae0-bc8e8f3780df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068405909Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188692024-02-28T22:20:20.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068438938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188702024-02-28T22:20:20.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068467397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188712024-02-28T22:20:20.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068492486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188722024-02-28T22:20:20.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068527435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188732024-02-28T22:20:20.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068572004Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188742024-02-28T22:20:20.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068613793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188752024-02-28T22:20:20.820Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068652632Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188762024-02-28T22:20:20.820Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068694961Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188772024-02-28T22:20:20.820Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.06873216Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188782024-02-28T22:20:20.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068770619Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188792024-02-28T22:20:20.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068821277Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188802024-02-28T22:20:20.820Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.068859986Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188812024-02-28T22:20:20.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.069472879Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188822024-02-28T22:20:20.820Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.069510388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188832024-02-28T22:20:20.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.069528668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188842024-02-28T22:20:20.820Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.069551457Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188852024-02-28T22:20:20.820Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070053203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188862024-02-28T22:20:20.820Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070127801Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188872024-02-28T22:20:20.820Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.07018245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188882024-02-28T22:20:20.820Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070228328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188892024-02-28T22:20:20.820Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070658387Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188902024-02-28T22:20:20.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070696326Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188912024-02-28T22:20:20.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070719965Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188922024-02-28T22:20:20.821Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070748074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188932024-02-28T22:20:20.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070822332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188942024-02-28T22:20:20.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070849851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188952024-02-28T22:20:20.821Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.070872021Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188962024-02-28T22:20:20.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071283579Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188972024-02-28T22:20:20.821Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071343728Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188982024-02-28T22:20:20.821Z{"msg":"[0]R flush_numbers[0..12]: [323, 321, 320, 316, 321, 323, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071382507Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
188992024-02-28T22:20:20.821Z{"msg":"[0]R generation[0..12]: [174, 173, 172, 170, 173, 174, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071406256Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189002024-02-28T22:20:20.821Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071430535Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189012024-02-28T22:20:20.821Z{"msg":"[1]R flush_numbers[0..12]: [323, 321, 320, 316, 321, 323, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071459154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189022024-02-28T22:20:20.821Z{"msg":"[1]R generation[0..12]: [174, 173, 172, 170, 173, 174, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071482164Z","Wait for a query_work_queue command to finish before sending IO
189032024-02-28T22:20:20.821Zhostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189042024-02-28T22:20:20.821Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071509913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189052024-02-28T22:20:20.821Z{"msg":"[2]R flush_numbers[0..12]: [323, 321, 320, 316, 321, 323, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071531043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189062024-02-28T22:20:20.821Z{"msg":"[2]R generation[0..12]: [174, 173, 172, 170, 173, 174, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071553102Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189072024-02-28T22:20:20.821Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071567842Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189082024-02-28T22:20:20.821Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071589051Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189092024-02-28T22:20:20.821Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.07161533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189102024-02-28T22:20:20.821Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.07163571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189112024-02-28T22:20:20.821Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071658179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"":"downstairs","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189122024-02-28T22:20:20.821Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071679568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189132024-02-28T22:20:20.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071700008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"0","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189142024-02-28T22:20:20.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071721567Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"1","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189152024-02-28T22:20:20.822Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071743197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"client":"2","":"downstairs","session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189162024-02-28T22:20:20.822Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071765236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189172024-02-28T22:20:20.822Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071786316Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189182024-02-28T22:20:20.822Z{"msg":"4dd06b35-8e19-4839-9ae0-bc8e8f3780df is now active with session: a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071807005Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189192024-02-28T22:20:20.822Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071827104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839,"session_id":"a5eba85d-1534-4cb8-bbf3-ce462c7a3bc0"}
189202024-02-28T22:20:20.822Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:19.071847284Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
189212024-02-28T22:20:20.822ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189222024-02-28T22:20:20.822ZLoading write count information from file "/tmp/test_repair_verify.data"
189232024-02-28T22:20:20.822ZRun Repair workload
189242024-02-28T22:20:20.828ZError: Difference in extent metadata found!
189252024-02-28T22:20:20.828Z01/30 Write block 46 len 20480 data: 10 11 12 14 12
189262024-02-28T22:20:20.828Z02/30 Read block 456 len 32768
189272024-02-28T22:20:20.829Z03/30 Read block 92 len 20480
189282024-02-28T22:20:20.829Z04/30 Read block 67 len 32768
189292024-02-28T22:20:20.829Z05/30 Read block 378 len 4096
189302024-02-28T22:20:20.829Z06/30 Write block 38 len 36864 data: 15 13 10 11 8 9 8 10 11
189312024-02-28T22:20:20.829Z07/30 Write block 62 len 12288 data: 17 14 14
189322024-02-28T22:20:20.829Z08/30 Write block 52 len 16384 data: 9 7 7 10
189332024-02-28T22:20:20.829Z09/30 Read block 11 len 32768
189342024-02-28T22:20:20.829Z10/30 Write block 574 len 32768 data: 13 14 14 15 15 15 13 10
189352024-02-28T22:20:20.829Z11/30 Read block 176 len 28672
189362024-02-28T22:20:20.829Z12/30 Read block 82 len 4096
189372024-02-28T22:20:20.829Z13/30 Write block 490 len 20480 data: 14 16 15 14 14
189382024-02-28T22:20:20.829Z14/30 Flush
189392024-02-28T22:20:20.829Z15/30 Write block 243 len 28672 data: 13 13 15 11 18 21 16
189402024-02-28T22:20:20.829Z16/30 Read block 399 len 36864
189412024-02-28T22:20:20.829Z17/30 Write block 530 len 36864 data: 9 9 10 12 15 12 13 12 10
189422024-02-28T22:20:20.829Z18/30 Flush
189432024-02-28T22:20:20.829Z19/30 Write block 344 len 40960 data: 15 13 11 12 13 11 12 14 15 16
189442024-02-28T22:20:20.829Z20/30 Read block 197 len 24576
189452024-02-28T22:20:20.829Z21/30 Read block 114 len 40960
189462024-02-28T22:20:20.829Z22/30 Write block 247 len 32768 data: 19 22 17 16 16 13 15 13
189472024-02-28T22:20:20.829Z23/30 Read block 433 len 24576
189482024-02-28T22:20:20.829Z24/30 Read block 436 len 20480
189492024-02-28T22:20:20.830Z25/30 Write block 52 len 36864 data: 10 8 8 11 13 14 13 16 17
189502024-02-28T22:20:20.830Z26/30 Read block 156 len 28672
189512024-02-28T22:20:20.830Z27/30 Write block 99 len 32768 data: 11 12 11 11 10 10 10 11
189522024-02-28T22:20:20.830Z28/30 Read block 566 len 36864
189532024-02-28T22:20:20.830Z29/30 Read block 144 len 36864
189542024-02-28T22:20:20.830Z30/30 Read block 481 len 20480
189552024-02-28T22:20:20.830Z----------------------------------------------------------------
189562024-02-28T22:20:20.830Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
189572024-02-28T22:20:20.830ZStates: Active Active Active
189582024-02-28T22:20:20.830ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189592024-02-28T22:20:20.830Z 1 Acked 1000 Write 5 Done Done Done false
189602024-02-28T22:20:20.830Z 2 Acked 1001 Read 8 Done Done Done false
189612024-02-28T22:20:20.830Z 3 Acked 1002 Read 5 Done Done Done false
189622024-02-28T22:20:20.830Z 4 Acked 1003 Read 8 Done Done Done false
189632024-02-28T22:20:20.830Z 5 Acked 1004 Read 1 Done Done Sent false
189642024-02-28T22:20:20.830Z 6 Acked 1005 Write 9 Done Done Sent false
189652024-02-28T22:20:20.830Z 7 Acked 1006 Write 3 Done Done Sent false
189662024-02-28T22:20:20.830Z 8 Acked 1007 Write 4 Done Done Sent false
189672024-02-28T22:20:20.830Z 9 Acked 1008 Read 8 Done Done Sent false
189682024-02-28T22:20:20.830Z 10 Acked 1009 Write 8 Done Done Sent false
189692024-02-28T22:20:20.830Z 11 Acked 1010 Read 7 Done Done Sent false
189702024-02-28T22:20:20.830Z 12 Acked 1011 Read 1 Done Done Sent false
189712024-02-28T22:20:20.830Z 13 Acked 1012 Write 5 Done Done Sent false
189722024-02-28T22:20:20.830Z 14 Acked 1013 Flush 0 Done Done Sent false
189732024-02-28T22:20:20.831Z 15 Acked 1014 Write 7 Done Done Sent false
189742024-02-28T22:20:20.831Z 16 Acked 1015 Read 9 Done Done Sent false
189752024-02-28T22:20:20.831Z 17 Acked 1016 Write 9 Done Done Sent false
189762024-02-28T22:20:20.831Z 18 Acked 1017 Flush 0 Done Done Sent false
189772024-02-28T22:20:20.831Z 19 Acked 1018 Write 10 Done Done Sent false
189782024-02-28T22:20:20.831Z 20 Acked 1019 Read 6 Done Done Sent false
189792024-02-28T22:20:20.831Z 21 Acked 1020 Read 10 Done Done Sent false
189802024-02-28T22:20:20.831Z 22 Acked 1021 Write 8 Done Done Sent false
189812024-02-28T22:20:20.831Z 23 Acked 1022 Read 6 Done Done Sent false
189822024-02-28T22:20:20.831Z 24 Acked 1023 Read 5 Done Done Sent false
189832024-02-28T22:20:20.831Z 25 Acked 1024 Write 9 Done Done Sent false
189842024-02-28T22:20:20.831Z 26 Acked 1025 Read 7 Done Done Sent false
189852024-02-28T22:20:20.831Z 27 Acked 1026 Write 8 Done Done Sent false
189862024-02-28T22:20:20.831Z 28 Acked 1027 Read 9 Done Done Sent false
189872024-02-28T22:20:20.831Z 29 Acked 1028 Read 9 Done Done Sent false
189882024-02-28T22:20:20.831Z 30 Acked 1029 Read 5 Done Sent Sent false
189892024-02-28T22:20:20.831Z STATES DS:0 DS:1 DS:2 TOTAL
189902024-02-28T22:20:20.831Z New 0 0 0 0
189912024-02-28T22:20:20.831Z Sent 0 1 26 27
189922024-02-28T22:20:20.831Z Done 30 29 4 63
189932024-02-28T22:20:20.831Z Skipped 0 0 0 0
189942024-02-28T22:20:20.831Z Error 0 0 0 0
189952024-02-28T22:20:20.831ZLast Flush: 1017 1017 0
189962024-02-28T22:20:20.832ZDownstairs last five completed:
189972024-02-28T22:20:20.832ZUpstairs last five completed: 30 29 28 27 26
189982024-02-28T22:20:20.832Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:19.107398024Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1839}
189992024-02-28T22:20:20.832ZWrote out file "/tmp/test_repair_verify.data"
190002024-02-28T22:20:20.832Z
190012024-02-28T22:20:20.832ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190022024-02-28T22:20:20.832ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190032024-02-28T22:20:20.832Z 0 000-019 174 174 174 323 323 323 F F F
190042024-02-28T22:20:20.832Z 1 020-039 176 176 173 325 325 321 F F F <---
190052024-02-28T22:20:20.832Z 2 040-059 176 176 172 325 325 320 T T T <---
190062024-02-28T22:20:20.832Z 3 060-079 176 176 170 325 325 316 T T F <---
190072024-02-28T22:20:20.832Z 4 080-099 173 173 173 321 321 321 T T F <---
190082024-02-28T22:20:20.832Z 5 100-119 174 174 174 323 323 323 T T F <---
190092024-02-28T22:20:20.832Z 6 120-139 175 175 175 324 324 324 F F F
190102024-02-28T22:20:20.832Z 7 140-159 175 175 175 324 324 324 F F F
190112024-02-28T22:20:20.832Z 8 160-179 174 174 174 322 322 322 F F F
190122024-02-28T22:20:20.832Z 9 180-199 175 175 175 324 324 324 F F F
190132024-02-28T22:20:20.832Z 10 200-219 167 167 167 306 306 306 F F F
190142024-02-28T22:20:20.832Z 11 220-239 172 172 172 320 320 320 F F F
190152024-02-28T22:20:20.832Z 12 240-259 176 176 172 326 326 320 T T F <---
190162024-02-28T22:20:20.832Z 13 260-279 169 169 169 313 313 313 F F F
190172024-02-28T22:20:20.832Z 14 280-299 172 172 172 320 320 320 F F F
190182024-02-28T22:20:20.832Z 15 300-319 172 172 172 320 320 320 F F F
190192024-02-28T22:20:20.832Z 16 320-339 172 172 172 320 320 320 F F F
190202024-02-28T22:20:20.833Z 17 340-359 173 173 173 321 321 321 T T F <---
190212024-02-28T22:20:20.833Z 18 360-379 173 173 173 321 321 321 F F F
190222024-02-28T22:20:20.833Z 19 380-399 172 172 172 320 320 320 F F F
190232024-02-28T22:20:20.833Z 20 400-419 171 171 171 317 317 317 F F F
190242024-02-28T22:20:20.837Z 21 420-439 174 174 174 323 323 323 F F F
190252024-02-28T22:20:20.837Z 22 440-459 174 174 174 323 323 323 F F F
190262024-02-28T22:20:20.837Z 23 460-479 174 174 174 323 323 323 F F F
190272024-02-28T22:20:20.837Z 24 480-499 176 176 171 325 325 317 F F F <---
190282024-02-28T22:20:20.837Z 25 500-519 174 174 174 323 323 323 F F F
190292024-02-28T22:20:20.837Z 26 520-539 176 176 174 326 326 322 F F F <---
190302024-02-28T22:20:20.837Z 27 540-559 175 175 175 324 324 324 F F F
190312024-02-28T22:20:20.837Z 28 560-579 176 176 174 325 325 323 F F F <---
190322024-02-28T22:20:20.837Z 29 580-599 176 176 174 325 325 323 F F F <---
190332024-02-28T22:20:20.837ZMax gen: 176, Max flush: 326
190342024-02-28T22:20:20.837ZA Difference in extent metadata is expected here
190352024-02-28T22:20:20.837ZOn loop 88
190362024-02-28T22:20:20.837Z
190372024-02-28T22:20:20.837ZVerifying data now
190382024-02-28T22:20:24.838ZNo change after verify
190392024-02-28T22:20:24.841ZLoop: 88 Downstairs dump after verify (and repair):
190402024-02-28T22:20:24.852ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190412024-02-28T22:20:24.853Z 0 000-019 174 174 174 323 323 323 F F F
190422024-02-28T22:20:24.853Z 1 020-039 176 176 176 325 325 325 F F F
190432024-02-28T22:20:24.853Z 2 040-059 177 177 177 327 327 327 F F F
190442024-02-28T22:20:24.853Z 3 060-079 177 177 177 327 327 327 F F F
190452024-02-28T22:20:24.853Z 4 080-099 177 177 177 327 327 327 F F F
190462024-02-28T22:20:24.853Z 5 100-119 177 177 177 327 327 327 F F F
190472024-02-28T22:20:24.853Z 6 120-139 175 175 175 324 324 324 F F F
190482024-02-28T22:20:24.853Z 7 140-159 175 175 175 324 324 324 F F F
190492024-02-28T22:20:24.853Z 8 160-179 174 174 174 322 322 322 F F F
190502024-02-28T22:20:24.853Z 9 180-199 175 175 175 324 324 324 F F F
190512024-02-28T22:20:24.853Z 10 200-219 167 167 167 306 306 306 F F F
190522024-02-28T22:20:24.853Z 11 220-239 172 172 172 320 320 320 F F F
190532024-02-28T22:20:24.857Z 12 240-259 177 177 177 327 327 327 F F F
190542024-02-28T22:20:24.857Z 13 260-279 169 169 169 313 313 313 F F F
190552024-02-28T22:20:24.857Z 14 280-299 172 172 172 320 320 320 F F F
190562024-02-28T22:20:24.857Z 15 300-319 172 172 172 320 320 320 F F F
190572024-02-28T22:20:24.857Z 16 320-339 172 172 172 320 320 320 F F F
190582024-02-28T22:20:24.857Z 17 340-359 177 177 177 327 327 327 F F F
190592024-02-28T22:20:24.857Z 18 360-379 173 173 173 321 321 321 F F F
190602024-02-28T22:20:24.857Z 19 380-399 172 172 172 320 320 320 F F F
190612024-02-28T22:20:24.857Z 20 400-419 171 171 171 317 317 317 F F F
190622024-02-28T22:20:24.857Z 21 420-439 174 174 174 323 323 323 F F F
190632024-02-28T22:20:24.857Z 22 440-459 174 174 174 323 323 323 F F F
190642024-02-28T22:20:24.857Z 23 460-479 174 174 174 323 323 323 F F F
190652024-02-28T22:20:24.857Z 24 480-499 176 176 176 325 325 325 F F F
190662024-02-28T22:20:24.857Z 25 500-519 174 174 174 323 323 323 F F F
190672024-02-28T22:20:24.857Z 26 520-539 176 176 176 326 326 326 F F F
190682024-02-28T22:20:24.857Z 27 540-559 175 175 175 324 324 324 F F F
190692024-02-28T22:20:24.857Z 28 560-579 176 176 176 325 325 325 F F F
190702024-02-28T22:20:24.858Z 29 580-599 176 176 176 325 325 325 F F F
190712024-02-28T22:20:24.858ZMax gen: 177, Max flush: 327
190722024-02-28T22:20:24.858Z
190732024-02-28T22:20:24.858ZBegin loop 89
190742024-02-28T22:20:24.858ZDownstairs to restart: 0
190752024-02-28T22:20:24.875Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754514381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
190762024-02-28T22:20:24.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754787994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
190772024-02-28T22:20:24.882Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754826883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
190782024-02-28T22:20:24.882Z{"msg":"Crucible d14a8025-a137-47bf-9ec5-8e0f988d0d79 has session id: 9dde5c3d-d9b8-4d54-8758-adf20d264735","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754859382Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847Crucible runtime is spawned
190792024-02-28T22:20:24.883Z,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190802024-02-28T22:20:24.883Z{"msg":"Upstairs opts: Upstairs UUID: d14a8025-a137-47bf-9ec5-8e0f988d0d79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754895611Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190812024-02-28T22:20:24.883Z{"msg":"Crucible stats registered with UUID: d14a8025-a137-47bf-9ec5-8e0f988d0d79","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.75491566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190822024-02-28T22:20:24.883Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.754946389Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
190832024-02-28T22:20:24.883Z{"msg":"d14a8025-a137-47bf-9ec5-8e0f988d0d79 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755068346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190842024-02-28T22:20:24.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755101345Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190852024-02-28T22:20:24.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755127214Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190862024-02-28T22:20:24.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755163893Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190872024-02-28T22:20:24.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755190113Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190882024-02-28T22:20:24.883Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755231841Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190892024-02-28T22:20:24.883Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755260461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190902024-02-28T22:20:24.883Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.7552938Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190912024-02-28T22:20:24.883Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755378557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190922024-02-28T22:20:24.883Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755419246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190932024-02-28T22:20:24.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755446415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190942024-02-28T22:20:24.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755513594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190952024-02-28T22:20:24.883Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.755539603Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190962024-02-28T22:20:24.883Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756088998Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190972024-02-28T22:20:24.883Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756127517Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190982024-02-28T22:20:24.883Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756268223Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
190992024-02-28T22:20:24.883Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756330241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191002024-02-28T22:20:24.884Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756631483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191012024-02-28T22:20:24.884Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756684081Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191022024-02-28T22:20:24.884Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.756713681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191032024-02-28T22:20:24.884Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.75675561Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191042024-02-28T22:20:24.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757201887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191052024-02-28T22:20:24.884Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757240286Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191062024-02-28T22:20:24.884Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757258106Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191072024-02-28T22:20:24.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757311784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191082024-02-28T22:20:24.884Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757360573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191092024-02-28T22:20:24.884Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757405642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191102024-02-28T22:20:24.884Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757444071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191112024-02-28T22:20:24.884Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757920367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191122024-02-28T22:20:24.884Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.757966386Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191132024-02-28T22:20:24.889Z{"msg":"[0]R flush_numbers[0..12]: [323, 325, 327, 327, 327, 327, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758007475Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191142024-02-28T22:20:24.889Z{"msg":"[0]R generation[0..12]: [174, 176, 177, 177, 177, 177, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758046794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191152024-02-28T22:20:24.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
191162024-02-28T22:20:24.890Z,"time":"2024-02-28T22:20:23.758081213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191172024-02-28T22:20:24.890Z{"msg":"[1]R flush_numbers[0..12]: [323, 325, 327, 327, 327, 327, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758117672Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191182024-02-28T22:20:24.890Z{"msg":"[1]R generation[0..12]: [174, 176, 177, 177, 177, 177, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758155851Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191192024-02-28T22:20:24.890Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.75818025Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191202024-02-28T22:20:24.890Z{"msg":"[2]R flush_numbers[0..12]: [323, 325, 327, 327, 327, 327, 324, 324, 322, 324, 306, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.75819546Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191212024-02-28T22:20:24.890Z{"msg":"[2]R generation[0..12]: [174, 176, 177, 177, 177, 177, 175, 175, 174, 175, 167, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758218639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191222024-02-28T22:20:24.890Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758238109Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191232024-02-28T22:20:24.890Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758267458Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191242024-02-28T22:20:24.890Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758291337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191252024-02-28T22:20:24.890Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758306067Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191262024-02-28T22:20:24.890Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758331506Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"":"downstairs","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191272024-02-28T22:20:24.890Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758357105Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191282024-02-28T22:20:24.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758372155Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"0","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191292024-02-28T22:20:24.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758393684Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"1","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191302024-02-28T22:20:24.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758408374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"client":"2","":"downstairs","session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191312024-02-28T22:20:24.890Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191322024-02-28T22:20:24.890Z,"time":"2024-02-28T22:20:23.758429133Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191332024-02-28T22:20:24.890Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758445093Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191342024-02-28T22:20:24.890Z{"msg":"d14a8025-a137-47bf-9ec5-8e0f988d0d79 is now active with session: 9dde5c3d-d9b8-4d54-8758-adf20d264735","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758464823Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191352024-02-28T22:20:24.890Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758477242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847,"session_id":"9dde5c3d-d9b8-4d54-8758-adf20d264735"}
191362024-02-28T22:20:24.890Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:23.758496511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
191372024-02-28T22:20:24.890ZLoading write count information from file "/tmp/test_repair_verify.data"
191382024-02-28T22:20:24.891ZRun Repair workload
191392024-02-28T22:20:24.891Z01/30 Write block 33 len 20480 data: 11 9 9 12 13
191402024-02-28T22:20:24.891Z02/30 Read block 44 len 20480
191412024-02-28T22:20:24.891Z03/30 Read block 265 len 40960
191422024-02-28T22:20:24.891Z04/30 Write block 455 len 40960 data: 17 16 14 14 11 8 8 9 12 11
191432024-02-28T22:20:24.891Z05/30 Write block 200 len 16384 data: 9 11 11 10
191442024-02-28T22:20:24.891Z06/30 Flush
191452024-02-28T22:20:24.894Z07/30 Write block 215 len 12288 data: 10 11 12
191462024-02-28T22:20:24.894Z08/30 Read block 331 len 24576
191472024-02-28T22:20:24.894Z09/30 Read block 450 len 16384
191482024-02-28T22:20:24.894Z10/30 Read block 472 len 28672
191492024-02-28T22:20:24.894Z11/30 Write block 179 len 32768 data: 8 9 9 11 13 13 12 9
191502024-02-28T22:20:24.894Z12/30 Read block 261 len 32768
191512024-02-28T22:20:24.895Z13/30 Write block 574 len 20480 data: 14 15 15 16 16
191522024-02-28T22:20:24.895Z14/30 Read block 221 len 28672
191532024-02-28T22:20:24.899Z15/30 Write block 520 len 12288 data: 19 17 15
191542024-02-28T22:20:24.899Z16/30 Write block 105 len 16384 data: 11 12 12 16
191552024-02-28T22:20:24.899Z17/30 Write block 19 len 24576 data: 14 12 11 9 8 8
191562024-02-28T22:20:24.899Z18/30 Read block 281 len 24576
191572024-02-28T22:20:24.899Z19/30 Read block 549 len 32768
191582024-02-28T22:20:24.899Z20/30 Read block 502 len 40960
191592024-02-28T22:20:24.902Z21/30 Write block 511 len 28672 data: 12 15 15 16 17 18 20
191602024-02-28T22:20:24.902Z22/30 Read block 471 len 28672
191612024-02-28T22:20:24.902Z23/30 Read block 359 len 16384
191622024-02-28T22:20:24.905Z24/30 Read block 114 len 12288
191632024-02-28T22:20:24.905Z25/30 Flush
191642024-02-28T22:20:24.909Z26/30 Write block 74 len 12288 data: 13 10 8
191652024-02-28T22:20:24.914Z27/30 Write block 572 len 12288 data: 13 15 15
191662024-02-28T22:20:24.914Z28/30 Write block 123 len 4096 data: 13
191672024-02-28T22:20:24.914Z29/30 Read block 462 len 4096
191682024-02-28T22:20:24.914Z30/30 Read block 206 len 36864
191692024-02-28T22:20:24.914Z----------------------------------------------------------------
191702024-02-28T22:20:24.914Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
191712024-02-28T22:20:24.914ZStates: Active Active Active
191722024-02-28T22:20:24.914ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
191732024-02-28T22:20:24.914Z 1 Acked 1000 Write 5 Done Done Done false
191742024-02-28T22:20:24.914Z 2 Acked 1001 Read 5 Done Done Done false
191752024-02-28T22:20:24.914Z 3 Acked 1002 Read 10 Done Done Done false
191762024-02-28T22:20:24.914Z 4 Acked 1003 Write 10 Done Done Done false
191772024-02-28T22:20:24.914Z 5 Acked 1004 Write 4 Done Done Done false
191782024-02-28T22:20:24.914Z 6 Acked 1005 Flush 0 Sent Done Done false
191792024-02-28T22:20:24.914Z 7 Acked 1006 Write 3 Sent Done Done false
191802024-02-28T22:20:24.914Z 8 Acked 1007 Read 6 Sent Done Done false
191812024-02-28T22:20:24.914Z 9 Acked 1008 Read 4 Sent Done Done false
191822024-02-28T22:20:24.915Z 10 Acked 1009 Read 7 Sent Done Done false
191832024-02-28T22:20:24.915Z 11 Acked 1010 Write 8 Sent Done Done false
191842024-02-28T22:20:24.915Z 12 Acked 1011 Read 8 Sent Done Done false
191852024-02-28T22:20:24.915Z 13 Acked 1012 Write 5 Sent Done Done false
191862024-02-28T22:20:24.915Z 14 Acked 1013 Read 7 Sent Done Done false
191872024-02-28T22:20:24.915Z 15 Acked 1014 Write 3 Sent Done Done false
191882024-02-28T22:20:24.915Z 16 Acked 1015 Write 4 Sent Done Done false
191892024-02-28T22:20:24.915Z 17 Acked 1016 Write 6 Sent Done Done false
191902024-02-28T22:20:24.915Z 18 Acked 1017 Read 6 Sent Done Done false
191912024-02-28T22:20:24.915Z 19 Acked 1018 Read 8 Sent Done Done false
191922024-02-28T22:20:24.915Z 20 Acked 1019 Read 10 Sent Done Done false
191932024-02-28T22:20:24.915Z 21 Acked 1020 Write 7 Sent Done Done false
191942024-02-28T22:20:24.915Z 22 Acked 1021 Read 7 Sent Done Done false
191952024-02-28T22:20:24.915Z 23 Acked 1022 Read 4 Sent Done Done false
191962024-02-28T22:20:24.915Z 24 Acked 1023 Read 3 Sent Done Done false
191972024-02-28T22:20:24.915Z 25 Acked 1024 Flush 0 Sent Done Done false
191982024-02-28T22:20:24.915Z 26 Acked 1025 Write 3 Sent Done Done false
191992024-02-28T22:20:24.915Z 27 Acked 1026 Write 3 Sent Done Done false
192002024-02-28T22:20:24.915Z 28 Acked 1027 Write 1 Sent Done Done false
192012024-02-28T22:20:24.915Z 29 Acked 1028 Read 1 Sent Done Done false
192022024-02-28T22:20:24.915Z 30 Acked 1029 Read 9 Sent Sent Done false
192032024-02-28T22:20:24.915Z STATES DS:0 DS:1 DS:2 TOTAL
192042024-02-28T22:20:24.915Z New 0 0 0 0
192052024-02-28T22:20:24.916Z Sent 25 1 0 26
192062024-02-28T22:20:24.916Z Done 5 29 30 64
192072024-02-28T22:20:24.916Z Skipped 0 0 0 0
192082024-02-28T22:20:24.916Z Error 0 0 0 0
192092024-02-28T22:20:24.916ZLast Flush: 0 1024 1024
192102024-02-28T22:20:24.916ZDownstairs last five completed:
192112024-02-28T22:20:24.916ZUpstairs last five completed: 30 29 28 27 26
192122024-02-28T22:20:24.916Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:23.791815954Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1847}
192132024-02-28T22:20:24.981ZWrote out file "/tmp/test_repair_verify.data"
192142024-02-28T22:20:24.982Z
192152024-02-28T22:20:24.982ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192162024-02-28T22:20:24.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192172024-02-28T22:20:24.985Z 0 000-019 174 178 178 323 329 329 F F F <---
192182024-02-28T22:20:24.985Z 1 020-039 176 178 178 325 329 329 T F F <---
192192024-02-28T22:20:24.985Z 2 040-059 177 177 177 327 327 327 F F F
192202024-02-28T22:20:24.985Z 3 060-079 177 177 177 327 327 327 F T T <---
192212024-02-28T22:20:24.985Z 4 080-099 177 177 177 327 327 327 F F F
192222024-02-28T22:20:24.985Z 5 100-119 177 178 178 327 329 329 F F F <---
192232024-02-28T22:20:24.985Z 6 120-139 175 175 175 324 324 324 F T T <---
192242024-02-28T22:20:24.985Z 7 140-159 175 175 175 324 324 324 F F F
192252024-02-28T22:20:24.985Z 8 160-179 174 178 178 322 329 329 F F F <---
192262024-02-28T22:20:24.986Z 9 180-199 175 178 178 324 329 329 F F F <---
192272024-02-28T22:20:24.986Z 10 200-219 167 178 178 306 329 329 T F F <---
192282024-02-28T22:20:24.986Z 11 220-239 172 172 172 320 320 320 F F F
192292024-02-28T22:20:24.986Z 12 240-259 177 177 177 327 327 327 F F F
192302024-02-28T22:20:24.986Z 13 260-279 169 169 169 313 313 313 F F F
192312024-02-28T22:20:24.986Z 14 280-299 172 172 172 320 320 320 F F F
192322024-02-28T22:20:24.986Z 15 300-319 172 172 172 320 320 320 F F F
192332024-02-28T22:20:24.986Z 16 320-339 172 172 172 320 320 320 F F F
192342024-02-28T22:20:24.986Z 17 340-359 177 177 177 327 327 327 F F F
192352024-02-28T22:20:24.986Z 18 360-379 173 173 173 321 321 321 F F F
192362024-02-28T22:20:24.986Z 19 380-399 172 172 172 320 320 320 F F F
192372024-02-28T22:20:24.986Z 20 400-419 171 171 171 317 317 317 F F F
192382024-02-28T22:20:24.986Z 21 420-439 174 174 174 323 323 323 F F F
192392024-02-28T22:20:24.986Z 22 440-459 174 178 178 323 328 328 T F F <---
192402024-02-28T22:20:24.986Z 23 460-479 174 178 178 323 328 328 T F F <---
192412024-02-28T22:20:24.986Z 24 480-499 176 176 176 325 325 325 F F F
192422024-02-28T22:20:24.986Z 25 500-519 174 178 178 323 329 329 F F F <---
192432024-02-28T22:20:24.986Z 26 520-539 176 178 178 326 329 329 F F F <---
192442024-02-28T22:20:24.986Z 27 540-559 175 175 175 324 324 324 F F F
192452024-02-28T22:20:24.986Z 28 560-579 176 178 178 325 329 329 F T T <---
192462024-02-28T22:20:24.986Z 29 580-599 176 176 176 325 325 325 F F F
192472024-02-28T22:20:24.986ZMax gen: 178, Max flush: 329
192482024-02-28T22:20:24.986ZError: Difference in extent metadata found!
192492024-02-28T22:20:24.987ZA Difference in extent metadata is expected here
192502024-02-28T22:20:24.987ZOn loop 89
192512024-02-28T22:20:24.987Z
192522024-02-28T22:20:24.987ZVerifying data now
192532024-02-28T22:20:30.605ZNo change after verify
192542024-02-28T22:20:30.605ZLoop: 89 Downstairs dump after verify (and repair):
192552024-02-28T22:20:30.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192562024-02-28T22:20:30.617Z 0 000-019 178 178 178 329 329 329 F F F
192572024-02-28T22:20:30.618Z 1 020-039 178 178 178 329 329 329 F F F
192582024-02-28T22:20:30.618Z 2 040-059 177 177 177 327 327 327 F F F
192592024-02-28T22:20:30.618Z 3 060-079 179 179 179 330 330 330 F F F
192602024-02-28T22:20:30.618Z 4 080-099 177 177 177 327 327 327 F F F
192612024-02-28T22:20:30.618Z 5 100-119 178 178 178 329 329 329 F F F
192622024-02-28T22:20:30.618Z 6 120-139 179 179 179 330 330 330 F F F
192632024-02-28T22:20:30.624Z 7 140-159 175 175 175 324 324 324 F F F
192642024-02-28T22:20:30.624Z 8 160-179 178 178 178 329 329 329 F F F
192652024-02-28T22:20:30.624Z 9 180-199 178 178 178 329 329 329 F F F
192662024-02-28T22:20:30.624Z 10 200-219 178 178 178 329 329 329 F F F
192672024-02-28T22:20:30.624Z 11 220-239 172 172 172 320 320 320 F F F
192682024-02-28T22:20:30.625Z 12 240-259 177 177 177 327 327 327 F F F
192692024-02-28T22:20:30.625Z 13 260-279 169 169 169 313 313 313 F F F
192702024-02-28T22:20:30.625Z 14 280-299 172 172 172 320 320 320 F F F
192712024-02-28T22:20:30.625Z 15 300-319 172 172 172 320 320 320 F F F
192722024-02-28T22:20:30.625Z 16 320-339 172 172 172 320 320 320 F F F
192732024-02-28T22:20:30.626Z 17 340-359 177 177 177 327 327 327 F F F
192742024-02-28T22:20:30.626Z 18 360-379 173 173 173 321 321 321 F F F
192752024-02-28T22:20:30.626Z 19 380-399 172 172 172 320 320 320 F F F
192762024-02-28T22:20:30.626Z 20 400-419 171 171 171 317 317 317 F F F
192772024-02-28T22:20:30.626Z 21 420-439 174 174 174 323 323 323 F F F
192782024-02-28T22:20:30.626Z 22 440-459 178 178 178 328 328 328 F F F
192792024-02-28T22:20:30.627Z 23 460-479 178 178 178 328 328 328 F F F
192802024-02-28T22:20:30.627Z 24 480-499 176 176 176 325 325 325 F F F
192812024-02-28T22:20:30.627Z 25 500-519 178 178 178 329 329 329 F F F
192822024-02-28T22:20:30.627Z 26 520-539 178 178 178 329 329 329 F F F
192832024-02-28T22:20:30.627Z 27 540-559 175 175 175 324 324 324 F F F
192842024-02-28T22:20:30.627Z 28 560-579 179 179 179 330 330 330 F F F
192852024-02-28T22:20:30.627Z 29 580-599 176 176 176 325 325 325 F F F
192862024-02-28T22:20:30.627ZMax gen: 179, Max flush: 330
192872024-02-28T22:20:30.627Z
192882024-02-28T22:20:30.628ZBegin loop 90
192892024-02-28T22:20:30.628ZDownstairs to restart: 2
192902024-02-28T22:20:30.636Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.514559748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
192912024-02-28T22:20:30.636ZCrucible runtime is spawned
192922024-02-28T22:20:30.672Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.514944727Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
192932024-02-28T22:20:30.672Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.514984066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
192942024-02-28T22:20:30.672Z{"msg":"Crucible 06f86650-74cf-4b23-bb6e-358284ef7cbc has session id: 5ce6b4d7-305f-433f-a682-0b24a6ca8e22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515013976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
192952024-02-28T22:20:30.672Z{"msg":"Upstairs opts: Upstairs UUID: 06f86650-74cf-4b23-bb6e-358284ef7cbc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515083663Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
192962024-02-28T22:20:30.672Z{"msg":"Crucible stats registered with UUID: 06f86650-74cf-4b23-bb6e-358284ef7cbc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515128802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
192972024-02-28T22:20:30.673Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515173001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
192982024-02-28T22:20:30.673Z{"msg":"06f86650-74cf-4b23-bb6e-358284ef7cbc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.5152122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
192992024-02-28T22:20:30.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515240009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193002024-02-28T22:20:30.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515282198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193012024-02-28T22:20:30.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515312817Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193022024-02-28T22:20:30.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515358236Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193032024-02-28T22:20:30.673Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515390635Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193042024-02-28T22:20:30.673Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515464713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193052024-02-28T22:20:30.673Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515499732Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193062024-02-28T22:20:30.674Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515529001Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193072024-02-28T22:20:30.674Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.51555399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193082024-02-28T22:20:30.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.51559064Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193092024-02-28T22:20:30.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515615289Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193102024-02-28T22:20:30.674Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.515643218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193112024-02-28T22:20:30.674Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.5162866Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193122024-02-28T22:20:30.675Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516322899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193132024-02-28T22:20:30.675Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516350799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193142024-02-28T22:20:30.675Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516374678Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193152024-02-28T22:20:30.675Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516389627Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193162024-02-28T22:20:30.683Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516404097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193172024-02-28T22:20:30.683Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516923303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193182024-02-28T22:20:30.683Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.516987411Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193192024-02-28T22:20:30.683Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.5170261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193202024-02-28T22:20:30.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517434069Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193212024-02-28T22:20:30.684Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517481128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193222024-02-28T22:20:30.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517510347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193232024-02-28T22:20:30.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517534576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193242024-02-28T22:20:30.684Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517550586Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193252024-02-28T22:20:30.684Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517575275Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193262024-02-28T22:20:30.684Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517588594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193272024-02-28T22:20:30.684Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517608794Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193282024-02-28T22:20:30.684Z{"msg":"[0]R flush_numbers[0..12]: [329, 329, 327, 330, 327, 329, 330, 324, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517623233Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193292024-02-28T22:20:30.685Z{"msg":"[0]R generation[0..12]: [178, 178, 177, 179, 177, 178, 179, 175, 178, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517643863Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193302024-02-28T22:20:30.685Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517658332Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193312024-02-28T22:20:30.685Z{"msg":"[1]R flush_numbers[0..12]: [329, 329, 327, 330, 327, 329, 330, 324, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517679122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193322024-02-28T22:20:30.685Z{"msg":"[1]R generation[0..12]: [178, 178, 177, 179, 177, 178, 179, 175, 178, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517693262Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193332024-02-28T22:20:30.685Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517714091Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193342024-02-28T22:20:30.685Z{"msg":"[2]R flush_numbers[0..12]: [329, 329, 327, 330, 327, 329, 330, 324, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517728411Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193352024-02-28T22:20:30.686Z{"msg":"[2]R generation[0..12]: [178, 178, 177, 179, 177, 178, 179, 175, 178, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.51774877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193362024-02-28T22:20:30.686Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.51776306Z","hostname":"Wait for a query_work_queue command to finish before sending IO
193372024-02-28T22:20:30.686Zip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193382024-02-28T22:20:30.686Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517790799Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193392024-02-28T22:20:30.686Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517805399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193402024-02-28T22:20:30.686Z{"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517825388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193412024-02-28T22:20:30.687Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517839078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"":"downstairs","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193422024-02-28T22:20:30.687Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517860237Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193432024-02-28T22:20:30.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517873287Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"0","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193442024-02-28T22:20:30.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517893636Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"1","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193452024-02-28T22:20:30.687Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517907186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"client":"2","":"downstairs","session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193462024-02-28T22:20:30.687Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517927045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193472024-02-28T22:20:30.687Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517939725Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193482024-02-28T22:20:30.687Z{"msg":"06f86650-74cf-4b23-bb6e-358284ef7cbc is now active with session: 5ce6b4d7-305f-433f-a682-0b24a6ca8e22","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517958914Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193492024-02-28T22:20:30.687Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517971274Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855,"session_id":"5ce6b4d7-305f-433f-a682-0b24a6ca8e22"}
193502024-02-28T22:20:30.687Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:29.517990003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
193512024-02-28T22:20:30.688ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193522024-02-28T22:20:30.688ZLoading write count information from file "/tmp/test_repair_verify.data"
193532024-02-28T22:20:30.688ZRun Repair workload
193542024-02-28T22:20:30.688Z01/30 Write block 349 len 40960 data: 12 13 15 16 17 17 16 13 15 13
193552024-02-28T22:20:30.699Z02/30 Read block 44 len 40960
193562024-02-28T22:20:30.699Z03/30 Write block 511 len 36864 data: 13 16 16 17 18 19 21 18 19
193572024-02-28T22:20:30.699Z04/30 Read block 443 len 20480
193582024-02-28T22:20:30.699Z05/30 Read block 219 len 20480
193592024-02-28T22:20:30.699Z06/30 Flush
193602024-02-28T22:20:30.700Z07/30 Write block 325 len 32768 data: 16 17 17 21 21 20 18 13
193612024-02-28T22:20:30.700Z08/30 Read block 286 len 32768
193622024-02-28T22:20:30.700Z09/30 Write block 467 len 40960 data: 12 12 11 14 16 17 14 15 12 13
193632024-02-28T22:20:30.700Z10/30 Read block 453 len 36864
193642024-02-28T22:20:30.700Z11/30 Write block 17 len 8192 data: 14 13
193652024-02-28T22:20:30.701Z12/30 Write block 106 len 16384 data: 13 13 17 20
193662024-02-28T22:20:30.701Z13/30 Flush
193672024-02-28T22:20:30.701Z14/30 Write block 107 len 12288 data: 14 18 21
193682024-02-28T22:20:30.701Z15/30 Write block 44 len 8192 data: 9 11
193692024-02-28T22:20:30.701Z16/30 Write block 146 len 32768 data: 19 18 19 17 14 11 11 7
193702024-02-28T22:20:30.701Z17/30 Write block 419 len 28672 data: 14 16 17 17 15 14 12
193712024-02-28T22:20:30.702Z18/30 Write block 597 len 8192 data: 10 10
193722024-02-28T22:20:30.702Z19/30 Read block 409 len 36864
193732024-02-28T22:20:30.702Z20/30 Write block 65 len 24576 data: 13 15 16 16 12 14
193742024-02-28T22:20:30.702Z21/30 Read block 518 len 20480
193752024-02-28T22:20:30.702Z22/30 Read block 482 len 4096
193762024-02-28T22:20:30.702Z23/30 Read block 136 len 28672
193772024-02-28T22:20:30.703Z24/30 Read block 415 len 4096
193782024-02-28T22:20:30.703Z25/30 Read block 582 len 40960
193792024-02-28T22:20:30.703Z26/30 Flush
193802024-02-28T22:20:30.703Z27/30 Write block 568 len 8192 data: 17 14
193812024-02-28T22:20:30.703Z28/30 Read block 388 len 36864
193822024-02-28T22:20:30.704Z29/30 Read block 397 len 20480
193832024-02-28T22:20:30.704Z30/30 Write block 308 len 36864 data: 15 14 11 15 15 15 15 11 11
193842024-02-28T22:20:30.704Z----------------------------------------------------------------
193852024-02-28T22:20:30.704Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
193862024-02-28T22:20:30.704ZStates: Active Active Active
193872024-02-28T22:20:30.704ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
193882024-02-28T22:20:30.705Z 1 Acked 1000 Write 10 Done Done Sent false
193892024-02-28T22:20:30.705Z 2 Acked 1001 Read 10 Done Done Sent false
193902024-02-28T22:20:30.705Z 3 Acked 1002 Write 9 Done Done Sent false
193912024-02-28T22:20:30.705Z 4 Acked 1003 Read 5 Done Done Sent false
193922024-02-28T22:20:30.705Z 5 Acked 1004 Read 5 Done Done Sent false
193932024-02-28T22:20:30.705Z 6 Acked 1005 Flush 0 Done Done Sent false
193942024-02-28T22:20:30.706Z 7 Acked 1006 Write 8 Done Done Sent false
193952024-02-28T22:20:30.706Z 8 Acked 1007 Read 8 Done Done Sent false
193962024-02-28T22:20:30.706Z 9 Acked 1008 Write 10 Done Done Sent false
193972024-02-28T22:20:30.706Z 10 Acked 1009 Read 9 Done Done Sent false
193982024-02-28T22:20:30.706Z 11 Acked 1010 Write 2 Done Done Sent false
193992024-02-28T22:20:30.706Z 12 Acked 1011 Write 4 Done Done Sent false
194002024-02-28T22:20:30.707Z 13 Acked 1012 Flush 0 Done Done Sent false
194012024-02-28T22:20:30.707Z 14 Acked 1013 Write 3 Done Done Sent false
194022024-02-28T22:20:30.707Z 15 Acked 1014 Write 2 Done Done Sent false
194032024-02-28T22:20:30.707Z 16 Acked 1015 Write 8 Done Done Sent false
194042024-02-28T22:20:30.707Z 17 Acked 1016 Write 7 Done Done Sent false
194052024-02-28T22:20:30.718Z 18 Acked 1017 Write 2 Done Done Sent false
194062024-02-28T22:20:30.718Z 19 Acked 1018 Read 9 Done Done Sent false
194072024-02-28T22:20:30.718Z 20 Acked 1019 Write 6 Done Done Sent false
194082024-02-28T22:20:30.719Z 21 Acked 1020 Read 5 Done Done Sent false
194092024-02-28T22:20:30.719Z 22 Acked 1021 Read 1 Done Done Sent false
194102024-02-28T22:20:30.733Z 23 Acked 1022 Read 7 Done Done Sent false
194112024-02-28T22:20:30.733Z 24 Acked 1023 Read 1 Done Done Sent false
194122024-02-28T22:20:30.733Z 25 Acked 1024 Read 10 Done Done Sent false
194132024-02-28T22:20:30.733Z 26 Acked 1025 Flush 0 Done Done Sent false
194142024-02-28T22:20:30.733Z 27 Acked 1026 Write 2 Done Done Sent false
194152024-02-28T22:20:30.733Z 28 Acked 1027 Read 9 Done Done Sent false
194162024-02-28T22:20:30.733Z 29 Acked 1028 Read 5 Done Done Sent false
194172024-02-28T22:20:30.734Z 30 Acked 1029 Write 9 Sent Sent Sent false
194182024-02-28T22:20:30.734Z STATES DS:0 DS:1 DS:2 TOTAL
194192024-02-28T22:20:30.734Z New 0 0 0 0
194202024-02-28T22:20:30.734Z Sent 1 1 30 32
194212024-02-28T22:20:30.734Z Done 29 29 0 58
194222024-02-28T22:20:30.734Z Skipped 0 0 0 0
194232024-02-28T22:20:30.734Z Error 0 0 0 0
194242024-02-28T22:20:30.734ZLast Flush: 1025 1025 0
194252024-02-28T22:20:30.734ZDownstairs last five completed:
194262024-02-28T22:20:30.734ZUpstairs last five completed: 30 29 28 27 26
194272024-02-28T22:20:30.734Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:29.551254637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1855}
194282024-02-28T22:20:30.735ZWrote out file "/tmp/test_repair_verify.data"
194292024-02-28T22:20:30.735Z
194302024-02-28T22:20:30.735ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194312024-02-28T22:20:30.735ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194322024-02-28T22:20:30.735Z 0 000-019 180 180 178 332 332 329 F F F <---
194332024-02-28T22:20:30.735Z 1 020-039 178 178 178 329 329 329 F F F
194342024-02-28T22:20:30.735Z 2 040-059 180 180 177 333 333 327 F F F <---
194352024-02-28T22:20:30.735Z 3 060-079 180 180 179 333 333 330 F F F <---
194362024-02-28T22:20:30.735Z 4 080-099 177 177 177 327 327 327 F F F
194372024-02-28T22:20:30.735Z 5 100-119 180 180 178 333 333 329 F F F <---
194382024-02-28T22:20:30.735Z 6 120-139 179 179 179 330 330 330 F F F
194392024-02-28T22:20:30.735Z 7 140-159 180 180 175 333 333 324 F F F <---
194402024-02-28T22:20:30.735Z 8 160-179 178 178 178 329 329 329 F F F
194412024-02-28T22:20:30.736Z 9 180-199 178 178 178 329 329 329 F F F
194422024-02-28T22:20:30.736Z 10 200-219 178 178 178 329 329 329 F F F
194432024-02-28T22:20:30.736Z 11 220-239 172 172 172 320 320 320 F F F
194442024-02-28T22:20:30.736Z 12 240-259 177 177 177 327 327 327 F F F
194452024-02-28T22:20:30.736Z 13 260-279 169 169 169 313 313 313 F F F
194462024-02-28T22:20:30.736Z 14 280-299 172 172 172 320 320 320 F F F
194472024-02-28T22:20:30.736Z 15 300-319 172 172 172 320 320 320 T T F <---
194482024-02-28T22:20:30.736Z 16 320-339 180 180 172 332 332 320 F F F <---
194492024-02-28T22:20:30.736Z 17 340-359 180 180 177 331 331 327 F F F <---
194502024-02-28T22:20:30.736Z 18 360-379 173 173 173 321 321 321 F F F
194512024-02-28T22:20:30.736Z 19 380-399 172 172 172 320 320 320 F F F
194522024-02-28T22:20:30.736Z 20 400-419 180 180 171 333 333 317 F F F <---
194532024-02-28T22:20:30.737Z 21 420-439 180 180 174 333 333 323 F F F <---
194542024-02-28T22:20:30.737Z 22 440-459 178 178 178 328 328 328 F F F
194552024-02-28T22:20:30.754Z 23 460-479 180 180 178 332 332 328 F F F <---
194562024-02-28T22:20:30.755Z 24 480-499 176 176 176 325 325 325 F F F
194572024-02-28T22:20:30.755Z 25 500-519 180 180 178 331 331 329 F F F <---
194582024-02-28T22:20:30.755Z 26 520-539 178 178 178 329 329 329 F F F
194592024-02-28T22:20:30.755Z 27 540-559 175 175 175 324 324 324 F F F
194602024-02-28T22:20:30.755Z 28 560-579 179 179 179 330 330 330 T T F <---
194612024-02-28T22:20:30.755Z 29 580-599 180 180 176 333 333 325 F F F <---
194622024-02-28T22:20:30.755ZMax gen: 180, Max flush: 333
194632024-02-28T22:20:30.755ZError: Difference in extent metadata found!
194642024-02-28T22:20:30.755ZA Difference in extent metadata is expected here
194652024-02-28T22:20:30.755ZOn loop 90
194662024-02-28T22:20:30.755Z
194672024-02-28T22:20:30.756ZVerifying data now
194682024-02-28T22:20:35.798ZNo change after verify
194692024-02-28T22:20:35.799ZLoop: 90 Downstairs dump after verify (and repair):
194702024-02-28T22:20:35.807ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194712024-02-28T22:20:35.807Z 0 000-019 180 180 180 332 332 332 F F F
194722024-02-28T22:20:35.807Z 1 020-039 178 178 178 329 329 329 F F F
194732024-02-28T22:20:35.807Z 2 040-059 180 180 180 333 333 333 F F F
194742024-02-28T22:20:35.807Z 3 060-079 180 180 180 333 333 333 F F F
194752024-02-28T22:20:35.807Z 4 080-099 177 177 177 327 327 327 F F F
194762024-02-28T22:20:35.808Z 5 100-119 180 180 180 333 333 333 F F F
194772024-02-28T22:20:35.808Z 6 120-139 179 179 179 330 330 330 F F F
194782024-02-28T22:20:35.808Z 7 140-159 180 180 180 333 333 333 F F F
194792024-02-28T22:20:35.808Z 8 160-179 178 178 178 329 329 329 F F F
194802024-02-28T22:20:35.808Z 9 180-199 178 178 178 329 329 329 F F F
194812024-02-28T22:20:35.808Z 10 200-219 178 178 178 329 329 329 F F F
194822024-02-28T22:20:35.808Z 11 220-239 172 172 172 320 320 320 F F F
194832024-02-28T22:20:35.808Z 12 240-259 177 177 177 327 327 327 F F F
194842024-02-28T22:20:35.808Z 13 260-279 169 169 169 313 313 313 F F F
194852024-02-28T22:20:35.808Z 14 280-299 172 172 172 320 320 320 F F F
194862024-02-28T22:20:35.808Z 15 300-319 181 181 181 334 334 334 F F F
194872024-02-28T22:20:35.808Z 16 320-339 180 180 180 332 332 332 F F F
194882024-02-28T22:20:35.808Z 17 340-359 180 180 180 331 331 331 F F F
194892024-02-28T22:20:35.808Z 18 360-379 173 173 173 321 321 321 F F F
194902024-02-28T22:20:35.809Z 19 380-399 172 172 172 320 320 320 F F F
194912024-02-28T22:20:35.809Z 20 400-419 180 180 180 333 333 333 F F F
194922024-02-28T22:20:35.809Z 21 420-439 180 180 180 333 333 333 F F F
194932024-02-28T22:20:35.809Z 22 440-459 178 178 178 328 328 328 F F F
194942024-02-28T22:20:35.809Z 23 460-479 180 180 180 332 332 332 F F F
194952024-02-28T22:20:35.809Z 24 480-499 176 176 176 325 325 325 F F F
194962024-02-28T22:20:35.809Z 25 500-519 180 180 180 331 331 331 F F F
194972024-02-28T22:20:35.809Z 26 520-539 178 178 178 329 329 329 F F F
194982024-02-28T22:20:35.809Z 27 540-559 175 175 175 324 324 324 F F F
194992024-02-28T22:20:35.809Z 28 560-579 181 181 181 334 334 334 F F F
195002024-02-28T22:20:35.809Z 29 580-599 180 180 180 333 333 333 F F F
195012024-02-28T22:20:35.809ZMax gen: 181, Max flush: 334
195022024-02-28T22:20:35.809Z
195032024-02-28T22:20:35.809ZBegin loop 91
195042024-02-28T22:20:35.809ZDownstairs to restart: 0
195052024-02-28T22:20:35.809Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301028415Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
195062024-02-28T22:20:35.809ZCrucible runtime is spawned
195072024-02-28T22:20:35.809Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301386225Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
195082024-02-28T22:20:35.809Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301425944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
195092024-02-28T22:20:35.810Z{"msg":"Crucible 7a0fcdfd-60bc-49e0-bdd0-9d6042bf91b6 has session id: 076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301442844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195102024-02-28T22:20:35.810Z{"msg":"Upstairs opts: Upstairs UUID: 7a0fcdfd-60bc-49e0-bdd0-9d6042bf91b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301484343Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195112024-02-28T22:20:35.810Z{"msg":"Crucible stats registered with UUID: 7a0fcdfd-60bc-49e0-bdd0-9d6042bf91b6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301517542Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195122024-02-28T22:20:35.810Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301541791Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
195132024-02-28T22:20:35.810Z{"msg":"7a0fcdfd-60bc-49e0-bdd0-9d6042bf91b6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301572771Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195142024-02-28T22:20:35.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30159346Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195152024-02-28T22:20:35.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301628219Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195162024-02-28T22:20:35.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301652148Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195172024-02-28T22:20:35.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301686137Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195182024-02-28T22:20:35.810Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301709327Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195192024-02-28T22:20:35.810Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301741786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195202024-02-28T22:20:35.810Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301769135Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195212024-02-28T22:20:35.810Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301832163Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195222024-02-28T22:20:35.810Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.301857923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195232024-02-28T22:20:35.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30194533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195242024-02-28T22:20:35.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302021708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195252024-02-28T22:20:35.810Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302067007Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195262024-02-28T22:20:35.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302768347Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195272024-02-28T22:20:35.811Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302819926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195282024-02-28T22:20:35.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302846765Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195292024-02-28T22:20:35.811Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302873695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195302024-02-28T22:20:35.811Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302909714Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195312024-02-28T22:20:35.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.302938073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195322024-02-28T22:20:35.811Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.303513467Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195332024-02-28T22:20:35.811Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.303551926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195342024-02-28T22:20:35.811Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30377751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195352024-02-28T22:20:35.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304065122Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195362024-02-28T22:20:35.811Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304107461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195372024-02-28T22:20:35.811Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30412466Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195382024-02-28T22:20:35.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304156039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195392024-02-28T22:20:35.811Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304181899Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195402024-02-28T22:20:35.811Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304216667Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195412024-02-28T22:20:35.811Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304340234Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195422024-02-28T22:20:35.811Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304406552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195432024-02-28T22:20:35.811Z{"msg":"[0]R flush_numbers[0..12]: [332, 329, 333, 333, 327, 333, 330, 333, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304448511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195442024-02-28T22:20:35.812Z{"msg":"[0]R generation[0..12]: [180, 178, 180, 180, 177, 180, 179, 180, 178, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30448381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195452024-02-28T22:20:35.812Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30450271Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195462024-02-28T22:20:35.812Z{"msg":"[1]R flush_numbers[0..12]: [332, 329, 333, 333, 327, 333, 330, 333, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304525839Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195472024-02-28T22:20:35.812Z{"msg":"[1]R generation[0..12]: [180, 178, 180, 180, 177, 180, 179, 180, 178, 178, 178, 172]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
195482024-02-28T22:20:35.812Zcrucible","level":30,"time":"2024-02-28T22:20:34.304544009Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195492024-02-28T22:20:35.812Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304566138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195502024-02-28T22:20:35.812Z{"msg":"[2]R flush_numbers[0..12]: [332, 329, 333, 333, 327, 333, 330, 333, 329, 329, 329, 320]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304581097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195512024-02-28T22:20:35.812Z{"msg":"[2]R generation[0..12]: [180, 178, 180, 180, 177, 180, 179, 180, 178, 178, 178, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304607917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195522024-02-28T22:20:35.812Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304622696Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195532024-02-28T22:20:35.812Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304637916Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195542024-02-28T22:20:35.812Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304663245Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195552024-02-28T22:20:35.812Z{"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304677325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195562024-02-28T22:20:35.812Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304694825Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"":"downstairs","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195572024-02-28T22:20:35.812Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304727953Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195582024-02-28T22:20:35.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304758283Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"0","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195592024-02-28T22:20:35.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304793142Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"1","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195602024-02-28T22:20:35.812Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304827551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"client":"2","":"downstairs","session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195612024-02-28T22:20:35.813ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195622024-02-28T22:20:35.813Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.30486207Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195632024-02-28T22:20:35.813Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304893729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195642024-02-28T22:20:35.813Z{"msg":"7a0fcdfd-60bc-49e0-bdd0-9d6042bf91b6 is now active with session: 076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304908708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195652024-02-28T22:20:35.813Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304928908Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863,"session_id":"076fe9d0-a4c7-4d2c-9e73-92bf39bf7bd5"}
195662024-02-28T22:20:35.813Z{"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:34.304941748Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
195672024-02-28T22:20:35.813ZLoading write count information from file "/tmp/test_repair_verify.data"
195682024-02-28T22:20:35.813ZRun Repair workload
195692024-02-28T22:20:35.813Z01/30 Write block 44 len 28672 data: 10 12 12 12 13 15 13
195702024-02-28T22:20:35.819Z02/30 Write block 307 len 4096 data: 16
195712024-02-28T22:20:35.819Z03/30 Write block 325 len 12288 data: 17 18 18
195722024-02-28T22:20:35.819Z04/30 Write block 191 len 16384 data: 12 15 13 12
195732024-02-28T22:20:35.820Z05/30 Read block 304 len 28672
195742024-02-28T22:20:35.820Z06/30 Write block 384 len 24576 data: 19 19 22 19 15 15
195752024-02-28T22:20:35.820Z07/30 Write block 533 len 16384 data: 13 16 13 14
195762024-02-28T22:20:35.820Z08/30 Read block 485 len 32768
195772024-02-28T22:20:35.820Z09/30 Read block 552 len 40960
195782024-02-28T22:20:35.820Z10/30 Write block 414 len 12288 data: 15 16 15
195792024-02-28T22:20:35.820Z11/30 Write block 282 len 36864 data: 13 14 15 13 14 11 10 10 8
195802024-02-28T22:20:35.820Z12/30 Read block 478 len 16384
195812024-02-28T22:20:35.820Z13/30 Write block 378 len 20480 data: 15 13 14 14 18
195822024-02-28T22:20:35.820Z14/30 Read block 519 len 24576
195832024-02-28T22:20:35.820Z15/30 Flush
195842024-02-28T22:20:35.820Z16/30 Write block 374 len 24576 data: 17 15 15 14 16 14
195852024-02-28T22:20:35.820Z17/30 Read block 593 len 12288
195862024-02-28T22:20:35.820Z18/30 Write block 413 len 24576 data: 13 16 17 16 11 13
195872024-02-28T22:20:35.820Z19/30 Write block 203 len 32768 data: 11 12 13 10 8 9 8 10
195882024-02-28T22:20:35.821Z20/30 Flush
195892024-02-28T22:20:35.821Z21/30 Write block 258 len 28672 data: 16 16 15 17 20 20 20
195902024-02-28T22:20:35.821Z22/30 Flush
195912024-02-28T22:20:35.821Z23/30 Write block 508 len 32768 data: 15 18 16 14 17 17 18 19
195922024-02-28T22:20:35.821Z24/30 Write block 236 len 24576 data: 12 13 12 8 8 8
195932024-02-28T22:20:35.821Z25/30 Read block 171 len 4096
195942024-02-28T22:20:35.821Z26/30 Write block 265 len 12288 data: 20 20 16
195952024-02-28T22:20:35.821Z27/30 Read block 410 len 16384
195962024-02-28T22:20:35.821Z28/30 Write block 46 len 16384 data: 13 13 14 16
195972024-02-28T22:20:35.821Z29/30 Write block 44 len 4096 data: 11
195982024-02-28T22:20:35.821Z30/30 Read block 404 len 20480
195992024-02-28T22:20:35.821Z----------------------------------------------------------------
196002024-02-28T22:20:35.821Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196012024-02-28T22:20:35.821ZStates: Active Active Active
196022024-02-28T22:20:35.821ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196032024-02-28T22:20:35.821Z 1 Acked 1000 Write 7 Done Done Done false
196042024-02-28T22:20:35.822Z 2 Acked 1001 Write 1 Done Done Done false
196052024-02-28T22:20:35.822Z 3 Acked 1002 Write 3 Done Done Done false
196062024-02-28T22:20:35.822Z 4 Acked 1003 Write 4 Done Done Done false
196072024-02-28T22:20:35.822Z 5 Acked 1004 Read 7 Done Done Done false
196082024-02-28T22:20:35.822Z 6 Acked 1005 Write 6 Done Done Done false
196092024-02-28T22:20:35.822Z 7 Acked 1006 Write 4 Sent Done Done false
196102024-02-28T22:20:35.822Z 8 Acked 1007 Read 8 Sent Done Done false
196112024-02-28T22:20:35.822Z 9 Acked 1008 Read 10 Sent Done Done false
196122024-02-28T22:20:35.822Z 10 Acked 1009 Write 3 Sent Done Done false
196132024-02-28T22:20:35.822Z 11 Acked 1010 Write 9 Sent Done Done false
196142024-02-28T22:20:35.822Z 12 Acked 1011 Read 4 Sent Done Done false
196152024-02-28T22:20:35.822Z 13 Acked 1012 Write 5 Sent Done Done false
196162024-02-28T22:20:35.822Z 14 Acked 1013 Read 6 Sent Done Done false
196172024-02-28T22:20:35.822Z 15 Acked 1014 Flush 0 Sent Done Done false
196182024-02-28T22:20:35.822Z 16 Acked 1015 Write 6 Sent Done Done false
196192024-02-28T22:20:35.822Z 17 Acked 1016 Read 3 Sent Done Done false
196202024-02-28T22:20:35.823Z 18 Acked 1017 Write 6 Sent Done Done false
196212024-02-28T22:20:35.823Z 19 Acked 1018 Write 8 Sent Done Done false
196222024-02-28T22:20:35.823Z 20 Acked 1019 Flush 0 Sent Done Done false
196232024-02-28T22:20:35.823Z 21 Acked 1020 Write 7 Sent Done Done false
196242024-02-28T22:20:35.823Z 22 Acked 1021 Flush 0 Sent Done Done false
196252024-02-28T22:20:35.823Z 23 Acked 1022 Write 8 Sent Done Done false
196262024-02-28T22:20:35.823Z 24 Acked 1023 Write 6 Sent Done Done false
196272024-02-28T22:20:35.823Z 25 Acked 1024 Read 1 Sent Done Done false
196282024-02-28T22:20:35.823Z 26 Acked 1025 Write 3 Sent Done Done false
196292024-02-28T22:20:35.823Z 27 Acked 1026 Read 4 Sent Done Done false
196302024-02-28T22:20:35.823Z 28 Acked 1027 Write 4 Sent Done Done false
196312024-02-28T22:20:35.823Z 29 Acked 1028 Write 1 Sent Done Done false
196322024-02-28T22:20:35.823Z 30 Acked 1029 Read 5 Sent Done Done false
196332024-02-28T22:20:35.823Z STATES DS:0 DS:1 DS:2 TOTAL
196342024-02-28T22:20:35.823Z New 0 0 0 0
196352024-02-28T22:20:35.824Z Sent 24 0 0 24
196362024-02-28T22:20:35.824Z Done 6 30 30 66
196372024-02-28T22:20:35.824Z Skipped 0 0 0 0
196382024-02-28T22:20:35.824Z Error 0 0 0 0
196392024-02-28T22:20:35.824ZLast Flush: 0 1021 1021
196402024-02-28T22:20:35.824ZDownstairs last five completed:
196412024-02-28T22:20:35.824ZUpstairs last five completed: 30 29 28 27 26
196422024-02-28T22:20:35.824Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:34.338529802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1863}
196432024-02-28T22:20:35.824ZWrote out file "/tmp/test_repair_verify.data"
196442024-02-28T22:20:35.824Z
196452024-02-28T22:20:35.824ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196462024-02-28T22:20:35.824ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196472024-02-28T22:20:35.824Z 0 000-019 180 180 180 332 332 332 F F F
196482024-02-28T22:20:35.824Z 1 020-039 178 178 178 329 329 329 F F F
196492024-02-28T22:20:35.824Z 2 040-059 180 182 182 333 335 335 T T T <---
196502024-02-28T22:20:35.824Z 3 060-079 180 180 180 333 333 333 F F F
196512024-02-28T22:20:35.824Z 4 080-099 177 177 177 327 327 327 F F F
196522024-02-28T22:20:35.824Z 5 100-119 180 180 180 333 333 333 F F F
196532024-02-28T22:20:35.824Z 6 120-139 179 179 179 330 330 330 F F F
196542024-02-28T22:20:35.824Z 7 140-159 180 180 180 333 333 333 F F F
196552024-02-28T22:20:35.824Z 8 160-179 178 178 178 329 329 329 F F F
196562024-02-28T22:20:35.825Z 9 180-199 178 182 182 329 335 335 T F F <---
196572024-02-28T22:20:35.825Z 10 200-219 178 182 182 329 336 336 F F F <---
196582024-02-28T22:20:35.825Z 11 220-239 172 172 172 320 320 320 F T T <---
196592024-02-28T22:20:35.825Z 12 240-259 177 182 182 327 337 337 F T T <---
196602024-02-28T22:20:35.825Z 13 260-279 169 182 182 313 337 337 F T T <---
196612024-02-28T22:20:35.825Z 14 280-299 172 182 182 320 335 335 F F F <---
196622024-02-28T22:20:35.825Z 15 300-319 181 182 182 334 335 335 T F F <---
196632024-02-28T22:20:35.825Z 16 320-339 180 182 182 332 335 335 T F F <---
196642024-02-28T22:20:35.825Z 17 340-359 180 180 180 331 331 331 F F F
196652024-02-28T22:20:35.825Z 18 360-379 173 182 182 321 336 336 F F F <---
196662024-02-28T22:20:35.825Z 19 380-399 172 182 182 320 335 335 T F F <---
196672024-02-28T22:20:35.825Z 20 400-419 180 182 182 333 336 336 F F F <---
196682024-02-28T22:20:35.825Z 21 420-439 180 180 180 333 333 333 F F F
196692024-02-28T22:20:35.825Z 22 440-459 178 178 178 328 328 328 F F F
196702024-02-28T22:20:35.830Z 23 460-479 180 180 180 332 332 332 F F F
196712024-02-28T22:20:35.830Z 24 480-499 176 176 176 325 325 325 F F F
196722024-02-28T22:20:35.830Z 25 500-519 180 180 180 331 331 331 F T T <---
196732024-02-28T22:20:35.830Z 26 520-539 178 182 182 329 335 335 F F F <---
196742024-02-28T22:20:35.830Z 27 540-559 175 175 175 324 324 324 F F F
196752024-02-28T22:20:35.830Z 28 560-579 181 181 181 334 334 334 F F F
196762024-02-28T22:20:35.830Z 29 580-599 180 180 180 333 333 333 F F F
196772024-02-28T22:20:35.830ZMax gen: 182, Max flush: 337
196782024-02-28T22:20:35.830ZError: Difference in extent metadata found!
196792024-02-28T22:20:35.831ZA Difference in extent metadata is expected here
196802024-02-28T22:20:35.831ZOn loop 91
196812024-02-28T22:20:35.831Z
196822024-02-28T22:20:35.831ZVerifying data now
196832024-02-28T22:20:41.210ZNo change after verify
196842024-02-28T22:20:41.211ZLoop: 91 Downstairs dump after verify (and repair):
196852024-02-28T22:20:41.216ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196862024-02-28T22:20:41.216Z 0 000-019 180 180 180 332 332 332 F F F
196872024-02-28T22:20:41.216Z 1 020-039 178 178 178 329 329 329 F F F
196882024-02-28T22:20:41.217Z 2 040-059 183 183 183 338 338 338 F F F
196892024-02-28T22:20:41.217Z 3 060-079 180 180 180 333 333 333 F F F
196902024-02-28T22:20:41.217Z 4 080-099 177 177 177 327 327 327 F F F
196912024-02-28T22:20:41.217Z 5 100-119 180 180 180 333 333 333 F F F
196922024-02-28T22:20:41.217Z 6 120-139 179 179 179 330 330 330 F F F
196932024-02-28T22:20:41.217Z 7 140-159 180 180 180 333 333 333 F F F
196942024-02-28T22:20:41.217Z 8 160-179 178 178 178 329 329 329 F F F
196952024-02-28T22:20:41.218Z 9 180-199 182 182 182 335 335 335 F F F
196962024-02-28T22:20:41.218Z 10 200-219 182 182 182 336 336 336 F F F
196972024-02-28T22:20:41.218Z 11 220-239 183 183 183 338 338 338 F F F
196982024-02-28T22:20:41.218Z 12 240-259 183 183 183 338 338 338 F F F
196992024-02-28T22:20:41.218Z 13 260-279 183 183 183 338 338 338 F F F
197002024-02-28T22:20:41.218Z 14 280-299 182 182 182 335 335 335 F F F
197012024-02-28T22:20:41.218Z 15 300-319 182 182 182 335 335 335 F F F
197022024-02-28T22:20:41.218Z 16 320-339 182 182 182 335 335 335 F F F
197032024-02-28T22:20:41.218Z 17 340-359 180 180 180 331 331 331 F F F
197042024-02-28T22:20:41.218Z 18 360-379 182 182 182 336 336 336 F F F
197052024-02-28T22:20:41.218Z 19 380-399 182 182 182 335 335 335 F F F
197062024-02-28T22:20:41.218Z 20 400-419 182 182 182 336 336 336 F F F
197072024-02-28T22:20:41.219Z 21 420-439 180 180 180 333 333 333 F F F
197082024-02-28T22:20:41.219Z 22 440-459 178 178 178 328 328 328 F F F
197092024-02-28T22:20:41.219Z 23 460-479 180 180 180 332 332 332 F F F
197102024-02-28T22:20:41.219Z 24 480-499 176 176 176 325 325 325 F F F
197112024-02-28T22:20:41.219Z 25 500-519 183 183 183 338 338 338 F F F
197122024-02-28T22:20:41.219Z 26 520-539 182 182 182 335 335 335 F F F
197132024-02-28T22:20:41.219Z 27 540-559 175 175 175 324 324 324 F F F
197142024-02-28T22:20:41.219Z 28 560-579 181 181 181 334 334 334 F F F
197152024-02-28T22:20:41.219Z 29 580-599 180 180 180 333 333 333 F F F
197162024-02-28T22:20:41.219ZMax gen: 183, Max flush: 338
197172024-02-28T22:20:41.219Z
197182024-02-28T22:20:41.219ZBegin loop 92
197192024-02-28T22:20:41.220ZDownstairs to restart: 0
197202024-02-28T22:20:41.230Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105297043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871}
197212024-02-28T22:20:41.230Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105646853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal"Crucible runtime is spawned
197222024-02-28T22:20:41.230Z,"pid":1871}
197232024-02-28T22:20:41.231Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105691472Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871}
197242024-02-28T22:20:41.253Z{"msg":"Crucible 658a75e2-1e82-4b6b-9ed7-3ac7268bf235 has session id: ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10574353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197252024-02-28T22:20:41.253Z{"msg":"Upstairs opts: Upstairs UUID: 658a75e2-1e82-4b6b-9ed7-3ac7268bf235, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105796539Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197262024-02-28T22:20:41.253Z{"msg":"Crucible stats registered with UUID: 658a75e2-1e82-4b6b-9ed7-3ac7268bf235","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105843637Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197272024-02-28T22:20:41.253Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.105874617Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871}
197282024-02-28T22:20:41.254Z{"msg":"658a75e2-1e82-4b6b-9ed7-3ac7268bf235 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106017043Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197292024-02-28T22:20:41.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106060902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197302024-02-28T22:20:41.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10612512Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197312024-02-28T22:20:41.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106175738Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197322024-02-28T22:20:41.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106195698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197332024-02-28T22:20:41.254Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106210257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197342024-02-28T22:20:41.254Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106224717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197352024-02-28T22:20:41.254Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106251776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197362024-02-28T22:20:41.254Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106295435Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197372024-02-28T22:20:41.255Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106329894Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197382024-02-28T22:20:41.255Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106401702Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197392024-02-28T22:20:41.255Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.106456291Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197402024-02-28T22:20:41.255Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10648877Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197412024-02-28T22:20:41.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107225159Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197422024-02-28T22:20:41.255Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107261889Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197432024-02-28T22:20:41.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107279218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197442024-02-28T22:20:41.255Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107307197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197452024-02-28T22:20:41.255Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107321897Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197462024-02-28T22:20:41.256Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107335746Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197472024-02-28T22:20:41.256Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.107903121Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197482024-02-28T22:20:41.256Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.1079328Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197492024-02-28T22:20:41.256Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10794979Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197502024-02-28T22:20:41.256Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108505854Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197512024-02-28T22:20:41.256Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108560633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197522024-02-28T22:20:41.256Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108583272Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197532024-02-28T22:20:41.256Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108623901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197542024-02-28T22:20:41.256Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10865018Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197552024-02-28T22:20:41.256Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10866861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197562024-02-28T22:20:41.256Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108689349Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197572024-02-28T22:20:41.256Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108704179Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197582024-02-28T22:20:41.256Z{"msg":"[0]R flush_numbers[0..12]: [332, 329, 338, 333, 327, 333, 330, 333, 329, 335, 336, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108725078Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197592024-02-28T22:20:41.257Z{"msg":"[0]R generation[0..12]: [180, 178, 183, 180, 177, 180, 179, 180, 178, 182, 182, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108739728Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197602024-02-28T22:20:41.257Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108760607Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197612024-02-28T22:20:41.257Z{"msg":"[1]R flush_numbers[0..12]: [332, 329, 338, 333, 327, 333, 330, 333, 329, 335, 336, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108774917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197622024-02-28T22:20:41.257Z{"msg":"[1]R generation[0..12]: [180, 178, 183, 180, 177, 180, 179, 180, 178, 182, 182, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108809766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197632024-02-28T22:20:41.257Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108826825Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197642024-02-28T22:20:41.257Z{"msg":"[2]R flush_numbers[0..12]: [332, 329, 338, 333, 327, 333, 330, 333, 329, 335, 336, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108848815Z","hostname"Wait for a query_work_queue command to finish before sending IO
197652024-02-28T22:20:41.257Z:"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197662024-02-28T22:20:41.257Z{"msg":"[2]R generation[0..12]: [180, 178, 183, 180, 177, 180, 179, 180, 178, 182, 182, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108868844Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197672024-02-28T22:20:41.257Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108891153Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197682024-02-28T22:20:41.257Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108905103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197692024-02-28T22:20:41.257Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108925312Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197702024-02-28T22:20:41.257Z{"msg":"Next flush: 339","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108938582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197712024-02-28T22:20:41.257Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108958712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"":"downstairs","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197722024-02-28T22:20:41.258Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108972721Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197732024-02-28T22:20:41.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.108985551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"0","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197742024-02-28T22:20:41.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10900616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"1","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197752024-02-28T22:20:41.258Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10901989Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"client":"2","":"downstairs","session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197762024-02-28T22:20:41.258Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.10903388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197772024-02-28T22:20:41.258Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.109046199Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197782024-02-28T22:20:41.258Z{"msg":"658a75e2-1e82-4b6b-9ed7-3ac7268bf235 is now active with session: ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.109065949Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197792024-02-28T22:20:41.258Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.109078258Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871,"session_id":"ba3c4ff6-1ae6-4b6d-8034-75b8e4f25c85"}
197802024-02-28T22:20:41.258Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:40.109097128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871}
197812024-02-28T22:20:41.258ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197822024-02-28T22:20:41.259ZLoading write count information from file "/tmp/test_repair_verify.data"
197832024-02-28T22:20:41.259ZRun Repair workload
197842024-02-28T22:20:41.259Z01/30 Write block 246 len 28672 data: 12 20 23 18 17 17 14
197852024-02-28T22:20:41.259Z02/30 Read block 387 len 16384
197862024-02-28T22:20:41.259Z03/30 Read block 432 len 4096
197872024-02-28T22:20:41.259Z04/30 Read block 276 len 32768
197882024-02-28T22:20:41.259Z05/30 Write block 529 len 16384 data: 10 10 10 11
197892024-02-28T22:20:41.259Z06/30 Flush
197902024-02-28T22:20:41.259Z07/30 Write block 90 len 16384 data: 13 15 15 13
197912024-02-28T22:20:41.260Z08/30 Write block 247 len 32768 data: 21 24 19 18 18 15 16 14
197922024-02-28T22:20:41.260Z09/30 Read block 513 len 40960
197932024-02-28T22:20:41.260Z10/30 Write block 378 len 40960 data: 17 15 15 15 19 20 20 20 23 20
197942024-02-28T22:20:41.260Z11/30 Read block 515 len 20480
197952024-02-28T22:20:41.260Z12/30 Read block 387 len 16384
197962024-02-28T22:20:41.260Z13/30 Read block 73 len 16384
197972024-02-28T22:20:41.260Z14/30 Read block 334 len 8192
197982024-02-28T22:20:41.260Z15/30 Read block 221 len 36864
197992024-02-28T22:20:41.260Z16/30 Read block 428 len 16384
198002024-02-28T22:20:41.266Z17/30 Read block 397 len 40960
198012024-02-28T22:20:41.266Z18/30 Flush
198022024-02-28T22:20:41.266Z19/30 Write block 382 len 32768 data: 20 21 21 21 24 21 16 16
198032024-02-28T22:20:41.266Z20/30 Read block 292 len 28672
198042024-02-28T22:20:41.267Z21/30 Read block 38 len 4096
198052024-02-28T22:20:41.267Z22/30 Read block 236 len 32768
198062024-02-28T22:20:41.267Z23/30 Read block 342 len 8192
198072024-02-28T22:20:41.267Z24/30 Write block 202 len 4096 data: 12
198082024-02-28T22:20:41.267Z25/30 Read block 556 len 16384
198092024-02-28T22:20:41.267Z26/30 Read block 41 len 20480
198102024-02-28T22:20:41.267Z27/30 Read block 588 len 16384
198112024-02-28T22:20:41.267Z28/30 Read block 567 len 16384
198122024-02-28T22:20:41.267Z29/30 Read block 467 len 28672
198132024-02-28T22:20:41.267Z30/30 Write block 145 len 12288 data: 19 20 19
198142024-02-28T22:20:41.267Z----------------------------------------------------------------
198152024-02-28T22:20:41.267Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198162024-02-28T22:20:41.267ZStates: Active Active Active
198172024-02-28T22:20:41.267ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198182024-02-28T22:20:41.267Z 1 Acked 1000 Write 7 Sent Done Done false
198192024-02-28T22:20:41.267Z 2 Acked 1001 Read 4 Sent Done Done false
198202024-02-28T22:20:41.267Z 3 Acked 1002 Read 1 Sent Done Done false
198212024-02-28T22:20:41.267Z 4 Acked 1003 Read 8 Sent Done Done false
198222024-02-28T22:20:41.267Z 5 Acked 1004 Write 4 Sent Done Done false
198232024-02-28T22:20:41.267Z 6 Acked 1005 Flush 0 Sent Done Done false
198242024-02-28T22:20:41.267Z 7 Acked 1006 Write 4 Sent Done Done false
198252024-02-28T22:20:41.268Z 8 Acked 1007 Write 8 Sent Done Done false
198262024-02-28T22:20:41.268Z 9 Acked 1008 Read 10 Sent Done Done false
198272024-02-28T22:20:41.268Z 10 Acked 1009 Write 10 Sent Done Done false
198282024-02-28T22:20:41.268Z 11 Acked 1010 Read 5 Sent Done Done false
198292024-02-28T22:20:41.268Z 12 Acked 1011 Read 4 Sent Done Done false
198302024-02-28T22:20:41.268Z 13 Acked 1012 Read 4 Sent Done Done false
198312024-02-28T22:20:41.268Z 14 Acked 1013 Read 2 Sent Done Done false
198322024-02-28T22:20:41.268Z 15 Acked 1014 Read 9 Sent Done Done false
198332024-02-28T22:20:41.268Z 16 Acked 1015 Read 4 Sent Done Done false
198342024-02-28T22:20:41.268Z 17 Acked 1016 Read 10 Sent Done Done false
198352024-02-28T22:20:41.268Z 18 Acked 1017 Flush 0 Sent Done Done false
198362024-02-28T22:20:41.268Z 19 Acked 1018 Write 8 Sent Done Done false
198372024-02-28T22:20:41.268Z 20 Acked 1019 Read 7 Sent Done Done false
198382024-02-28T22:20:41.268Z 21 Acked 1020 Read 1 Sent Done Done false
198392024-02-28T22:20:41.268Z 22 Acked 1021 Read 8 Sent Done Done false
198402024-02-28T22:20:41.268Z 23 Acked 1022 Read 2 Sent Done Done false
198412024-02-28T22:20:41.268Z 24 Acked 1023 Write 1 Sent Done Done false
198422024-02-28T22:20:41.268Z 25 Acked 1024 Read 4 Sent Done Done false
198432024-02-28T22:20:41.268Z 26 Acked 1025 Read 5 Sent Done Done false
198442024-02-28T22:20:41.268Z 27 Acked 1026 Read 4 Sent Done Done false
198452024-02-28T22:20:41.268Z 28 Acked 1027 Read 4 Sent Done Done false
198462024-02-28T22:20:41.268Z 29 Acked 1028 Read 7 Sent Done Done false
198472024-02-28T22:20:41.268Z 30 Acked 1029 Write 3 Sent Sent Sent false
198482024-02-28T22:20:41.269Z STATES DS:0 DS:1 DS:2 TOTAL
198492024-02-28T22:20:41.269Z New 0 0 0 0
198502024-02-28T22:20:41.269Z Sent 30 1 1 32
198512024-02-28T22:20:41.269Z Done 0 29 29 58
198522024-02-28T22:20:41.269Z Skipped 0 0 0 0
198532024-02-28T22:20:41.269Z Error 0 0 0 0
198542024-02-28T22:20:41.269ZLast Flush: 0 1017 1017
198552024-02-28T22:20:41.269ZDownstairs last five completed:
198562024-02-28T22:20:41.269ZUpstairs last five completed: 30 29 28 27 26
198572024-02-28T22:20:41.269Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:40.140067195Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1871}
198582024-02-28T22:20:41.269ZWrote out file "/tmp/test_repair_verify.data"
198592024-02-28T22:20:41.276Z
198602024-02-28T22:20:41.276ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198612024-02-28T22:20:41.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198622024-02-28T22:20:41.285Z 0 000-019 180 180 180 332 332 332 F F F
198632024-02-28T22:20:41.286Z 1 020-039 178 178 178 329 329 329 F F F
198642024-02-28T22:20:41.286Z 2 040-059 183 183 183 338 338 338 F F F
198652024-02-28T22:20:41.286Z 3 060-079 180 180 180 333 333 333 F F F
198662024-02-28T22:20:41.286Z 4 080-099 177 184 184 327 340 340 F F F <---
198672024-02-28T22:20:41.286Z 5 100-119 180 180 180 333 333 333 F F F
198682024-02-28T22:20:41.286Z 6 120-139 179 179 179 330 330 330 F F F
198692024-02-28T22:20:41.286Z 7 140-159 180 180 180 333 333 333 F T T <---
198702024-02-28T22:20:41.286Z 8 160-179 178 178 178 329 329 329 F F F
198712024-02-28T22:20:41.286Z 9 180-199 182 182 182 335 335 335 F F F
198722024-02-28T22:20:41.286Z 10 200-219 182 182 182 336 336 336 F T T <---
198732024-02-28T22:20:41.286Z 11 220-239 183 183 183 338 338 338 F F F
198742024-02-28T22:20:41.286Z 12 240-259 183 184 184 338 340 340 F F F <---
198752024-02-28T22:20:41.294Z 13 260-279 183 183 183 338 338 338 F F F
198762024-02-28T22:20:41.294Z 14 280-299 182 182 182 335 335 335 F F F
198772024-02-28T22:20:41.294Z 15 300-319 182 182 182 335 335 335 F F F
198782024-02-28T22:20:41.294Z 16 320-339 182 182 182 335 335 335 F F F
198792024-02-28T22:20:41.294Z 17 340-359 180 180 180 331 331 331 F F F
198802024-02-28T22:20:41.294Z 18 360-379 182 184 184 336 340 340 F F F <---
198812024-02-28T22:20:41.294Z 19 380-399 182 184 184 335 340 340 F T T <---
198822024-02-28T22:20:41.294Z 20 400-419 182 182 182 336 336 336 F F F
198832024-02-28T22:20:41.294Z 21 420-439 180 180 180 333 333 333 F F F
198842024-02-28T22:20:41.295Z 22 440-459 178 178 178 328 328 328 F F F
198852024-02-28T22:20:41.295Z 23 460-479 180 180 180 332 332 332 F F F
198862024-02-28T22:20:41.295Z 24 480-499 176 176 176 325 325 325 F F F
198872024-02-28T22:20:41.295Z 25 500-519 183 183 183 338 338 338 F F F
198882024-02-28T22:20:41.295Z 26 520-539 182 184 184 335 339 339 F F F <---
198892024-02-28T22:20:41.295Z 27 540-559 175 175 175 324 324 324 F F F
198902024-02-28T22:20:41.295Z 28 560-579 181 181 181 334 334 334 F F F
198912024-02-28T22:20:41.295Z 29 580-599 180 180 180 333 333 333 F F F
198922024-02-28T22:20:41.295ZMax gen: 184, Max flush: 340
198932024-02-28T22:20:41.295ZError: Difference in extent metadata found!
198942024-02-28T22:20:41.295ZA Difference in extent metadata is expected here
198952024-02-28T22:20:41.295ZOn loop 92
198962024-02-28T22:20:41.295Z
198972024-02-28T22:20:41.296ZVerifying data now
198982024-02-28T22:20:45.699ZNo change after verify
198992024-02-28T22:20:45.699ZLoop: 92 Downstairs dump after verify (and repair):
199002024-02-28T22:20:45.714ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199012024-02-28T22:20:45.714Z 0 000-019 180 180 180 332 332 332 F F F
199022024-02-28T22:20:45.714Z 1 020-039 178 178 178 329 329 329 F F F
199032024-02-28T22:20:45.714Z 2 040-059 183 183 183 338 338 338 F F F
199042024-02-28T22:20:45.714Z 3 060-079 180 180 180 333 333 333 F F F
199052024-02-28T22:20:45.714Z 4 080-099 184 184 184 340 340 340 F F F
199062024-02-28T22:20:45.714Z 5 100-119 180 180 180 333 333 333 F F F
199072024-02-28T22:20:45.714Z 6 120-139 179 179 179 330 330 330 F F F
199082024-02-28T22:20:45.714Z 7 140-159 185 185 185 341 341 341 F F F
199092024-02-28T22:20:45.714Z 8 160-179 178 178 178 329 329 329 F F F
199102024-02-28T22:20:45.714Z 9 180-199 182 182 182 335 335 335 F F F
199112024-02-28T22:20:45.714Z 10 200-219 185 185 185 341 341 341 F F F
199122024-02-28T22:20:45.714Z 11 220-239 183 183 183 338 338 338 F F F
199132024-02-28T22:20:45.714Z 12 240-259 184 184 184 340 340 340 F F F
199142024-02-28T22:20:45.714Z 13 260-279 183 183 183 338 338 338 F F F
199152024-02-28T22:20:45.714Z 14 280-299 182 182 182 335 335 335 F F F
199162024-02-28T22:20:45.718Z 15 300-319 182 182 182 335 335 335 F F F
199172024-02-28T22:20:45.718Z 16 320-339 182 182 182 335 335 335 F F F
199182024-02-28T22:20:45.718Z 17 340-359 180 180 180 331 331 331 F F F
199192024-02-28T22:20:45.718Z 18 360-379 184 184 184 340 340 340 F F F
199202024-02-28T22:20:45.718Z 19 380-399 185 185 185 341 341 341 F F F
199212024-02-28T22:20:45.718Z 20 400-419 182 182 182 336 336 336 F F F
199222024-02-28T22:20:45.718Z 21 420-439 180 180 180 333 333 333 F F F
199232024-02-28T22:20:45.719Z 22 440-459 178 178 178 328 328 328 F F F
199242024-02-28T22:20:45.719Z 23 460-479 180 180 180 332 332 332 F F F
199252024-02-28T22:20:45.719Z 24 480-499 176 176 176 325 325 325 F F F
199262024-02-28T22:20:45.719Z 25 500-519 183 183 183 338 338 338 F F F
199272024-02-28T22:20:45.719Z 26 520-539 184 184 184 339 339 339 F F F
199282024-02-28T22:20:45.719Z 27 540-559 175 175 175 324 324 324 F F F
199292024-02-28T22:20:45.719Z 28 560-579 181 181 181 334 334 334 F F F
199302024-02-28T22:20:45.719Z 29 580-599 180 180 180 333 333 333 F F F
199312024-02-28T22:20:45.719ZMax gen: 185, Max flush: 341
199322024-02-28T22:20:45.719Z
199332024-02-28T22:20:45.719ZBegin loop 93
199342024-02-28T22:20:45.719ZDownstairs to restart: 1
199352024-02-28T22:20:45.736Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614163568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
199362024-02-28T22:20:45.736Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
199372024-02-28T22:20:45.736Z,"time":"2024-02-28T22:20:44.614482709Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
199382024-02-28T22:20:45.741Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614527478Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
199392024-02-28T22:20:45.741Z{"msg":"Crucible 5e89c8a6-e166-4c2a-a70b-7945dcffe20d has session id: 9401c809-b469-4570-aee5-bb166dfd594f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614568886Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199402024-02-28T22:20:45.742Z{"msg":"Upstairs opts: Upstairs UUID: 5e89c8a6-e166-4c2a-a70b-7945dcffe20d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614627285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199412024-02-28T22:20:45.742Z{"msg":"Crucible stats registered with UUID: 5e89c8a6-e166-4c2a-a70b-7945dcffe20d","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614672374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199422024-02-28T22:20:45.742Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614701073Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
199432024-02-28T22:20:45.742Z{"msg":"5e89c8a6-e166-4c2a-a70b-7945dcffe20d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614716642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199442024-02-28T22:20:45.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614737792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199452024-02-28T22:20:45.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614781231Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199462024-02-28T22:20:45.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61480254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199472024-02-28T22:20:45.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614849289Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199482024-02-28T22:20:45.742Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614933887Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199492024-02-28T22:20:45.742Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.614972875Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199502024-02-28T22:20:45.742Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615003425Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199512024-02-28T22:20:45.742Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615048983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199522024-02-28T22:20:45.742Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615076092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199532024-02-28T22:20:45.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615106402Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199542024-02-28T22:20:45.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615190559Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199552024-02-28T22:20:45.742Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615230898Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199562024-02-28T22:20:45.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615844461Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199572024-02-28T22:20:45.742Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61589699Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199582024-02-28T22:20:45.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615929169Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199592024-02-28T22:20:45.742Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615956468Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199602024-02-28T22:20:45.742Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.615981038Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199612024-02-28T22:20:45.742Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.616021066Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199622024-02-28T22:20:45.742Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.616458754Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199632024-02-28T22:20:45.743Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.616524903Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199642024-02-28T22:20:45.743Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61662767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199652024-02-28T22:20:45.743Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617147035Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199662024-02-28T22:20:45.743Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617195154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199672024-02-28T22:20:45.743Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617231573Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199682024-02-28T22:20:45.743Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617247213Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199692024-02-28T22:20:45.743Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617268912Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199702024-02-28T22:20:45.743Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617283552Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199712024-02-28T22:20:45.743Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617296441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199722024-02-28T22:20:45.743Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617310131Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199732024-02-28T22:20:45.743Z{"msg":"[0]R flush_numbers[0..12]: [332, 329, 338, 333, 340, 333, 330, 341, 329, 335, 341, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61733105Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199742024-02-28T22:20:45.743Z{"msg":"[0]R generation[0..12]: [180, 178, 183, 180, 184, 180, 179, 185, 178, 182, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61734647Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199752024-02-28T22:20:45.743Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617367529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199762024-02-28T22:20:45.743Z{"msg":"[1]R flush_numbers[0..12]: [332, 329, 338, 333, 340, 333, 330, 341, 329, 335, 341, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617382379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199772024-02-28T22:20:45.743Z{"msg":"[1]R generation[0..12]: [180, 178, 183, 180, 184, 180, 179, 185, 178, 182, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617403459Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199782024-02-28T22:20:45.743Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617418048Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199792024-02-28T22:20:45.743Z{"msg":"[2]R flush_numbers[0..12]: [332, 329, 338, 333, 340, 333, 330, 341, 329, 335, 341, 338]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617438698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199802024-02-28T22:20:45.743Z{"msg":"[2]R generation[0..12]: [180, 178, 183, 180, 184, 180, 179, 185, 178, 182, 185, 183]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617452787Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"Wait for a query_work_queue command to finish before sending IO
199812024-02-28T22:20:45.743Z}
199822024-02-28T22:20:45.743Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617481426Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199832024-02-28T22:20:45.743Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617494556Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199842024-02-28T22:20:45.743Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617528705Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199852024-02-28T22:20:45.744Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617556394Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199862024-02-28T22:20:45.744Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617570994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"":"downstairs","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199872024-02-28T22:20:45.744Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617584743Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199882024-02-28T22:20:45.744Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617597003Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"0","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199892024-02-28T22:20:45.744Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617612343Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"1","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199902024-02-28T22:20:45.744Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617644282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"client":"2","":"downstairs","session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199912024-02-28T22:20:45.744Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617674381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199922024-02-28T22:20:45.744Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.6177045Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199932024-02-28T22:20:45.744Z{"msg":"5e89c8a6-e166-4c2a-a70b-7945dcffe20d is now active with session: 9401c809-b469-4570-aee5-bb166dfd594f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.61772243Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199942024-02-28T22:20:45.744Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617744329Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879,"session_id":"9401c809-b469-4570-aee5-bb166dfd594f"}
199952024-02-28T22:20:45.744Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:44.617763058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
199962024-02-28T22:20:45.744ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
199972024-02-28T22:20:45.748ZLoading write count information from file "/tmp/test_repair_verify.data"
199982024-02-28T22:20:45.748ZRun Repair workload
199992024-02-28T22:20:45.748Z01/30 Write block 366 len 20480 data: 12 11 11 13 14
200002024-02-28T22:20:45.748Z02/30 Read block 324 len 32768
200012024-02-28T22:20:45.748Z03/30 Read block 520 len 40960
200022024-02-28T22:20:45.748Z04/30 Read block 208 len 16384
200032024-02-28T22:20:45.748Z05/30 Flush
200042024-02-28T22:20:45.748Z06/30 Write block 235 len 36864 data: 13 13 14 13 9 9 9 10 14
200052024-02-28T22:20:45.748Z07/30 Read block 501 len 16384
200062024-02-28T22:20:45.751Z08/30 Read block 263 len 4096
200072024-02-28T22:20:45.752Z09/30 Read block 256 len 20480
200082024-02-28T22:20:45.752Z10/30 Write block 321 len 32768 data: 10 12 13 16 18 19 19 22
200092024-02-28T22:20:45.752Z11/30 Write block 94 len 16384 data: 12 11 13 14
200102024-02-28T22:20:45.755Z12/30 Read block 218 len 28672
200112024-02-28T22:20:45.755Z13/30 Write block 249 len 12288 data: 20 19 19
200122024-02-28T22:20:45.755Z14/30 Read block 108 len 28672
200132024-02-28T22:20:45.755Z15/30 Read block 361 len 40960
200142024-02-28T22:20:45.760Z16/30 Write block 589 len 12288 data: 10 12 13
200152024-02-28T22:20:45.760Z17/30 Write block 390 len 28672 data: 18 19 17 13 13 16 13
200162024-02-28T22:20:45.760Z18/30 Write block 497 len 32768 data: 15 12 13 14 15 16 16 17
200172024-02-28T22:20:45.760Z19/30 Read block 126 len 20480
200182024-02-28T22:20:45.763Z20/30 Read block 577 len 28672
200192024-02-28T22:20:45.763Z21/30 Write block 102 len 36864 data: 12 11 11 12 14 15 19 22 18
200202024-02-28T22:20:45.763Z22/30 Write block 330 len 24576 data: 21 19 14 13 13 15
200212024-02-28T22:20:45.763Z23/30 Write block 206 len 24576 data: 11 9 10 9 11 11
200222024-02-28T22:20:45.763Z24/30 Flush
200232024-02-28T22:20:45.768Z25/30 Write block 29 len 8192 data: 13 12
200242024-02-28T22:20:45.772Z26/30 Read block 232 len 28672
200252024-02-28T22:20:45.772Z27/30 Read block 359 len 12288
200262024-02-28T22:20:45.772Z28/30 Write block 571 len 32768 data: 12 14 16 16 16 16 17 17
200272024-02-28T22:20:45.772Z29/30 Read block 265 len 12288
200282024-02-28T22:20:45.776Z30/30 Read block 296 len 4096
200292024-02-28T22:20:45.776Z----------------------------------------------------------------
200302024-02-28T22:20:45.776Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200312024-02-28T22:20:45.776ZStates: Active Active Active
200322024-02-28T22:20:45.776ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200332024-02-28T22:20:45.776Z 1 Acked 1000 Write 5 Done Done Done false
200342024-02-28T22:20:45.776Z 2 Acked 1001 Read 8 Done Done Done false
200352024-02-28T22:20:45.776Z 3 Acked 1002 Read 10 Done Done Done false
200362024-02-28T22:20:45.776Z 4 Acked 1003 Read 4 Done Done Done false
200372024-02-28T22:20:45.776Z 5 Acked 1004 Flush 0 Done Sent Done false
200382024-02-28T22:20:45.776Z 6 Acked 1005 Write 9 Done Sent Done false
200392024-02-28T22:20:45.776Z 7 Acked 1006 Read 4 Done Sent Done false
200402024-02-28T22:20:45.776Z 8 Acked 1007 Read 1 Done Sent Done false
200412024-02-28T22:20:45.776Z 9 Acked 1008 Read 5 Done Sent Done false
200422024-02-28T22:20:45.776Z 10 Acked 1009 Write 8 Done Sent Done false
200432024-02-28T22:20:45.776Z 11 Acked 1010 Write 4 Done Sent Done false
200442024-02-28T22:20:45.776Z 12 Acked 1011 Read 7 Done Sent Done false
200452024-02-28T22:20:45.776Z 13 Acked 1012 Write 3 Done Sent Done false
200462024-02-28T22:20:45.776Z 14 Acked 1013 Read 7 Done Sent Done false
200472024-02-28T22:20:45.777Z 15 Acked 1014 Read 10 Done Sent Done false
200482024-02-28T22:20:45.777Z 16 Acked 1015 Write 3 Done Sent Done false
200492024-02-28T22:20:45.777Z 17 Acked 1016 Write 7 Done Sent Done false
200502024-02-28T22:20:45.777Z 18 Acked 1017 Write 8 Done Sent Done false
200512024-02-28T22:20:45.777Z 19 Acked 1018 Read 5 Done Sent Done false
200522024-02-28T22:20:45.777Z 20 Acked 1019 Read 7 Done Sent Done false
200532024-02-28T22:20:45.777Z 21 Acked 1020 Write 9 Done Sent Done false
200542024-02-28T22:20:45.777Z 22 Acked 1021 Write 6 Done Sent Done false
200552024-02-28T22:20:45.777Z 23 Acked 1022 Write 6 Done Sent Done false
200562024-02-28T22:20:45.777Z 24 Acked 1023 Flush 0 Done Sent Done false
200572024-02-28T22:20:45.777Z 25 Acked 1024 Write 2 Done Sent Done false
200582024-02-28T22:20:45.777Z 26 Acked 1025 Read 7 Done Sent Done false
200592024-02-28T22:20:45.777Z 27 Acked 1026 Read 3 Done Sent Done false
200602024-02-28T22:20:45.777Z 28 Acked 1027 Write 8 Done Sent Done false
200612024-02-28T22:20:45.777Z 29 Acked 1028 Read 3 Done Sent Done false
200622024-02-28T22:20:45.777Z 30 Acked 1029 Read 1 Done Sent Sent false
200632024-02-28T22:20:45.777Z STATES DS:0 DS:1 DS:2 TOTAL
200642024-02-28T22:20:45.777Z New 0 0 0 0
200652024-02-28T22:20:45.777Z Sent 0 26 1 27
200662024-02-28T22:20:45.777Z Done 30 4 29 63
200672024-02-28T22:20:45.777Z Skipped 0 0 0 0
200682024-02-28T22:20:45.777Z Error 0 0 0 0
200692024-02-28T22:20:45.778ZLast Flush: 1023 0 1023
200702024-02-28T22:20:45.778ZDownstairs last five completed:
200712024-02-28T22:20:45.778ZUpstairs last five completed: 30 29 28 27 26
200722024-02-28T22:20:45.778Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:44.651588087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1879}
200732024-02-28T22:20:45.778ZWrote out file "/tmp/test_repair_verify.data"
200742024-02-28T22:20:45.784Z
200752024-02-28T22:20:45.784ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200762024-02-28T22:20:45.798ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200772024-02-28T22:20:45.798Z 0 000-019 180 180 180 332 332 332 F F F
200782024-02-28T22:20:45.798Z 1 020-039 178 178 178 329 329 329 T F T <---
200792024-02-28T22:20:45.798Z 2 040-059 183 183 183 338 338 338 F F F
200802024-02-28T22:20:45.798Z 3 060-079 180 180 180 333 333 333 F F F
200812024-02-28T22:20:45.798Z 4 080-099 186 184 186 343 340 343 F F F <---
200822024-02-28T22:20:45.798Z 5 100-119 186 180 186 343 333 343 F F F <---
200832024-02-28T22:20:45.798Z 6 120-139 179 179 179 330 330 330 F F F
200842024-02-28T22:20:45.798Z 7 140-159 185 185 185 341 341 341 F F F
200852024-02-28T22:20:45.798Z 8 160-179 178 178 178 329 329 329 F F F
200862024-02-28T22:20:45.798Z 9 180-199 182 182 182 335 335 335 F F F
200872024-02-28T22:20:45.798Z 10 200-219 186 185 186 343 341 343 F F F <---
200882024-02-28T22:20:45.798Z 11 220-239 186 183 186 343 338 343 F F F <---
200892024-02-28T22:20:45.802Z 12 240-259 186 184 186 343 340 343 F F F <---
200902024-02-28T22:20:45.802Z 13 260-279 183 183 183 338 338 338 F F F
200912024-02-28T22:20:45.802Z 14 280-299 182 182 182 335 335 335 F F F
200922024-02-28T22:20:45.802Z 15 300-319 182 182 182 335 335 335 F F F
200932024-02-28T22:20:45.802Z 16 320-339 186 182 186 343 335 343 F F F <---
200942024-02-28T22:20:45.802Z 17 340-359 180 180 180 331 331 331 F F F
200952024-02-28T22:20:45.802Z 18 360-379 186 184 186 342 340 342 F T F <---
200962024-02-28T22:20:45.802Z 19 380-399 186 185 186 343 341 343 F F F <---
200972024-02-28T22:20:45.802Z 20 400-419 182 182 182 336 336 336 F F F
200982024-02-28T22:20:45.802Z 21 420-439 180 180 180 333 333 333 F F F
200992024-02-28T22:20:45.802Z 22 440-459 178 178 178 328 328 328 F F F
201002024-02-28T22:20:45.802Z 23 460-479 180 180 180 332 332 332 F F F
201012024-02-28T22:20:45.802Z 24 480-499 186 176 186 343 325 343 F F F <---
201022024-02-28T22:20:45.802Z 25 500-519 186 183 186 343 338 343 F F F <---
201032024-02-28T22:20:45.802Z 26 520-539 184 184 184 339 339 339 F F F
201042024-02-28T22:20:45.803Z 27 540-559 175 175 175 324 324 324 F F F
201052024-02-28T22:20:45.803Z 28 560-579 181 181 181 334 334 334 T F T <---
201062024-02-28T22:20:45.803Z 29 580-599 186 180 186 343 333 343 F F F <---
201072024-02-28T22:20:45.803ZMax gen: 186, Max flush: 343
201082024-02-28T22:20:45.803ZError: Difference in extent metadata found!
201092024-02-28T22:20:45.803ZA Difference in extent metadata is expected here
201102024-02-28T22:20:45.803ZOn loop 93
201112024-02-28T22:20:45.803Z
201122024-02-28T22:20:45.806ZVerifying data now
201132024-02-28T22:20:50.453ZNo change after verify
201142024-02-28T22:20:50.453ZLoop: 93 Downstairs dump after verify (and repair):
201152024-02-28T22:20:50.467ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201162024-02-28T22:20:50.467Z 0 000-019 180 180 180 332 332 332 F F F
201172024-02-28T22:20:50.467Z 1 020-039 187 187 187 344 344 344 F F F
201182024-02-28T22:20:50.467Z 2 040-059 183 183 183 338 338 338 F F F
201192024-02-28T22:20:50.467Z 3 060-079 180 180 180 333 333 333 F F F
201202024-02-28T22:20:50.467Z 4 080-099 186 186 186 343 343 343 F F F
201212024-02-28T22:20:50.467Z 5 100-119 186 186 186 343 343 343 F F F
201222024-02-28T22:20:50.467Z 6 120-139 179 179 179 330 330 330 F F F
201232024-02-28T22:20:50.467Z 7 140-159 185 185 185 341 341 341 F F F
201242024-02-28T22:20:50.467Z 8 160-179 178 178 178 329 329 329 F F F
201252024-02-28T22:20:50.471Z 9 180-199 182 182 182 335 335 335 F F F
201262024-02-28T22:20:50.471Z 10 200-219 186 186 186 343 343 343 F F F
201272024-02-28T22:20:50.471Z 11 220-239 186 186 186 343 343 343 F F F
201282024-02-28T22:20:50.471Z 12 240-259 186 186 186 343 343 343 F F F
201292024-02-28T22:20:50.471Z 13 260-279 183 183 183 338 338 338 F F F
201302024-02-28T22:20:50.471Z 14 280-299 182 182 182 335 335 335 F F F
201312024-02-28T22:20:50.471Z 15 300-319 182 182 182 335 335 335 F F F
201322024-02-28T22:20:50.471Z 16 320-339 186 186 186 343 343 343 F F F
201332024-02-28T22:20:50.471Z 17 340-359 180 180 180 331 331 331 F F F
201342024-02-28T22:20:50.471Z 18 360-379 186 186 186 342 342 342 F F F
201352024-02-28T22:20:50.471Z 19 380-399 186 186 186 343 343 343 F F F
201362024-02-28T22:20:50.471Z 20 400-419 182 182 182 336 336 336 F F F
201372024-02-28T22:20:50.471Z 21 420-439 180 180 180 333 333 333 F F F
201382024-02-28T22:20:50.472Z 22 440-459 178 178 178 328 328 328 F F F
201392024-02-28T22:20:50.472Z 23 460-479 180 180 180 332 332 332 F F F
201402024-02-28T22:20:50.472Z 24 480-499 186 186 186 343 343 343 F F F
201412024-02-28T22:20:50.472Z 25 500-519 186 186 186 343 343 343 F F F
201422024-02-28T22:20:50.472Z 26 520-539 184 184 184 339 339 339 F F F
201432024-02-28T22:20:50.472Z 27 540-559 175 175 175 324 324 324 F F F
201442024-02-28T22:20:50.472Z 28 560-579 187 187 187 344 344 344 F F F
201452024-02-28T22:20:50.472Z 29 580-599 186 186 186 343 343 343 F F F
201462024-02-28T22:20:50.472ZMax gen: 187, Max flush: 344
201472024-02-28T22:20:50.472Z
201482024-02-28T22:20:50.472ZBegin loop 94
201492024-02-28T22:20:50.472ZDownstairs to restart: 2
201502024-02-28T22:20:50.488Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.366656961Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
201512024-02-28T22:20:50.488Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.366928384Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
201522024-02-28T22:20:50.493Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.366964103Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
201532024-02-28T22:20:50.494Z{"msg":"Crucible 2989e5eb-8d5e-4ac2-9ee0-256737482f99 has session id: 96f3e4d3-2b9f-4983-abba-602c8a68b5db","v":0,"name":"crucible","level":30Crucible runtime is spawned
201542024-02-28T22:20:50.494Z,"time":"2024-02-28T22:20:49.367008481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201552024-02-28T22:20:50.494Z{"msg":"Upstairs opts: Upstairs UUID: 2989e5eb-8d5e-4ac2-9ee0-256737482f99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.36707385Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201562024-02-28T22:20:50.494Z{"msg":"Crucible stats registered with UUID: 2989e5eb-8d5e-4ac2-9ee0-256737482f99","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367128568Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201572024-02-28T22:20:50.494Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367158128Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
201582024-02-28T22:20:50.494Z{"msg":"2989e5eb-8d5e-4ac2-9ee0-256737482f99 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367207006Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201592024-02-28T22:20:50.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367238285Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201602024-02-28T22:20:50.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367278464Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201612024-02-28T22:20:50.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367330473Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201622024-02-28T22:20:50.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367371242Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201632024-02-28T22:20:50.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367405431Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201642024-02-28T22:20:50.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367450039Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201652024-02-28T22:20:50.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367484259Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201662024-02-28T22:20:50.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367540047Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201672024-02-28T22:20:50.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367592656Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201682024-02-28T22:20:50.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367644374Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201692024-02-28T22:20:50.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367681913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201702024-02-28T22:20:50.494Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.367715082Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201712024-02-28T22:20:50.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368347485Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201722024-02-28T22:20:50.494Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368383634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201732024-02-28T22:20:50.494Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368399793Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201742024-02-28T22:20:50.494Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368415483Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201752024-02-28T22:20:50.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368449652Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201762024-02-28T22:20:50.495Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.368486811Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201772024-02-28T22:20:50.495Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369009886Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201782024-02-28T22:20:50.495Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369046745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201792024-02-28T22:20:50.495Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369076404Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201802024-02-28T22:20:50.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369515772Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201812024-02-28T22:20:50.495Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369556511Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201822024-02-28T22:20:50.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369583301Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201832024-02-28T22:20:50.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369623369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201842024-02-28T22:20:50.495Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369648639Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201852024-02-28T22:20:50.495Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369681068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201862024-02-28T22:20:50.495Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369704767Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201872024-02-28T22:20:50.495Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369732197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201882024-02-28T22:20:50.495Z{"msg":"[0]R flush_numbers[0..12]: [332, 344, 338, 333, 343, 343, 330, 341, 329, 335, 343, 343]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369756996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201892024-02-28T22:20:50.495Z{"msg":"[0]R generation[0..12]: [180, 187, 183, 180, 186, 186, 179, 185, 178, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369784905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201902024-02-28T22:20:50.495Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369810624Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201912024-02-28T22:20:50.495Z{"msg":"[1]R flush_numbers[0..12]: [332, 344, 338, 333, 343, 343, 330, 341, 329, 335, 343, 343]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
201922024-02-28T22:20:50.495Z,"time":"2024-02-28T22:20:49.369840883Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201932024-02-28T22:20:50.495Z{"msg":"[1]R generation[0..12]: [180, 187, 183, 180, 186, 186, 179, 185, 178, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369866683Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201942024-02-28T22:20:50.495Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369894902Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201952024-02-28T22:20:50.495Z{"msg":"[2]R flush_numbers[0..12]: [332, 344, 338, 333, 343, 343, 330, 341, 329, 335, 343, 343]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.369920761Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201962024-02-28T22:20:50.495Z{"msg":"[2]R generation[0..12]: [180, 187, 183, 180, 186, 186, 179, 185, 178, 182, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.36995084Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201972024-02-28T22:20:50.496Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.36997664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201982024-02-28T22:20:50.496Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370004849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
201992024-02-28T22:20:50.496Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370029148Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202002024-02-28T22:20:50.496Z{"msg":"Next flush: 345","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370055388Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202012024-02-28T22:20:50.496Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370078497Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"":"downstairs","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202022024-02-28T22:20:50.496Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370105736Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202032024-02-28T22:20:50.496Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370127806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"0","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202042024-02-28T22:20:50.496Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370154775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"1","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202052024-02-28T22:20:50.500Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370178214Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"client":"2","":"downstairs","session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202062024-02-28T22:20:50.500Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370204704Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202072024-02-28T22:20:50.500Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370226943Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202082024-02-28T22:20:50.500Z}
202092024-02-28T22:20:50.500Z{"msg":"2989e5eb-8d5e-4ac2-9ee0-256737482f99 is now active with session: 96f3e4d3-2b9f-4983-abba-602c8a68b5db","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370259092Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202102024-02-28T22:20:50.500Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370280311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887,"session_id":"96f3e4d3-2b9f-4983-abba-602c8a68b5db"}
202112024-02-28T22:20:50.500Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:49.370307591Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
202122024-02-28T22:20:50.500ZLoading write count information from file "/tmp/test_repair_verify.data"
202132024-02-28T22:20:50.500ZRun Repair workload
202142024-02-28T22:20:50.500Z01/30 Write block 567 len 40960 data: 21 18 15 14 13 15 17 17 17 17
202152024-02-28T22:20:50.500Z02/30 Read block 108 len 24576
202162024-02-28T22:20:50.500Z03/30 Flush
202172024-02-28T22:20:50.500Z04/30 Write block 113 len 4096 data: 17
202182024-02-28T22:20:50.500Z05/30 Write block 203 len 4096 data: 12
202192024-02-28T22:20:50.500Z06/30 Write block 32 len 28672 data: 11 12 10 10 13 14 16
202202024-02-28T22:20:50.500Z07/30 Read block 475 len 20480
202212024-02-28T22:20:50.504Z08/30 Write block 421 len 24576 data: 18 18 16 15 13 9
202222024-02-28T22:20:50.504Z09/30 Read block 499 len 16384
202232024-02-28T22:20:50.504Z10/30 Read block 396 len 12288
202242024-02-28T22:20:50.504Z11/30 Read block 160 len 32768
202252024-02-28T22:20:50.504Z12/30 Read block 344 len 20480
202262024-02-28T22:20:50.508Z13/30 Read block 518 len 36864
202272024-02-28T22:20:50.508Z14/30 Read block 433 len 8192
202282024-02-28T22:20:50.509Z15/30 Write block 350 len 36864 data: 14 16 17 18 18 17 14 16 14
202292024-02-28T22:20:50.512Z16/30 Read block 263 len 28672
202302024-02-28T22:20:50.512Z17/30 Write block 7 len 16384 data: 19 19 13 12
202312024-02-28T22:20:50.512Z18/30 Write block 470 len 28672 data: 15 17 18 15 16 13 14
202322024-02-28T22:20:50.512Z19/30 Flush
202332024-02-28T22:20:50.515Z20/30 Write block 239 len 36864 data: 10 10 10 11 15 14 16 13 22
202342024-02-28T22:20:50.515Z21/30 Read block 67 len 36864
202352024-02-28T22:20:50.518Z22/30 Write block 123 len 36864 data: 14 13 15 18 22 25 21 20 20
202362024-02-28T22:20:50.519Z23/30 Write block 102 len 28672 data: 13 12 12 13 15 16 20
202372024-02-28T22:20:50.519Z24/30 Write block 14 len 28672 data: 13 13 14 15 14 15 13
202382024-02-28T22:20:50.528Z25/30 Write block 37 len 24576 data: 15 17 14 11 12 9
202392024-02-28T22:20:50.528Z26/30 Read block 129 len 4096
202402024-02-28T22:20:50.528Z27/30 Read block 43 len 28672
202412024-02-28T22:20:50.528Z28/30 Write block 40 len 4096 data: 12
202422024-02-28T22:20:50.528Z29/30 Read block 305 len 40960
202432024-02-28T22:20:50.557Z30/30 Read block 158 len 32768
202442024-02-28T22:20:50.557Z----------------------------------------------------------------
202452024-02-28T22:20:50.557Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202462024-02-28T22:20:50.557ZStates: Active Active Active
202472024-02-28T22:20:50.557ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202482024-02-28T22:20:50.557Z 1 Acked 1000 Write 10 Done Done Done false
202492024-02-28T22:20:50.557Z 2 Acked 1001 Read 6 Done Done Done false
202502024-02-28T22:20:50.557Z 3 Acked 1002 Flush 0 Done Done Sent false
202512024-02-28T22:20:50.557Z 4 Acked 1003 Write 1 Done Done Sent false
202522024-02-28T22:20:50.557Z 5 Acked 1004 Write 1 Done Done Sent false
202532024-02-28T22:20:50.557Z 6 Acked 1005 Write 7 Done Done Sent false
202542024-02-28T22:20:50.557Z 7 Acked 1006 Read 5 Done Done Sent false
202552024-02-28T22:20:50.557Z 8 Acked 1007 Write 6 Done Done Sent false
202562024-02-28T22:20:50.558Z 9 Acked 1008 Read 4 Done Done Sent false
202572024-02-28T22:20:50.558Z 10 Acked 1009 Read 3 Done Done Sent false
202582024-02-28T22:20:50.558Z 11 Acked 1010 Read 8 Done Done Sent false
202592024-02-28T22:20:50.558Z 12 Acked 1011 Read 5 Done Done Sent false
202602024-02-28T22:20:50.558Z 13 Acked 1012 Read 9 Done Done Sent false
202612024-02-28T22:20:50.558Z 14 Acked 1013 Read 2 Done Done Sent false
202622024-02-28T22:20:50.558Z 15 Acked 1014 Write 9 Done Done Sent false
202632024-02-28T22:20:50.558Z 16 Acked 1015 Read 7 Done Done Sent false
202642024-02-28T22:20:50.558Z 17 Acked 1016 Write 4 Done Done Sent false
202652024-02-28T22:20:50.558Z 18 Acked 1017 Write 7 Done Done Sent false
202662024-02-28T22:20:50.558Z 19 Acked 1018 Flush 0 Done Done Sent false
202672024-02-28T22:20:50.558Z 20 Acked 1019 Write 9 Done Done Sent false
202682024-02-28T22:20:50.558Z 21 Acked 1020 Read 9 Done Done Sent false
202692024-02-28T22:20:50.558Z 22 Acked 1021 Write 9 Done Done Sent false
202702024-02-28T22:20:50.558Z 23 Acked 1022 Write 7 Done Done Sent false
202712024-02-28T22:20:50.558Z 24 Acked 1023 Write 7 Done Done Sent false
202722024-02-28T22:20:50.558Z 25 Acked 1024 Write 6 Done Done Sent false
202732024-02-28T22:20:50.558Z 26 Acked 1025 Read 1 Done Done Sent false
202742024-02-28T22:20:50.558Z 27 Acked 1026 Read 7 Done Done Sent false
202752024-02-28T22:20:50.558Z 28 Acked 1027 Write 1 Done Done Sent false
202762024-02-28T22:20:50.558Z 29 Acked 1028 Read 10 Done Done Sent false
202772024-02-28T22:20:50.558Z 30 Acked 1029 Read 8 Done Done Sent false
202782024-02-28T22:20:50.559Z STATES DS:0 DS:1 DS:2 TOTAL
202792024-02-28T22:20:50.559Z New 0 0 0 0
202802024-02-28T22:20:50.559Z Sent 0 0 28 28
202812024-02-28T22:20:50.559Z Done 30 30 2 62
202822024-02-28T22:20:50.559Z Skipped 0 0 0 0
202832024-02-28T22:20:50.559Z Error 0 0 0 0
202842024-02-28T22:20:50.559ZLast Flush: 1018 1018 0
202852024-02-28T22:20:50.559ZDownstairs last five completed:
202862024-02-28T22:20:50.559ZUpstairs last five completed: 30 29 28 27 26
202872024-02-28T22:20:50.559Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:49.402772846Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1887}
202882024-02-28T22:20:50.559ZWrote out file "/tmp/test_repair_verify.data"
202892024-02-28T22:20:50.559Z
202902024-02-28T22:20:50.559ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202912024-02-28T22:20:50.559ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202922024-02-28T22:20:50.559Z 0 000-019 188 188 180 346 346 332 T T F <---
202932024-02-28T22:20:50.559Z 1 020-039 188 188 187 346 346 344 T T F <---
202942024-02-28T22:20:50.559Z 2 040-059 183 183 183 338 338 338 T T F <---
202952024-02-28T22:20:50.559Z 3 060-079 180 180 180 333 333 333 F F F
202962024-02-28T22:20:50.559Z 4 080-099 186 186 186 343 343 343 F F F
202972024-02-28T22:20:50.559Z 5 100-119 188 188 186 346 346 343 T T F <---
202982024-02-28T22:20:50.559Z 6 120-139 179 179 179 330 330 330 T T F <---
202992024-02-28T22:20:50.559Z 7 140-159 185 185 185 341 341 341 F F F
203002024-02-28T22:20:50.560Z 8 160-179 178 178 178 329 329 329 F F F
203012024-02-28T22:20:50.560Z 9 180-199 182 182 182 335 335 335 F F F
203022024-02-28T22:20:50.560Z 10 200-219 188 188 186 346 346 343 F F F <---
203032024-02-28T22:20:50.560Z 11 220-239 186 186 186 343 343 343 T T F <---
203042024-02-28T22:20:50.560Z 12 240-259 186 186 186 343 343 343 T T F <---
203052024-02-28T22:20:50.560Z 13 260-279 183 183 183 338 338 338 F F F
203062024-02-28T22:20:50.560Z 14 280-299 182 182 182 335 335 335 F F F
203072024-02-28T22:20:50.560Z 15 300-319 182 182 182 335 335 335 F F F
203082024-02-28T22:20:50.560Z 16 320-339 186 186 186 343 343 343 F F F
203092024-02-28T22:20:50.560Z 17 340-359 188 188 180 346 346 331 F F F <---
203102024-02-28T22:20:50.560Z 18 360-379 186 186 186 342 342 342 F F F
203112024-02-28T22:20:50.560Z 19 380-399 186 186 186 343 343 343 F F F
203122024-02-28T22:20:50.560Z 20 400-419 182 182 182 336 336 336 F F F
203132024-02-28T22:20:50.560Z 21 420-439 188 188 180 346 346 333 F F F <---
203142024-02-28T22:20:50.560Z 22 440-459 178 178 178 328 328 328 F F F
203152024-02-28T22:20:50.560Z 23 460-479 188 188 180 346 346 332 F F F <---
203162024-02-28T22:20:50.560Z 24 480-499 186 186 186 343 343 343 F F F
203172024-02-28T22:20:50.560Z 25 500-519 186 186 186 343 343 343 F F F
203182024-02-28T22:20:50.560Z 26 520-539 184 184 184 339 339 339 F F F
203192024-02-28T22:20:50.560Z 27 540-559 175 175 175 324 324 324 F F F
203202024-02-28T22:20:50.560Z 28 560-579 188 188 187 345 345 344 F F T <---
203212024-02-28T22:20:50.560Z 29 580-599 186 186 186 343 343 343 F F F
203222024-02-28T22:20:50.560ZMax gen: 188, Max flush: 346
203232024-02-28T22:20:50.560ZError: Difference in extent metadata found!
203242024-02-28T22:20:50.561ZA Difference in extent metadata is expected here
203252024-02-28T22:20:50.561ZOn loop 94
203262024-02-28T22:20:50.561Z
203272024-02-28T22:20:50.561ZVerifying data now
203282024-02-28T22:20:55.179ZNo change after verify
203292024-02-28T22:20:55.179ZLoop: 94 Downstairs dump after verify (and repair):
203302024-02-28T22:20:55.194ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203312024-02-28T22:20:55.194Z 0 000-019 189 189 189 347 347 347 F F F
203322024-02-28T22:20:55.194Z 1 020-039 189 189 189 347 347 347 F F F
203332024-02-28T22:20:55.195Z 2 040-059 189 189 189 347 347 347 F F F
203342024-02-28T22:20:55.195Z 3 060-079 180 180 180 333 333 333 F F F
203352024-02-28T22:20:55.195Z 4 080-099 186 186 186 343 343 343 F F F
203362024-02-28T22:20:55.195Z 5 100-119 189 189 189 347 347 347 F F F
203372024-02-28T22:20:55.195Z 6 120-139 189 189 189 347 347 347 F F F
203382024-02-28T22:20:55.196Z 7 140-159 185 185 185 341 341 341 F F F
203392024-02-28T22:20:55.196Z 8 160-179 178 178 178 329 329 329 F F F
203402024-02-28T22:20:55.196Z 9 180-199 182 182 182 335 335 335 F F F
203412024-02-28T22:20:55.196Z 10 200-219 188 188 188 346 346 346 F F F
203422024-02-28T22:20:55.196Z 11 220-239 189 189 189 347 347 347 F F F
203432024-02-28T22:20:55.204Z 12 240-259 189 189 189 347 347 347 F F F
203442024-02-28T22:20:55.204Z 13 260-279 183 183 183 338 338 338 F F F
203452024-02-28T22:20:55.204Z 14 280-299 182 182 182 335 335 335 F F F
203462024-02-28T22:20:55.204Z 15 300-319 182 182 182 335 335 335 F F F
203472024-02-28T22:20:55.205Z 16 320-339 186 186 186 343 343 343 F F F
203482024-02-28T22:20:55.205Z 17 340-359 188 188 188 346 346 346 F F F
203492024-02-28T22:20:55.205Z 18 360-379 186 186 186 342 342 342 F F F
203502024-02-28T22:20:55.205Z 19 380-399 186 186 186 343 343 343 F F F
203512024-02-28T22:20:55.205Z 20 400-419 182 182 182 336 336 336 F F F
203522024-02-28T22:20:55.205Z 21 420-439 188 188 188 346 346 346 F F F
203532024-02-28T22:20:55.205Z 22 440-459 178 178 178 328 328 328 F F F
203542024-02-28T22:20:55.206Z 23 460-479 188 188 188 346 346 346 F F F
203552024-02-28T22:20:55.206Z 24 480-499 186 186 186 343 343 343 F F F
203562024-02-28T22:20:55.206Z 25 500-519 186 186 186 343 343 343 F F F
203572024-02-28T22:20:55.206Z 26 520-539 184 184 184 339 339 339 F F F
203582024-02-28T22:20:55.206Z 27 540-559 175 175 175 324 324 324 F F F
203592024-02-28T22:20:55.207Z 28 560-579 188 188 188 345 345 345 F F F
203602024-02-28T22:20:55.207Z 29 580-599 186 186 186 343 343 343 F F F
203612024-02-28T22:20:55.207ZMax gen: 189, Max flush: 347
203622024-02-28T22:20:55.207Z
203632024-02-28T22:20:55.207ZBegin loop 95
203642024-02-28T22:20:55.207ZDownstairs to restart: 1
203652024-02-28T22:20:55.220Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.091833997Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
203662024-02-28T22:20:55.220Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\"Crucible runtime is spawned
203672024-02-28T22:20:55.246Z,\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092189008Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
203682024-02-28T22:20:55.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092236886Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
203692024-02-28T22:20:55.247Z{"msg":"Crucible 909901e5-551c-40d3-b267-d3d02f11782a has session id: 8d7c0afd-8fe2-4f62-ac66-f2b680898ef7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092258096Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203702024-02-28T22:20:55.247Z{"msg":"Upstairs opts: Upstairs UUID: 909901e5-551c-40d3-b267-d3d02f11782a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092314594Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203712024-02-28T22:20:55.247Z{"msg":"Crucible stats registered with UUID: 909901e5-551c-40d3-b267-d3d02f11782a","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092349973Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203722024-02-28T22:20:55.247Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092367893Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
203732024-02-28T22:20:55.247Z{"msg":"909901e5-551c-40d3-b267-d3d02f11782a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092391932Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203742024-02-28T22:20:55.247Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092408261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203752024-02-28T22:20:55.247Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092448151Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203762024-02-28T22:20:55.248Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092480959Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203772024-02-28T22:20:55.248Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092535548Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203782024-02-28T22:20:55.248Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092604716Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203792024-02-28T22:20:55.248Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092642625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203802024-02-28T22:20:55.248Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092692194Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203812024-02-28T22:20:55.248Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092732713Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203822024-02-28T22:20:55.248Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092764802Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203832024-02-28T22:20:55.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092793071Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203842024-02-28T22:20:55.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.09283152Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203852024-02-28T22:20:55.249Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.092868719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203862024-02-28T22:20:55.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.093504451Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203872024-02-28T22:20:55.249Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.09354171Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203882024-02-28T22:20:55.249Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.09355773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203892024-02-28T22:20:55.249Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.093579999Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203902024-02-28T22:20:55.250Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.093594269Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203912024-02-28T22:20:55.250Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.093617618Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203922024-02-28T22:20:55.250Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094195062Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203932024-02-28T22:20:55.250Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094250481Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203942024-02-28T22:20:55.250Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094458905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203952024-02-28T22:20:55.251Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094672599Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203962024-02-28T22:20:55.251Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094711308Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203972024-02-28T22:20:55.251Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094728898Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203982024-02-28T22:20:55.251Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094742097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
203992024-02-28T22:20:55.251Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094764487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204002024-02-28T22:20:55.251Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094779456Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204012024-02-28T22:20:55.251Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.094967381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204022024-02-28T22:20:55.251Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.09501529Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204032024-02-28T22:20:55.251Z{"msg":"[0]R flush_numbers[0..12]: [347, 347, 347, 333, 343, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095036929Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204042024-02-28T22:20:55.252Z{"msg":"[0]R generation[0..12]: [189, 189, 189, 180, 186, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095052849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204052024-02-28T22:20:55.252Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095075808Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204062024-02-28T22:20:55.252Z{"msg":"[1]R flush_numbers[0..12]: [347, 347, 347, 333, 343, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095090708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204072024-02-28T22:20:55.252Z{"msg":"[1]R generation[0..12]: [189, 189, 189, 180, 186, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095117227Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204082024-02-28T22:20:55.252ZWait for a query_work_queue command to finish before sending IO
204092024-02-28T22:20:55.252Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095159976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204102024-02-28T22:20:55.252Z{"msg":"[2]R flush_numbers[0..12]: [347, 347, 347, 333, 343, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095196595Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204112024-02-28T22:20:55.253Z{"msg":"[2]R generation[0..12]: [189, 189, 189, 180, 186, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095214724Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204122024-02-28T22:20:55.253Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095229354Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204132024-02-28T22:20:55.253Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095243263Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204142024-02-28T22:20:55.253Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095259533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204152024-02-28T22:20:55.253Z{"msg":"Next flush: 348","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095294012Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204162024-02-28T22:20:55.253Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095328111Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"":"downstairs","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204172024-02-28T22:20:55.254Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095344991Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204182024-02-28T22:20:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095373Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"0","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204192024-02-28T22:20:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095410359Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"1","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204202024-02-28T22:20:55.254Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095437808Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"client":"2","":"downstairs","session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204212024-02-28T22:20:55.254Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095466697Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204222024-02-28T22:20:55.254Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095502766Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204232024-02-28T22:20:55.254Z{"msg":"909901e5-551c-40d3-b267-d3d02f11782a is now active with session: 8d7c0afd-8fe2-4f62-ac66-f2b680898ef7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095522536Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204242024-02-28T22:20:55.254Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095537905Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895,"session_id":"8d7c0afd-8fe2-4f62-ac66-f2b680898ef7"}
204252024-02-28T22:20:55.255Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:54.095577944Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
204262024-02-28T22:20:55.255ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204272024-02-28T22:20:55.255ZLoading write count information from file "/tmp/test_repair_verify.data"
204282024-02-28T22:20:55.255ZRun Repair workload
204292024-02-28T22:20:55.255Z01/30 Write block 77 len 20480 data: 11 11 10 11 12
204302024-02-28T22:20:55.255Z02/30 Flush
204312024-02-28T22:20:55.255Z03/30 Write block 307 len 16384 data: 17 16 15 12
204322024-02-28T22:20:55.256Z04/30 Read block 247 len 4096
204332024-02-28T22:20:55.256Z05/30 Write block 371 len 20480 data: 15 17 14 18 16
204342024-02-28T22:20:55.256Z06/30 Read block 547 len 32768
204352024-02-28T22:20:55.266Z07/30 Write block 266 len 32768 data: 21 17 13 11 10 11 9 7
204362024-02-28T22:20:55.266Z08/30 Write block 401 len 16384 data: 11 11 10 13
204372024-02-28T22:20:55.267Z09/30 Read block 86 len 24576
204382024-02-28T22:20:55.267Z10/30 Write block 403 len 32768 data: 11 14 14 15 16 14 12 11
204392024-02-28T22:20:55.267Z11/30 Read block 242 len 8192
204402024-02-28T22:20:55.267Z12/30 Write block 285 len 32768 data: 14 15 12 11 11 9 8 8
204412024-02-28T22:20:55.267Z13/30 Read block 453 len 8192
204422024-02-28T22:20:55.268Z14/30 Read block 420 len 12288
204432024-02-28T22:20:55.268Z15/30 Flush
204442024-02-28T22:20:55.268Z16/30 Write block 517 len 24576 data: 22 19 20 20 18 16
204452024-02-28T22:20:55.268Z17/30 Write block 339 len 4096 data: 12
204462024-02-28T22:20:55.268Z18/30 Write block 448 len 32768 data: 17 17 19 19 18 21 19 18
204472024-02-28T22:20:55.268Z19/30 Write block 34 len 8192 data: 11 11
204482024-02-28T22:20:55.268Z20/30 Write block 328 len 24576 data: 23 22 22 20 15 14
204492024-02-28T22:20:55.268Z21/30 Read block 69 len 36864
204502024-02-28T22:20:55.269Z22/30 Read block 367 len 12288
204512024-02-28T22:20:55.269Z23/30 Read block 438 len 24576
204522024-02-28T22:20:55.269Z24/30 Read block 567 len 32768
204532024-02-28T22:20:55.269Z25/30 Read block 89 len 4096
204542024-02-28T22:20:55.269Z26/30 Write block 344 len 4096 data: 16
204552024-02-28T22:20:55.269Z27/30 Write block 556 len 28672 data: 11 12 14 13 14 16 15
204562024-02-28T22:20:55.269Z28/30 Read block 428 len 12288
204572024-02-28T22:20:55.269Z29/30 Read block 127 len 24576
204582024-02-28T22:20:55.269Z30/30 Read block 469 len 16384
204592024-02-28T22:20:55.269Z----------------------------------------------------------------
204602024-02-28T22:20:55.269Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:28
204612024-02-28T22:20:55.269ZStates: Active Active Active
204622024-02-28T22:20:55.269ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204632024-02-28T22:20:55.269Z 3 Acked 1002 Write 4 Done Done Done false
204642024-02-28T22:20:55.269Z 4 Acked 1003 Read 1 Done Done Done false
204652024-02-28T22:20:55.270Z 5 Acked 1004 Write 5 Done Done Done false
204662024-02-28T22:20:55.270Z 6 Acked 1005 Read 8 Done Done Done false
204672024-02-28T22:20:55.270Z 7 Acked 1006 Write 8 Done Done Done false
204682024-02-28T22:20:55.270Z 8 Acked 1007 Write 4 Done Done Done false
204692024-02-28T22:20:55.270Z 9 Acked 1008 Read 6 Done Done Done false
204702024-02-28T22:20:55.270Z 10 Acked 1009 Write 8 Done Sent Done false
204712024-02-28T22:20:55.270Z 11 Acked 1010 Read 2 Done Sent Done false
204722024-02-28T22:20:55.270Z 12 Acked 1011 Write 8 Done Sent Done false
204732024-02-28T22:20:55.270Z 13 Acked 1012 Read 2 Done Sent Done false
204742024-02-28T22:20:55.270Z 14 Acked 1013 Read 3 Done Sent Done false
204752024-02-28T22:20:55.270Z 15 Acked 1014 Flush 0 Done Sent Done false
204762024-02-28T22:20:55.270Z 16 Acked 1015 Write 6 Done Sent Done false
204772024-02-28T22:20:55.271Z 17 Acked 1016 Write 1 Done Sent Done false
204782024-02-28T22:20:55.271Z 18 Acked 1017 Write 8 Done Sent Done false
204792024-02-28T22:20:55.271Z 19 Acked 1018 Write 2 Done Sent Done false
204802024-02-28T22:20:55.271Z 20 Acked 1019 Write 6 Done Sent Done false
204812024-02-28T22:20:55.271Z 21 Acked 1020 Read 9 Done Sent Done false
204822024-02-28T22:20:55.271Z 22 Acked 1021 Read 3 Done Sent Done false
204832024-02-28T22:20:55.271Z 23 Acked 1022 Read 6 Done Sent Done false
204842024-02-28T22:20:55.272Z 24 Acked 1023 Read 8 Done Sent Done false
204852024-02-28T22:20:55.272Z 25 Acked 1024 Read 1 Done Sent Done false
204862024-02-28T22:20:55.272Z 26 Acked 1025 Write 1 Done Sent Done false
204872024-02-28T22:20:55.272Z 27 Acked 1026 Write 7 Done Sent Done false
204882024-02-28T22:20:55.272Z 28 Acked 1027 Read 3 Done Sent Done false
204892024-02-28T22:20:55.272Z 29 Acked 1028 Read 6 Done Sent Done false
204902024-02-28T22:20:55.272Z 30 Acked 1029 Read 4 Sent Sent Done false
204912024-02-28T22:20:55.273Z STATES DS:0 DS:1 DS:2 TOTAL
204922024-02-28T22:20:55.273Z New 0 0 0 0
204932024-02-28T22:20:55.273Z Sent 1 21 0 22
204942024-02-28T22:20:55.273Z Done 27 7 28 62
204952024-02-28T22:20:55.273Z Skipped 0 0 0 0
204962024-02-28T22:20:55.273Z Error 0 0 0 0
204972024-02-28T22:20:55.274ZLast Flush: 1014 1001 1014
204982024-02-28T22:20:55.274ZDownstairs last five completed: 1001 1000
204992024-02-28T22:20:55.274ZUpstairs last five completed: 30 29 28 27 26
205002024-02-28T22:20:55.274Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:54.125659585Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1895}
205012024-02-28T22:20:55.274ZWrote out file "/tmp/test_repair_verify.data"
205022024-02-28T22:20:55.274Z
205032024-02-28T22:20:55.274ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205042024-02-28T22:20:55.294ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205052024-02-28T22:20:55.294Z 0 000-019 189 189 189 347 347 347 F F F
205062024-02-28T22:20:55.294Z 1 020-039 189 189 189 347 347 347 T F T <---
205072024-02-28T22:20:55.294Z 2 040-059 189 189 189 347 347 347 F F F
205082024-02-28T22:20:55.294Z 3 060-079 190 190 190 348 348 348 F F F
205092024-02-28T22:20:55.294Z 4 080-099 190 190 190 348 348 348 F F F
205102024-02-28T22:20:55.294Z 5 100-119 189 189 189 347 347 347 F F F
205112024-02-28T22:20:55.295Z 6 120-139 189 189 189 347 347 347 F F F
205122024-02-28T22:20:55.295Z 7 140-159 185 185 185 341 341 341 F F F
205132024-02-28T22:20:55.295Z 8 160-179 178 178 178 329 329 329 F F F
205142024-02-28T22:20:55.295Z 9 180-199 182 182 182 335 335 335 F F F
205152024-02-28T22:20:55.295Z 10 200-219 188 188 188 346 346 346 F F F
205162024-02-28T22:20:55.295Z 11 220-239 189 189 189 347 347 347 F F F
205172024-02-28T22:20:55.296Z 12 240-259 189 189 189 347 347 347 F F F
205182024-02-28T22:20:55.296Z 13 260-279 190 183 190 349 338 349 F T F <---
205192024-02-28T22:20:55.296Z 14 280-299 190 182 190 349 335 349 F F F <---
205202024-02-28T22:20:55.296Z 15 300-319 190 182 190 349 335 349 F T F <---
205212024-02-28T22:20:55.296Z 16 320-339 186 186 186 343 343 343 T F T <---
205222024-02-28T22:20:55.296Z 17 340-359 188 188 188 346 346 346 T F T <---
205232024-02-28T22:20:55.297Z 18 360-379 190 186 190 349 342 349 F T F <---
205242024-02-28T22:20:55.297Z 19 380-399 186 186 186 343 343 343 F F F
205252024-02-28T22:20:55.297Z 20 400-419 190 182 190 349 336 349 F T F <---
205262024-02-28T22:20:55.297Z 21 420-439 188 188 188 346 346 346 F F F
205272024-02-28T22:20:55.297Z 22 440-459 178 178 178 328 328 328 T F T <---
205282024-02-28T22:20:55.297Z 23 460-479 188 188 188 346 346 346 F F F
205292024-02-28T22:20:55.297Z 24 480-499 186 186 186 343 343 343 F F F
205302024-02-28T22:20:55.297Z 25 500-519 186 186 186 343 343 343 T F T <---
205312024-02-28T22:20:55.297Z 26 520-539 184 184 184 339 339 339 T F T <---
205322024-02-28T22:20:55.298Z 27 540-559 175 175 175 324 324 324 T F T <---
205332024-02-28T22:20:55.298Z 28 560-579 188 188 188 345 345 345 T F T <---
205342024-02-28T22:20:55.298Z 29 580-599 186 186 186 343 343 343 F F F
205352024-02-28T22:20:55.298ZMax gen: 190, Max flush: 349
205362024-02-28T22:20:55.298ZError: Difference in extent metadata found!
205372024-02-28T22:20:55.298ZA Difference in extent metadata is expected here
205382024-02-28T22:20:55.298ZOn loop 95
205392024-02-28T22:20:55.298Z
205402024-02-28T22:20:55.298ZVerifying data now
205412024-02-28T22:21:00.952ZNo change after verify
205422024-02-28T22:21:00.952ZLoop: 95 Downstairs dump after verify (and repair):
205432024-02-28T22:21:00.968ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205442024-02-28T22:21:00.968Z 0 000-019 189 189 189 347 347 347 F F F
205452024-02-28T22:21:00.968Z 1 020-039 191 191 191 350 350 350 F F F
205462024-02-28T22:21:00.968Z 2 040-059 189 189 189 347 347 347 F F F
205472024-02-28T22:21:00.968Z 3 060-079 190 190 190 348 348 348 F F F
205482024-02-28T22:21:00.968Z 4 080-099 190 190 190 348 348 348 F F F
205492024-02-28T22:21:00.975Z 5 100-119 189 189 189 347 347 347 F F F
205502024-02-28T22:21:00.975Z 6 120-139 189 189 189 347 347 347 F F F
205512024-02-28T22:21:00.975Z 7 140-159 185 185 185 341 341 341 F F F
205522024-02-28T22:21:00.975Z 8 160-179 178 178 178 329 329 329 F F F
205532024-02-28T22:21:00.975Z 9 180-199 182 182 182 335 335 335 F F F
205542024-02-28T22:21:00.975Z 10 200-219 188 188 188 346 346 346 F F F
205552024-02-28T22:21:00.975Z 11 220-239 189 189 189 347 347 347 F F F
205562024-02-28T22:21:00.975Z 12 240-259 189 189 189 347 347 347 F F F
205572024-02-28T22:21:00.975Z 13 260-279 190 190 190 349 349 349 F F F
205582024-02-28T22:21:00.975Z 14 280-299 190 190 190 349 349 349 F F F
205592024-02-28T22:21:00.975Z 15 300-319 190 190 190 349 349 349 F F F
205602024-02-28T22:21:00.975Z 16 320-339 191 191 191 350 350 350 F F F
205612024-02-28T22:21:00.975Z 17 340-359 191 191 191 350 350 350 F F F
205622024-02-28T22:21:00.976Z 18 360-379 190 190 190 349 349 349 F F F
205632024-02-28T22:21:00.976Z 19 380-399 186 186 186 343 343 343 F F F
205642024-02-28T22:21:00.976Z 20 400-419 190 190 190 349 349 349 F F F
205652024-02-28T22:21:00.976Z 21 420-439 188 188 188 346 346 346 F F F
205662024-02-28T22:21:00.976Z 22 440-459 191 191 191 350 350 350 F F F
205672024-02-28T22:21:00.976Z 23 460-479 188 188 188 346 346 346 F F F
205682024-02-28T22:21:00.976Z 24 480-499 186 186 186 343 343 343 F F F
205692024-02-28T22:21:00.976Z 25 500-519 191 191 191 350 350 350 F F F
205702024-02-28T22:21:00.976Z 26 520-539 191 191 191 350 350 350 F F F
205712024-02-28T22:21:00.976Z 27 540-559 191 191 191 350 350 350 F F F
205722024-02-28T22:21:00.976Z 28 560-579 191 191 191 350 350 350 F F F
205732024-02-28T22:21:00.976Z 29 580-599 186 186 186 343 343 343 F F F
205742024-02-28T22:21:00.976ZMax gen: 191, Max flush: 350
205752024-02-28T22:21:00.976Z
205762024-02-28T22:21:00.976ZBegin loop 96
205772024-02-28T22:21:00.976ZDownstairs to restart: 0
205782024-02-28T22:21:00.982Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.859894983Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
205792024-02-28T22:21:00.983Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
205802024-02-28T22:21:00.983Z,"v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860266642Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
205812024-02-28T22:21:00.989Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860299292Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
205822024-02-28T22:21:00.989Z{"msg":"Crucible b588abe6-7f2a-4e5b-8515-70eb6e95188e has session id: 1a11c03f-0362-4156-8a6c-306bef37b6d0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860335441Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205832024-02-28T22:21:00.989Z{"msg":"Upstairs opts: Upstairs UUID: b588abe6-7f2a-4e5b-8515-70eb6e95188e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860388759Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205842024-02-28T22:21:00.989Z{"msg":"Crucible stats registered with UUID: b588abe6-7f2a-4e5b-8515-70eb6e95188e","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860417868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205852024-02-28T22:21:00.989Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860456487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
205862024-02-28T22:21:00.989Z{"msg":"b588abe6-7f2a-4e5b-8515-70eb6e95188e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860474707Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205872024-02-28T22:21:00.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860516566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205882024-02-28T22:21:00.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860546205Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205892024-02-28T22:21:00.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860575264Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205902024-02-28T22:21:00.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860614353Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205912024-02-28T22:21:00.989Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860643852Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205922024-02-28T22:21:00.989Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860682291Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205932024-02-28T22:21:00.990Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.86070397Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205942024-02-28T22:21:00.990Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860756399Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205952024-02-28T22:21:00.990Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860788688Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205962024-02-28T22:21:00.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860814198Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205972024-02-28T22:21:00.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.860890985Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205982024-02-28T22:21:00.990Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861007422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
205992024-02-28T22:21:00.990Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861561267Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206002024-02-28T22:21:00.990Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861598776Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206012024-02-28T22:21:00.990Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861615376Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206022024-02-28T22:21:00.990Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861654664Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206032024-02-28T22:21:00.990Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861685174Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206042024-02-28T22:21:00.990Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.861720422Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206052024-02-28T22:21:00.990Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862155881Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206062024-02-28T22:21:00.990Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862201489Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206072024-02-28T22:21:00.990Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862380394Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206082024-02-28T22:21:00.990Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862700126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206092024-02-28T22:21:00.990Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862753344Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206102024-02-28T22:21:00.990Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862798913Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206112024-02-28T22:21:00.990Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862829372Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206122024-02-28T22:21:00.990Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862872711Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206132024-02-28T22:21:00.990Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.86290229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206142024-02-28T22:21:00.990Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862934019Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206152024-02-28T22:21:00.991Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.862968318Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206162024-02-28T22:21:00.991Z{"msg":"[0]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863004527Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206172024-02-28T22:21:00.991Z{"msg":"[0]R generation[0..12]: [189, 191, 189, 190, 190, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863029186Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206182024-02-28T22:21:00.991Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863064745Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206192024-02-28T22:21:00.991Z{"msg":"[1]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863102155Z","hostname":"Wait for a query_work_queue command to finish before sending IO
206202024-02-28T22:21:00.991Zip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206212024-02-28T22:21:00.991Z{"msg":"[1]R generation[0..12]: [189, 191, 189, 190, 190, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863145583Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206222024-02-28T22:21:00.991Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863185792Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206232024-02-28T22:21:00.991Z{"msg":"[2]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 347, 347, 341, 329, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863225901Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206242024-02-28T22:21:00.991Z{"msg":"[2]R generation[0..12]: [189, 191, 189, 190, 190, 189, 189, 185, 178, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.86326368Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206252024-02-28T22:21:00.991Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863300449Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206262024-02-28T22:21:00.991Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863336948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206272024-02-28T22:21:00.991Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863375447Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206282024-02-28T22:21:00.991Z{"msg":"Next flush: 351","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863410946Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206292024-02-28T22:21:00.991Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863445945Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"":"downstairs","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206302024-02-28T22:21:00.991Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863483784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206312024-02-28T22:21:00.991Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863518503Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"0","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206322024-02-28T22:21:00.991ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206332024-02-28T22:21:00.991Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863558162Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"1","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206342024-02-28T22:21:00.991Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863593161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"client":"2","":"downstairs","session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206352024-02-28T22:21:00.991Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.86361708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206362024-02-28T22:21:00.991Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.86363132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206372024-02-28T22:21:00.991Z{"msg":"b588abe6-7f2a-4e5b-8515-70eb6e95188e is now active with session: 1a11c03f-0362-4156-8a6c-306bef37b6d0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863651869Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206382024-02-28T22:21:00.991Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863664779Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903,"session_id":"1a11c03f-0362-4156-8a6c-306bef37b6d0"}
206392024-02-28T22:21:00.992Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:20:59.863684348Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
206402024-02-28T22:21:00.992ZLoading write count information from file "/tmp/test_repair_verify.data"
206412024-02-28T22:21:00.992ZRun Repair workload
206422024-02-28T22:21:00.996Z01/30 Write block 419 len 12288 data: 15 17 19
206432024-02-28T22:21:00.996Z02/30 Read block 336 len 36864
206442024-02-28T22:21:00.996Z03/30 Read block 324 len 36864
206452024-02-28T22:21:00.996Z04/30 Read block 143 len 12288
206462024-02-28T22:21:00.996Z05/30 Read block 256 len 4096
206472024-02-28T22:21:00.996Z06/30 Write block 125 len 12288 data: 16 19 23
206482024-02-28T22:21:00.996Z07/30 Write block 110 len 12288 data: 19 18 18
206492024-02-28T22:21:00.996Z08/30 Write block 442 len 40960 data: 17 14 16 16 17 17 18 18 20 20
206502024-02-28T22:21:00.996Z09/30 Write block 309 len 36864 data: 16 13 16 16 16 16 12 12 10
206512024-02-28T22:21:00.996Z10/30 Read block 210 len 16384
206522024-02-28T22:21:01.138Z11/30 Write block 502 len 24576 data: 17 17 18 16 19 16
206532024-02-28T22:21:01.139Z12/30 Read block 78 len 12288
206542024-02-28T22:21:01.139Z13/30 Write block 476 len 32768 data: 15 12 9 8 9 8 12 13
206552024-02-28T22:21:01.139Z14/30 Write block 471 len 8192 data: 18 19
206562024-02-28T22:21:01.139Z15/30 Flush
206572024-02-28T22:21:01.139Z16/30 Write block 160 len 16384 data: 7 9 10 11
206582024-02-28T22:21:01.139Z17/30 Read block 148 len 12288
206592024-02-28T22:21:01.139Z18/30 Read block 188 len 16384
206602024-02-28T22:21:01.139Z19/30 Read block 472 len 24576
206612024-02-28T22:21:01.139Z20/30 Write block 538 len 4096 data: 11
206622024-02-28T22:21:01.139Z21/30 Read block 584 len 40960
206632024-02-28T22:21:01.139Z22/30 Write block 275 len 20480 data: 7 8 9 10 11
206642024-02-28T22:21:01.139Z23/30 Read block 124 len 16384
206652024-02-28T22:21:01.139Z24/30 Read block 474 len 8192
206662024-02-28T22:21:01.139Z25/30 Read block 294 len 24576
206672024-02-28T22:21:01.140Z26/30 Read block 202 len 32768
206682024-02-28T22:21:01.140Z27/30 Read block 71 len 8192
206692024-02-28T22:21:01.140Z28/30 Read block 512 len 32768
206702024-02-28T22:21:01.140Z29/30 Read block 467 len 40960
206712024-02-28T22:21:01.140Z30/30 Read block 447 len 8192
206722024-02-28T22:21:01.140Z----------------------------------------------------------------
206732024-02-28T22:21:01.140Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
206742024-02-28T22:21:01.140ZStates: Active Active Active
206752024-02-28T22:21:01.140ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206762024-02-28T22:21:01.141Z 1 Acked 1000 Write 3 Done Done Done false
206772024-02-28T22:21:01.141Z 2 Acked 1001 Read 9 Done Done Done false
206782024-02-28T22:21:01.141Z 3 Acked 1002 Read 9 Sent Done Done false
206792024-02-28T22:21:01.141Z 4 Acked 1003 Read 3 Sent Done Done false
206802024-02-28T22:21:01.141Z 5 Acked 1004 Read 1 Sent Done Done false
206812024-02-28T22:21:01.141Z 6 Acked 1005 Write 3 Sent Done Done false
206822024-02-28T22:21:01.141Z 7 Acked 1006 Write 3 Sent Done Done false
206832024-02-28T22:21:01.141Z 8 Acked 1007 Write 10 Sent Done Done false
206842024-02-28T22:21:01.141Z 9 Acked 1008 Write 9 Sent Done Done false
206852024-02-28T22:21:01.141Z 10 Acked 1009 Read 4 Sent Done Done false
206862024-02-28T22:21:01.141Z 11 Acked 1010 Write 6 Sent Done Done false
206872024-02-28T22:21:01.141Z 12 Acked 1011 Read 3 Sent Done Done false
206882024-02-28T22:21:01.141Z 13 Acked 1012 Write 8 Sent Done Done false
206892024-02-28T22:21:01.141Z 14 Acked 1013 Write 2 Sent Done Done false
206902024-02-28T22:21:01.141Z 15 Acked 1014 Flush 0 Sent Done Done false
206912024-02-28T22:21:01.141Z 16 Acked 1015 Write 4 Sent Done Done false
206922024-02-28T22:21:01.141Z 17 Acked 1016 Read 3 Sent Done Done false
206932024-02-28T22:21:01.142Z 18 Acked 1017 Read 4 Sent Done Done false
206942024-02-28T22:21:01.142Z 19 Acked 1018 Read 6 Sent Done Done false
206952024-02-28T22:21:01.142Z 20 Acked 1019 Write 1 Sent Done Done false
206962024-02-28T22:21:01.142Z 21 Acked 1020 Read 10 Sent Done Done false
206972024-02-28T22:21:01.142Z 22 Acked 1021 Write 5 Sent Done Done false
206982024-02-28T22:21:01.142Z 23 Acked 1022 Read 4 Sent Done Done false
206992024-02-28T22:21:01.142Z 24 Acked 1023 Read 2 Sent Done Done false
207002024-02-28T22:21:01.142Z 25 Acked 1024 Read 6 Sent Done Done false
207012024-02-28T22:21:01.142Z 26 Acked 1025 Read 8 Sent Done Done false
207022024-02-28T22:21:01.142Z 27 Acked 1026 Read 2 Sent Done Done false
207032024-02-28T22:21:01.142Z 28 Acked 1027 Read 8 Sent Done Done false
207042024-02-28T22:21:01.142Z 29 Acked 1028 Read 10 Sent Done Done false
207052024-02-28T22:21:01.142Z 30 Acked 1029 Read 2 Sent Done Done false
207062024-02-28T22:21:01.142Z STATES DS:0 DS:1 DS:2 TOTAL
207072024-02-28T22:21:01.142Z New 0 0 0 0
207082024-02-28T22:21:01.142Z Sent 28 0 0 28
207092024-02-28T22:21:01.142Z Done 2 30 30 62
207102024-02-28T22:21:01.142Z Skipped 0 0 0 0
207112024-02-28T22:21:01.142Z Error 0 0 0 0
207122024-02-28T22:21:01.142ZLast Flush: 0 1014 1014
207132024-02-28T22:21:01.142ZDownstairs last five completed:
207142024-02-28T22:21:01.143ZUpstairs last five completed: 30 29 28 27 26
207152024-02-28T22:21:01.143Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:20:59.893111698Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1903}
207162024-02-28T22:21:01.143ZWrote out file "/tmp/test_repair_verify.data"
207172024-02-28T22:21:01.144Z
207182024-02-28T22:21:01.144ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207192024-02-28T22:21:01.144ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207202024-02-28T22:21:01.144Z 0 000-019 189 189 189 347 347 347 F F F
207212024-02-28T22:21:01.144Z 1 020-039 191 191 191 350 350 350 F F F
207222024-02-28T22:21:01.145Z 2 040-059 189 189 189 347 347 347 F F F
207232024-02-28T22:21:01.145Z 3 060-079 190 190 190 348 348 348 F F F
207242024-02-28T22:21:01.145Z 4 080-099 190 190 190 348 348 348 F F F
207252024-02-28T22:21:01.145Z 5 100-119 189 192 192 347 351 351 F F F <---
207262024-02-28T22:21:01.145Z 6 120-139 189 192 192 347 351 351 F F F <---
207272024-02-28T22:21:01.145Z 7 140-159 185 185 185 341 341 341 F F F
207282024-02-28T22:21:01.145Z 8 160-179 178 178 178 329 329 329 F T T <---
207292024-02-28T22:21:01.145Z 9 180-199 182 182 182 335 335 335 F F F
207302024-02-28T22:21:01.145Z 10 200-219 188 188 188 346 346 346 F F F
207312024-02-28T22:21:01.145Z 11 220-239 189 189 189 347 347 347 F F F
207322024-02-28T22:21:01.145Z 12 240-259 189 189 189 347 347 347 F F F
207332024-02-28T22:21:01.145Z 13 260-279 190 190 190 349 349 349 F T T <---
207342024-02-28T22:21:01.146Z 14 280-299 190 190 190 349 349 349 F F F
207352024-02-28T22:21:01.146Z 15 300-319 190 192 192 349 351 351 F F F <---
207362024-02-28T22:21:01.146Z 16 320-339 191 191 191 350 350 350 F F F
207372024-02-28T22:21:01.147Z 17 340-359 191 191 191 350 350 350 F F F
207382024-02-28T22:21:01.147Z 18 360-379 190 190 190 349 349 349 F F F
207392024-02-28T22:21:01.147Z 19 380-399 186 186 186 343 343 343 F F F
207402024-02-28T22:21:01.147Z 20 400-419 190 192 192 349 351 351 T F F <---
207412024-02-28T22:21:01.147Z 21 420-439 188 192 192 346 351 351 T F F <---
207422024-02-28T22:21:01.147Z 22 440-459 191 192 192 350 351 351 F F F <---
207432024-02-28T22:21:01.147Z 23 460-479 188 192 192 346 351 351 F F F <---
207442024-02-28T22:21:01.147Z 24 480-499 186 192 192 343 351 351 F F F <---
207452024-02-28T22:21:01.147Z 25 500-519 191 192 192 350 351 351 F F F <---
207462024-02-28T22:21:01.147Z 26 520-539 191 191 191 350 350 350 F T T <---
207472024-02-28T22:21:01.147Z 27 540-559 191 191 191 350 350 350 F F F
207482024-02-28T22:21:01.148Z 28 560-579 191 191 191 350 350 350 F F F
207492024-02-28T22:21:01.148Z 29 580-599 186 186 186 343 343 343 F F F
207502024-02-28T22:21:01.148ZMax gen: 192, Max flush: 351
207512024-02-28T22:21:01.148ZError: Difference in extent metadata found!
207522024-02-28T22:21:01.591ZA Difference in extent metadata is expected here
207532024-02-28T22:21:01.592ZOn loop 96
207542024-02-28T22:21:01.592Z
207552024-02-28T22:21:01.592ZVerifying data now
207562024-02-28T22:21:06.659ZNo change after verify
207572024-02-28T22:21:06.660ZLoop: 96 Downstairs dump after verify (and repair):
207582024-02-28T22:21:06.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207592024-02-28T22:21:06.674Z 0 000-019 189 189 189 347 347 347 F F F
207602024-02-28T22:21:06.674Z 1 020-039 191 191 191 350 350 350 F F F
207612024-02-28T22:21:06.674Z 2 040-059 189 189 189 347 347 347 F F F
207622024-02-28T22:21:06.674Z 3 060-079 190 190 190 348 348 348 F F F
207632024-02-28T22:21:06.674Z 4 080-099 190 190 190 348 348 348 F F F
207642024-02-28T22:21:06.675Z 5 100-119 192 192 192 351 351 351 F F F
207652024-02-28T22:21:06.675Z 6 120-139 192 192 192 351 351 351 F F F
207662024-02-28T22:21:06.675Z 7 140-159 185 185 185 341 341 341 F F F
207672024-02-28T22:21:06.675Z 8 160-179 193 193 193 352 352 352 F F F
207682024-02-28T22:21:06.675Z 9 180-199 182 182 182 335 335 335 F F F
207692024-02-28T22:21:06.675Z 10 200-219 188 188 188 346 346 346 F F F
207702024-02-28T22:21:06.680Z 11 220-239 189 189 189 347 347 347 F F F
207712024-02-28T22:21:06.680Z 12 240-259 189 189 189 347 347 347 F F F
207722024-02-28T22:21:06.680Z 13 260-279 193 193 193 352 352 352 F F F
207732024-02-28T22:21:06.680Z 14 280-299 190 190 190 349 349 349 F F F
207742024-02-28T22:21:06.681Z 15 300-319 192 192 192 351 351 351 F F F
207752024-02-28T22:21:06.681Z 16 320-339 191 191 191 350 350 350 F F F
207762024-02-28T22:21:06.681Z 17 340-359 191 191 191 350 350 350 F F F
207772024-02-28T22:21:06.681Z 18 360-379 190 190 190 349 349 349 F F F
207782024-02-28T22:21:06.681Z 19 380-399 186 186 186 343 343 343 F F F
207792024-02-28T22:21:06.681Z 20 400-419 192 192 192 351 351 351 F F F
207802024-02-28T22:21:06.682Z 21 420-439 192 192 192 351 351 351 F F F
207812024-02-28T22:21:06.682Z 22 440-459 192 192 192 351 351 351 F F F
207822024-02-28T22:21:06.682Z 23 460-479 192 192 192 351 351 351 F F F
207832024-02-28T22:21:06.682Z 24 480-499 192 192 192 351 351 351 F F F
207842024-02-28T22:21:06.682Z 25 500-519 192 192 192 351 351 351 F F F
207852024-02-28T22:21:06.682Z 26 520-539 193 193 193 352 352 352 F F F
207862024-02-28T22:21:06.683Z 27 540-559 191 191 191 350 350 350 F F F
207872024-02-28T22:21:06.683Z 28 560-579 191 191 191 350 350 350 F F F
207882024-02-28T22:21:06.683Z 29 580-599 186 186 186 343 343 343 F F F
207892024-02-28T22:21:06.683ZMax gen: 193, Max flush: 352
207902024-02-28T22:21:06.683Z
207912024-02-28T22:21:06.683ZBegin loop 97
207922024-02-28T22:21:06.683ZDownstairs to restart: 2
207932024-02-28T22:21:06.697Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573118689Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
207942024-02-28T22:21:06.697Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573386612Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
207952024-02-28T22:21:06.697Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573404421Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
207962024-02-28T22:21:06.698Z{"msg":"Crucible c289eea3-0146-4074-bf82-eccd48c3bc14 has session id: 6f28a6a5-4bbd-4b71-998c-4e0202f36af7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573426241Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
207972024-02-28T22:21:06.704Z{"msg":"Upstairs opts: Upstairs UUID: c289eea3-0146-4074-bf82-eccd48c3bc14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573475829Z","Crucible runtime is spawned
207982024-02-28T22:21:06.704Zhostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
207992024-02-28T22:21:06.704Z{"msg":"Crucible stats registered with UUID: c289eea3-0146-4074-bf82-eccd48c3bc14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573522718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208002024-02-28T22:21:06.704Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573577126Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
208012024-02-28T22:21:06.704Z{"msg":"c289eea3-0146-4074-bf82-eccd48c3bc14 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573690643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208022024-02-28T22:21:06.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573731602Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208032024-02-28T22:21:06.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573769681Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208042024-02-28T22:21:06.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.57379661Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208052024-02-28T22:21:06.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.57382955Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208062024-02-28T22:21:06.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573871058Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208072024-02-28T22:21:06.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573898217Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208082024-02-28T22:21:06.706Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573930827Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208092024-02-28T22:21:06.706Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573969566Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208102024-02-28T22:21:06.706Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.573996715Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208112024-02-28T22:21:06.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574140051Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208122024-02-28T22:21:06.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574209369Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208132024-02-28T22:21:06.706Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574261167Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208142024-02-28T22:21:06.706Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574853571Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208152024-02-28T22:21:06.706Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574917429Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208162024-02-28T22:21:06.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.574999417Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208172024-02-28T22:21:06.707Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575034926Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208182024-02-28T22:21:06.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575127484Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208192024-02-28T22:21:06.707Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575156923Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208202024-02-28T22:21:06.707Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575583691Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208212024-02-28T22:21:06.707Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575655729Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208222024-02-28T22:21:06.707Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.575882513Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208232024-02-28T22:21:06.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576106097Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208242024-02-28T22:21:06.708Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576152806Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208252024-02-28T22:21:06.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576174655Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208262024-02-28T22:21:06.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576212634Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208272024-02-28T22:21:06.708Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576236303Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208282024-02-28T22:21:06.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576250673Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208292024-02-28T22:21:06.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.57635949Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208302024-02-28T22:21:06.709Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576397649Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208312024-02-28T22:21:06.709Z{"msg":"[0]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 351, 351, 341, 352, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576446127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208322024-02-28T22:21:06.709Z{"msg":"[0]R generation[0..12]: [189, 191, 189, 190, 190, 192, 192, 185, 193, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576480717Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208332024-02-28T22:21:06.709Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576499246Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208342024-02-28T22:21:06.709Z{"msg":"[1]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 351, 351, 341, 352, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576542265Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208352024-02-28T22:21:06.709Z{"msg":"[1]R generation[0..12]: [189, 191, 189, 190, 190, 192, 192, 185, 193, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576558104Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208362024-02-28T22:21:06.709Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
208372024-02-28T22:21:06.710Z,"time":"2024-02-28T22:21:05.576581154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208382024-02-28T22:21:06.710Z{"msg":"[2]R flush_numbers[0..12]: [347, 350, 347, 348, 348, 351, 351, 341, 352, 335, 346, 347]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576606123Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208392024-02-28T22:21:06.710Z{"msg":"[2]R generation[0..12]: [189, 191, 189, 190, 190, 192, 192, 185, 193, 182, 188, 189]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576621482Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208402024-02-28T22:21:06.710Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576642872Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208412024-02-28T22:21:06.710Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576657121Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208422024-02-28T22:21:06.710Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576678751Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208432024-02-28T22:21:06.710Z{"msg":"Next flush: 353","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576692361Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208442024-02-28T22:21:06.710Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.57671367Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"":"downstairs","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208452024-02-28T22:21:06.710Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.57672795Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208462024-02-28T22:21:06.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576740919Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"0","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208472024-02-28T22:21:06.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576754469Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"1","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208482024-02-28T22:21:06.711Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576767738Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"client":"2","":"downstairs","session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208492024-02-28T22:21:06.711Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576780948Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208502024-02-28T22:21:06.711Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576793868Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208512024-02-28T22:21:06.711Z{"msg":"c289eea3-0146-4074-bf82-eccd48c3bc14 is now active with session: 6f28a6a5-4bbd-4b71-998c-4e0202f36af7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576814337Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208522024-02-28T22:21:06.711Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576835087Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911,"session_id":"6f28a6a5-4bbd-4b71-998c-4e0202f36af7"}
208532024-02-28T22:21:06.711Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:05.576847966Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
208542024-02-28T22:21:06.711ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208552024-02-28T22:21:06.711ZLoading write count information from file "/tmp/test_repair_verify.data"
208562024-02-28T22:21:06.712ZRun Repair workload
208572024-02-28T22:21:06.712Z01/30 Write block 570 len 40960 data: 15 14 16 18 18 18 18 18 18 16
208582024-02-28T22:21:06.717Z02/30 Read block 348 len 12288
208592024-02-28T22:21:06.717Z03/30 Read block 322 len 12288
208602024-02-28T22:21:06.718Z04/30 Read block 257 len 32768
208612024-02-28T22:21:06.718Z05/30 Write block 486 len 4096 data: 14
208622024-02-28T22:21:06.718Z06/30 Read block 296 len 4096
208632024-02-28T22:21:06.718Z07/30 Write block 262 len 16384 data: 21 21 21 21
208642024-02-28T22:21:06.718Z08/30 Read block 249 len 20480
208652024-02-28T22:21:06.718Z09/30 Read block 23 len 36864
208662024-02-28T22:21:06.718Z10/30 Read block 142 len 16384
208672024-02-28T22:21:06.718Z11/30 Read block 289 len 40960
208682024-02-28T22:21:06.718Z12/30 Read block 425 len 36864
208692024-02-28T22:21:06.718Z13/30 Read block 229 len 32768
208702024-02-28T22:21:06.718Z14/30 Write block 434 len 40960 data: 20 21 24 23 23 25 21 16 18 15
208712024-02-28T22:21:06.719Z15/30 Read block 32 len 8192
208722024-02-28T22:21:06.719Z16/30 Write block 218 len 24576 data: 10 10 11 12 12 10
208732024-02-28T22:21:06.719Z17/30 Write block 82 len 24576 data: 16 14 13 16 15 15
208742024-02-28T22:21:06.724Z18/30 Write block 163 len 24576 data: 12 12 14 14 18 19
208752024-02-28T22:21:06.725Z19/30 Read block 560 len 20480
208762024-02-28T22:21:06.725Z20/30 Write block 322 len 36864 data: 13 14 17 19 20 20 24 23 23
208772024-02-28T22:21:06.725Z21/30 Write block 358 len 36864 data: 15 11 10 9 12 13 9 11 13
208782024-02-28T22:21:06.725Z22/30 Flush
208792024-02-28T22:21:06.730Z23/30 Write block 93 len 16384 data: 14 13 12 14
208802024-02-28T22:21:06.730Z24/30 Read block 464 len 20480
208812024-02-28T22:21:06.730Z25/30 Write block 456 len 16384 data: 17 15 15 12
208822024-02-28T22:21:06.731Z26/30 Write block 229 len 4096 data: 18
208832024-02-28T22:21:06.731Z27/30 Read block 172 len 12288
208842024-02-28T22:21:06.731Z28/30 Read block 48 len 40960
208852024-02-28T22:21:06.736Z29/30 Read block 148 len 8192
208862024-02-28T22:21:06.736Z30/30 Write block 438 len 16384 data: 24 26 22 17
208872024-02-28T22:21:06.737Z----------------------------------------------------------------
208882024-02-28T22:21:06.737Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
208892024-02-28T22:21:06.737ZStates: Active Active Active
208902024-02-28T22:21:06.737ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208912024-02-28T22:21:06.737Z 1 Acked 1000 Write 10 Done Done Done false
208922024-02-28T22:21:06.737Z 2 Acked 1001 Read 3 Done Done Done false
208932024-02-28T22:21:06.737Z 3 Acked 1002 Read 3 Done Done Done false
208942024-02-28T22:21:06.737Z 4 Acked 1003 Read 8 Done Done Done false
208952024-02-28T22:21:06.737Z 5 Acked 1004 Write 1 Done Done Sent false
208962024-02-28T22:21:06.738Z 6 Acked 1005 Read 1 Done Done Sent false
208972024-02-28T22:21:06.738Z 7 Acked 1006 Write 4 Done Done Sent false
208982024-02-28T22:21:06.738Z 8 Acked 1007 Read 5 Done Done Sent false
208992024-02-28T22:21:06.738Z 9 Acked 1008 Read 9 Done Done Sent false
209002024-02-28T22:21:06.738Z 10 Acked 1009 Read 4 Done Done Sent false
209012024-02-28T22:21:06.739Z 11 Acked 1010 Read 10 Done Done Sent false
209022024-02-28T22:21:06.739Z 12 Acked 1011 Read 9 Done Done Sent false
209032024-02-28T22:21:06.739Z 13 Acked 1012 Read 8 Done Done Sent false
209042024-02-28T22:21:06.739Z 14 Acked 1013 Write 10 Done Done Sent false
209052024-02-28T22:21:06.739Z 15 Acked 1014 Read 2 Done Done Sent false
209062024-02-28T22:21:06.739Z 16 Acked 1015 Write 6 Done Done Sent false
209072024-02-28T22:21:06.739Z 17 Acked 1016 Write 6 Done Done Sent false
209082024-02-28T22:21:06.739Z 18 Acked 1017 Write 6 Done Done Sent false
209092024-02-28T22:21:06.739Z 19 Acked 1018 Read 5 Done Done Sent false
209102024-02-28T22:21:06.740Z 20 Acked 1019 Write 9 Done Done Sent false
209112024-02-28T22:21:06.740Z 21 Acked 1020 Write 9 Done Done Sent false
209122024-02-28T22:21:06.740Z 22 Acked 1021 Flush 0 Done Done Sent false
209132024-02-28T22:21:06.740Z 23 Acked 1022 Write 4 Done Done Sent false
209142024-02-28T22:21:06.740Z 24 Acked 1023 Read 5 Done Done Sent false
209152024-02-28T22:21:06.740Z 25 Acked 1024 Write 4 Done Done Sent false
209162024-02-28T22:21:06.740Z 26 Acked 1025 Write 1 Done Done Sent false
209172024-02-28T22:21:06.740Z 27 Acked 1026 Read 3 Done Done Sent false
209182024-02-28T22:21:06.740Z 28 Acked 1027 Read 10 Done Done Sent false
209192024-02-28T22:21:06.741Z 29 Acked 1028 Read 2 Done Done Sent false
209202024-02-28T22:21:06.741Z 30 Acked 1029 Write 4 Sent Sent Sent false
209212024-02-28T22:21:06.741Z STATES DS:0 DS:1 DS:2 TOTAL
209222024-02-28T22:21:06.741Z New 0 0 0 0
209232024-02-28T22:21:06.741Z Sent 1 1 26 28
209242024-02-28T22:21:06.741Z Done 29 29 4 62
209252024-02-28T22:21:06.741Z Skipped 0 0 0 0
209262024-02-28T22:21:06.742Z Error 0 0 0 0
209272024-02-28T22:21:06.742ZLast Flush: 1021 1021 0
209282024-02-28T22:21:06.742ZDownstairs last five completed:
209292024-02-28T22:21:06.742ZUpstairs last five completed: 30 29 28 27 26
209302024-02-28T22:21:06.742Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:21:05.609476777Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1911}
209312024-02-28T22:21:06.742ZWrote out file "/tmp/test_repair_verify.data"
209322024-02-28T22:21:06.748Z
209332024-02-28T22:21:06.748ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209342024-02-28T22:21:06.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209352024-02-28T22:21:06.759Z 0 000-019 189 189 189 347 347 347 F F F
209362024-02-28T22:21:06.759Z 1 020-039 191 191 191 350 350 350 F F F
209372024-02-28T22:21:06.759Z 2 040-059 189 189 189 347 347 347 F F F
209382024-02-28T22:21:06.759Z 3 060-079 190 190 190 348 348 348 F F F
209392024-02-28T22:21:06.759Z 4 080-099 194 194 190 353 353 348 T T F <---
209402024-02-28T22:21:06.759Z 5 100-119 192 192 192 351 351 351 F F F
209412024-02-28T22:21:06.760Z 6 120-139 192 192 192 351 351 351 F F F
209422024-02-28T22:21:06.760Z 7 140-159 185 185 185 341 341 341 F F F
209432024-02-28T22:21:06.760Z 8 160-179 194 194 193 353 353 352 F F F <---
209442024-02-28T22:21:06.760Z 9 180-199 182 182 182 335 335 335 F F F
209452024-02-28T22:21:06.760Z 10 200-219 194 194 188 353 353 346 F F F <---
209462024-02-28T22:21:06.760Z 11 220-239 194 194 189 353 353 347 T T F <---
209472024-02-28T22:21:06.765Z 12 240-259 189 189 189 347 347 347 F F F
209482024-02-28T22:21:06.766Z 13 260-279 194 194 193 353 353 352 F F F <---
209492024-02-28T22:21:06.766Z 14 280-299 190 190 190 349 349 349 F F F
209502024-02-28T22:21:06.766Z 15 300-319 192 192 192 351 351 351 F F F
209512024-02-28T22:21:06.766Z 16 320-339 194 194 191 353 353 350 F F F <---
209522024-02-28T22:21:06.766Z 17 340-359 194 194 191 353 353 350 F F F <---
209532024-02-28T22:21:06.766Z 18 360-379 194 194 190 353 353 349 F F F <---
209542024-02-28T22:21:06.766Z 19 380-399 186 186 186 343 343 343 F F F
209552024-02-28T22:21:06.767Z 20 400-419 192 192 192 351 351 351 F F F
209562024-02-28T22:21:06.767Z 21 420-439 194 194 192 353 353 351 T T F <---
209572024-02-28T22:21:06.767Z 22 440-459 194 194 192 353 353 351 T T F <---
209582024-02-28T22:21:06.767Z 23 460-479 192 192 192 351 351 351 F F F
209592024-02-28T22:21:06.767Z 24 480-499 194 194 192 353 353 351 F F F <---
209602024-02-28T22:21:06.767Z 25 500-519 192 192 192 351 351 351 F F F
209612024-02-28T22:21:06.767Z 26 520-539 193 193 193 352 352 352 F F F
209622024-02-28T22:21:06.767Z 27 540-559 191 191 191 350 350 350 F F F
209632024-02-28T22:21:06.768Z 28 560-579 194 194 191 353 353 350 F F T <---
209642024-02-28T22:21:06.768Z 29 580-599 186 186 186 343 343 343 F F F
209652024-02-28T22:21:06.768ZMax gen: 194, Max flush: 353
209662024-02-28T22:21:06.768ZError: Difference in extent metadata found!
209672024-02-28T22:21:06.768ZA Difference in extent metadata is expected here
209682024-02-28T22:21:06.769ZOn loop 97
209692024-02-28T22:21:06.769Z
209702024-02-28T22:21:06.775ZVerifying data now
209712024-02-28T22:21:12.396ZNo change after verify
209722024-02-28T22:21:12.396ZLoop: 97 Downstairs dump after verify (and repair):
209732024-02-28T22:21:12.412ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209742024-02-28T22:21:12.412Z 0 000-019 189 189 189 347 347 347 F F F
209752024-02-28T22:21:12.412Z 1 020-039 191 191 191 350 350 350 F F F
209762024-02-28T22:21:12.412Z 2 040-059 189 189 189 347 347 347 F F F
209772024-02-28T22:21:12.412Z 3 060-079 190 190 190 348 348 348 F F F
209782024-02-28T22:21:12.412Z 4 080-099 195 195 195 354 354 354 F F F
209792024-02-28T22:21:12.428Z 5 100-119 192 192 192 351 351 351 F F F
209802024-02-28T22:21:12.428Z 6 120-139 192 192 192 351 351 351 F F F
209812024-02-28T22:21:12.428Z 7 140-159 185 185 185 341 341 341 F F F
209822024-02-28T22:21:12.429Z 8 160-179 194 194 194 353 353 353 F F F
209832024-02-28T22:21:12.429Z 9 180-199 182 182 182 335 335 335 F F F
209842024-02-28T22:21:12.429Z 10 200-219 194 194 194 353 353 353 F F F
209852024-02-28T22:21:12.429Z 11 220-239 195 195 195 354 354 354 F F F
209862024-02-28T22:21:12.429Z 12 240-259 189 189 189 347 347 347 F F F
209872024-02-28T22:21:12.429Z 13 260-279 194 194 194 353 353 353 F F F
209882024-02-28T22:21:12.429Z 14 280-299 190 190 190 349 349 349 F F F
209892024-02-28T22:21:12.429Z 15 300-319 192 192 192 351 351 351 F F F
209902024-02-28T22:21:12.430Z 16 320-339 194 194 194 353 353 353 F F F
209912024-02-28T22:21:12.430Z 17 340-359 194 194 194 353 353 353 F F F
209922024-02-28T22:21:12.430Z 18 360-379 194 194 194 353 353 353 F F F
209932024-02-28T22:21:12.430Z 19 380-399 186 186 186 343 343 343 F F F
209942024-02-28T22:21:12.430Z 20 400-419 192 192 192 351 351 351 F F F
209952024-02-28T22:21:12.431Z 21 420-439 195 195 195 354 354 354 F F F
209962024-02-28T22:21:12.431Z 22 440-459 195 195 195 354 354 354 F F F
209972024-02-28T22:21:12.431Z 23 460-479 192 192 192 351 351 351 F F F
209982024-02-28T22:21:12.431Z 24 480-499 194 194 194 353 353 353 F F F
209992024-02-28T22:21:12.431Z 25 500-519 192 192 192 351 351 351 F F F
210002024-02-28T22:21:12.432Z 26 520-539 193 193 193 352 352 352 F F F
210012024-02-28T22:21:12.432Z 27 540-559 191 191 191 350 350 350 F F F
210022024-02-28T22:21:12.432Z 28 560-579 194 194 194 353 353 353 F F F
210032024-02-28T22:21:12.432Z 29 580-599 186 186 186 343 343 343 F F F
210042024-02-28T22:21:12.432ZMax gen: 195, Max flush: 354
210052024-02-28T22:21:12.432Z
210062024-02-28T22:21:12.433ZBegin loop 98
210072024-02-28T22:21:12.433ZDownstairs to restart: 1
210082024-02-28T22:21:12.445Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304315861Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
210092024-02-28T22:21:12.445Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\nCrucible runtime is spawned
210102024-02-28T22:21:12.445Z git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304661812Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
210112024-02-28T22:21:12.446Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304695161Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
210122024-02-28T22:21:12.446Z{"msg":"Crucible 28985aec-eb7b-4dc8-9fe3-5ea9e3aa9638 has session id: 4ad7840d-7d53-4f10-8d74-9120dfc7e3c9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.30471052Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210132024-02-28T22:21:12.446Z{"msg":"Upstairs opts: Upstairs UUID: 28985aec-eb7b-4dc8-9fe3-5ea9e3aa9638, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304743289Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210142024-02-28T22:21:12.446Z{"msg":"Crucible stats registered with UUID: 28985aec-eb7b-4dc8-9fe3-5ea9e3aa9638","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304759229Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210152024-02-28T22:21:12.446Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304779218Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
210162024-02-28T22:21:12.446Z{"msg":"28985aec-eb7b-4dc8-9fe3-5ea9e3aa9638 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304791108Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210172024-02-28T22:21:12.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304804557Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210182024-02-28T22:21:12.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304820197Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210192024-02-28T22:21:12.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304834257Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210202024-02-28T22:21:12.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304855376Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210212024-02-28T22:21:12.447Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304870616Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210222024-02-28T22:21:12.447Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304895105Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210232024-02-28T22:21:12.447Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304928254Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210242024-02-28T22:21:12.447Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304949633Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210252024-02-28T22:21:12.447Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.304963773Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210262024-02-28T22:21:12.447Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305139068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210272024-02-28T22:21:12.447Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305177477Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210282024-02-28T22:21:12.447Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305298074Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210292024-02-28T22:21:12.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305875068Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210302024-02-28T22:21:12.448Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305912907Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210312024-02-28T22:21:12.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305930226Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210322024-02-28T22:21:12.448Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305944986Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210332024-02-28T22:21:12.448Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305967976Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210342024-02-28T22:21:12.448Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.305988645Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210352024-02-28T22:21:12.448Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.306429203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210362024-02-28T22:21:12.449Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.306468452Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210372024-02-28T22:21:12.449Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.306604518Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210382024-02-28T22:21:12.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307098654Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210392024-02-28T22:21:12.449Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307156143Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210402024-02-28T22:21:12.449Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307192282Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210412024-02-28T22:21:12.449Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307207051Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210422024-02-28T22:21:12.450Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307228561Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210432024-02-28T22:21:12.450Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.3072437Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210442024-02-28T22:21:12.450Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.30726335Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210452024-02-28T22:21:12.450Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.3072786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210462024-02-28T22:21:12.450Z{"msg":"[0]R flush_numbers[0..12]: [347, 350, 347, 348, 354, 351, 351, 341, 353, 335, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307299849Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210472024-02-28T22:21:12.450Z{"msg":"[0]R generation[0..12]: [189, 191, 189, 190, 195, 192, 192, 185, 194, 182, 194, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307314519Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210482024-02-28T22:21:12.450Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
210492024-02-28T22:21:12.451Z2024-02-28T22:21:11.307339818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210502024-02-28T22:21:12.451Z{"msg":"[1]R flush_numbers[0..12]: [347, 350, 347, 348, 354, 351, 351, 341, 353, 335, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307362487Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210512024-02-28T22:21:12.451Z{"msg":"[1]R generation[0..12]: [189, 191, 189, 190, 195, 192, 192, 185, 194, 182, 194, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307377127Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210522024-02-28T22:21:12.451Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307399786Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210532024-02-28T22:21:12.451Z{"msg":"[2]R flush_numbers[0..12]: [347, 350, 347, 348, 354, 351, 351, 341, 353, 335, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307414576Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210542024-02-28T22:21:12.451Z{"msg":"[2]R generation[0..12]: [189, 191, 189, 190, 195, 192, 192, 185, 194, 182, 194, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307435115Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210552024-02-28T22:21:12.451Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307449325Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210562024-02-28T22:21:12.452Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307469334Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210572024-02-28T22:21:12.452Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307483784Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210582024-02-28T22:21:12.452Z{"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307497853Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210592024-02-28T22:21:12.452Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307517643Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210602024-02-28T22:21:12.452Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307531593Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210612024-02-28T22:21:12.452Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307544962Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210622024-02-28T22:21:12.452Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307559042Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210632024-02-28T22:21:12.453Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307572551Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210642024-02-28T22:21:12.453Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307592451Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210652024-02-28T22:21:12.453Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.30760516Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210662024-02-28T22:21:12.453Z{"msg":"28985aec-eb7b-4dc8-9fe3-5ea9e3aa9638 is now active with session: 4ad7840d-7d53-4f10-8d74-9120dfc7e3c9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.3076247Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210672024-02-28T22:21:12.454Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307637279Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919,"session_id":"4ad7840d-7d53-4f10-8d74-9120dfc7e3c9"}
210682024-02-28T22:21:12.454Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:11.307649719Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
210692024-02-28T22:21:12.455ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
210702024-02-28T22:21:12.455ZLoading write count information from file "/tmp/test_repair_verify.data"
210712024-02-28T22:21:12.456ZRun Repair workload
210722024-02-28T22:21:12.456Z01/30 Write block 346 len 16384 data: 12 13 14 13
210732024-02-28T22:21:12.457Z02/30 Flush
210742024-02-28T22:21:12.457Z03/30 Write block 7 len 4096 data: 20
210752024-02-28T22:21:12.458Z04/30 Write block 180 len 40960 data: 10 10 12 14 14 13 10 12 12 14
210762024-02-28T22:21:12.458Z05/30 Read block 158 len 12288
210772024-02-28T22:21:12.459Z06/30 Read block 21 len 8192
210782024-02-28T22:21:12.459Z07/30 Write block 451 len 28672 data: 21 19 22 20 19 18 16
210792024-02-28T22:21:12.466Z08/30 Write block 145 len 32768 data: 20 21 20 20 18 15 12 12
210802024-02-28T22:21:12.466Z09/30 Write block 156 len 4096 data: 6
210812024-02-28T22:21:12.466Z10/30 Write block 501 len 16384 data: 16 18 18 19
210822024-02-28T22:21:12.466Z11/30 Read block 227 len 12288
210832024-02-28T22:21:12.467Z12/30 Write block 60 len 4096 data: 18
210842024-02-28T22:21:12.467Z13/30 Flush
210852024-02-28T22:21:12.467Z14/30 Write block 126 len 4096 data: 20
210862024-02-28T22:21:12.467Z15/30 Flush
210872024-02-28T22:21:12.467Z16/30 Write block 340 len 16384 data: 12 13 15 15
210882024-02-28T22:21:12.468Z17/30 Read block 288 len 36864
210892024-02-28T22:21:12.468Z18/30 Read block 509 len 16384
210902024-02-28T22:21:12.468Z19/30 Read block 157 len 36864
210912024-02-28T22:21:12.468Z20/30 Read block 444 len 16384
210922024-02-28T22:21:12.468Z21/30 Read block 458 len 12288
210932024-02-28T22:21:12.468Z22/30 Flush
210942024-02-28T22:21:12.469Z23/30 Flush
210952024-02-28T22:21:12.469Z24/30 Write block 140 len 16384 data: 13 12 13 15
210962024-02-28T22:21:12.469Z25/30 Read block 103 len 16384
210972024-02-28T22:21:12.469Z26/30 Write block 488 len 8192 data: 10 15
210982024-02-28T22:21:12.469Z27/30 Read block 486 len 28672
210992024-02-28T22:21:12.470Z28/30 Read block 497 len 28672
211002024-02-28T22:21:12.470Z29/30 Read block 229 len 16384
211012024-02-28T22:21:12.470Z30/30 Write block 460 len 32768 data: 9 9 10 13 12 13 12 13
211022024-02-28T22:21:12.470Z----------------------------------------------------------------
211032024-02-28T22:21:12.470Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
211042024-02-28T22:21:12.470ZStates: Active Active Active
211052024-02-28T22:21:12.470ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211062024-02-28T22:21:12.471Z 1 Acked 1000 Write 4 Done Sent Done false
211072024-02-28T22:21:12.471Z 2 Acked 1001 Flush 0 Done Sent Done false
211082024-02-28T22:21:12.471Z 3 Acked 1002 Write 1 Done Sent Done false
211092024-02-28T22:21:12.471Z 4 Acked 1003 Write 10 Done Sent Done false
211102024-02-28T22:21:12.471Z 5 Acked 1004 Read 3 Done Sent Done false
211112024-02-28T22:21:12.471Z 6 Acked 1005 Read 2 Done Sent Done false
211122024-02-28T22:21:12.472Z 7 Acked 1006 Write 7 Done Sent Done false
211132024-02-28T22:21:12.472Z 8 Acked 1007 Write 8 Done Sent Done false
211142024-02-28T22:21:12.472Z 9 Acked 1008 Write 1 Done Sent Done false
211152024-02-28T22:21:12.472Z 10 Acked 1009 Write 4 Done Sent Done false
211162024-02-28T22:21:12.472Z 11 Acked 1010 Read 3 Done Sent Done false
211172024-02-28T22:21:12.473Z 12 Acked 1011 Write 1 Done Sent Done false
211182024-02-28T22:21:12.473Z 13 Acked 1012 Flush 0 Done Sent Done false
211192024-02-28T22:21:12.473Z 14 Acked 1013 Write 1 Done Sent Done false
211202024-02-28T22:21:12.473Z 15 Acked 1014 Flush 0 Done Sent Done false
211212024-02-28T22:21:12.473Z 16 Acked 1015 Write 4 Done Sent Done false
211222024-02-28T22:21:12.473Z 17 Acked 1016 Read 9 Done Sent Done false
211232024-02-28T22:21:12.473Z 18 Acked 1017 Read 4 Done Sent Done false
211242024-02-28T22:21:12.474Z 19 Acked 1018 Read 9 Done Sent Done false
211252024-02-28T22:21:12.474Z 20 Acked 1019 Read 4 Done Sent Done false
211262024-02-28T22:21:12.474Z 21 Acked 1020 Read 3 Done Sent Done false
211272024-02-28T22:21:12.474Z 22 Acked 1021 Flush 0 Done Sent Done false
211282024-02-28T22:21:12.474Z 23 Acked 1022 Flush 0 Done Sent Done false
211292024-02-28T22:21:12.475Z 24 Acked 1023 Write 4 Done Sent Done false
211302024-02-28T22:21:12.475Z 25 Acked 1024 Read 4 Done Sent Done false
211312024-02-28T22:21:12.475Z 26 Acked 1025 Write 2 Done Sent Done false
211322024-02-28T22:21:12.475Z 27 Acked 1026 Read 7 Done Sent Done false
211332024-02-28T22:21:12.475Z 28 Acked 1027 Read 7 Done Sent Done false
211342024-02-28T22:21:12.476Z 29 Acked 1028 Read 4 Done Sent Done false
211352024-02-28T22:21:12.476Z 30 Acked 1029 Write 8 Sent Sent Sent false
211362024-02-28T22:21:12.476Z STATES DS:0 DS:1 DS:2 TOTAL
211372024-02-28T22:21:12.476Z New 0 0 0 0
211382024-02-28T22:21:12.476Z Sent 1 30 1 32
211392024-02-28T22:21:12.476Z Done 29 0 29 58
211402024-02-28T22:21:12.476Z Skipped 0 0 0 0
211412024-02-28T22:21:12.476Z Error 0 0 0 0
211422024-02-28T22:21:12.476ZLast Flush: 1022 0 1022
211432024-02-28T22:21:12.476ZDownstairs last five completed:
211442024-02-28T22:21:12.477ZUpstairs last five completed: 30 29 28 27 26
211452024-02-28T22:21:12.477Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:21:11.337829668Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1919}
211462024-02-28T22:21:12.477ZWrote out file "/tmp/test_repair_verify.data"
211472024-02-28T22:21:12.488Z
211482024-02-28T22:21:12.488ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211492024-02-28T22:21:12.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211502024-02-28T22:21:12.502Z 0 000-019 196 189 196 356 347 356 F F F <---
211512024-02-28T22:21:12.502Z 1 020-039 191 191 191 350 350 350 F F F
211522024-02-28T22:21:12.502Z 2 040-059 189 189 189 347 347 347 F F F
211532024-02-28T22:21:12.502Z 3 060-079 196 190 196 356 348 356 F F F <---
211542024-02-28T22:21:12.503Z 4 080-099 195 195 195 354 354 354 F F F
211552024-02-28T22:21:12.503Z 5 100-119 192 192 192 351 351 351 F F F
211562024-02-28T22:21:12.503Z 6 120-139 196 192 196 357 351 357 F F F <---
211572024-02-28T22:21:12.503Z 7 140-159 196 185 196 356 341 356 T F T <---
211582024-02-28T22:21:12.503Z 8 160-179 194 194 194 353 353 353 F F F
211592024-02-28T22:21:12.503Z 9 180-199 196 182 196 356 335 356 F F F <---
211602024-02-28T22:21:12.504Z 10 200-219 194 194 194 353 353 353 F F F
211612024-02-28T22:21:12.504Z 11 220-239 195 195 195 354 354 354 F F F
211622024-02-28T22:21:12.504Z 12 240-259 189 189 189 347 347 347 F F F
211632024-02-28T22:21:12.504Z 13 260-279 194 194 194 353 353 353 F F F
211642024-02-28T22:21:12.504Z 14 280-299 190 190 190 349 349 349 F F F
211652024-02-28T22:21:12.504Z 15 300-319 192 192 192 351 351 351 F F F
211662024-02-28T22:21:12.504Z 16 320-339 194 194 194 353 353 353 F F F
211672024-02-28T22:21:12.505Z 17 340-359 196 194 196 358 353 358 F F F <---
211682024-02-28T22:21:12.505Z 18 360-379 194 194 194 353 353 353 F F F
211692024-02-28T22:21:12.505Z 19 380-399 186 186 186 343 343 343 F F F
211702024-02-28T22:21:12.505Z 20 400-419 192 192 192 351 351 351 F F F
211712024-02-28T22:21:12.505Z 21 420-439 195 195 195 354 354 354 F F F
211722024-02-28T22:21:12.506Z 22 440-459 196 195 196 356 354 356 F F F <---
211732024-02-28T22:21:12.506Z 23 460-479 192 192 192 351 351 351 T F T <---
211742024-02-28T22:21:12.506Z 24 480-499 194 194 194 353 353 353 T F T <---
211752024-02-28T22:21:12.506Z 25 500-519 196 192 196 356 351 356 F F F <---
211762024-02-28T22:21:12.506Z 26 520-539 193 193 193 352 352 352 F F F
211772024-02-28T22:21:12.506Z 27 540-559 191 191 191 350 350 350 F F F
211782024-02-28T22:21:12.507Z 28 560-579 194 194 194 353 353 353 F F F
211792024-02-28T22:21:12.507Z 29 580-599 186 186 186 343 343 343 F F F
211802024-02-28T22:21:12.507ZMax gen: 196, Max flush: 358
211812024-02-28T22:21:12.507ZError: Difference in extent metadata found!
211822024-02-28T22:21:12.507ZA Difference in extent metadata is expected here
211832024-02-28T22:21:12.507ZOn loop 98
211842024-02-28T22:21:12.507Z
211852024-02-28T22:21:12.517ZVerifying data now
211862024-02-28T22:21:17.030ZNo change after verify
211872024-02-28T22:21:17.031ZLoop: 98 Downstairs dump after verify (and repair):
211882024-02-28T22:21:17.044ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211892024-02-28T22:21:17.045Z 0 000-019 196 196 196 356 356 356 F F F
211902024-02-28T22:21:17.045Z 1 020-039 191 191 191 350 350 350 F F F
211912024-02-28T22:21:17.045Z 2 040-059 189 189 189 347 347 347 F F F
211922024-02-28T22:21:17.045Z 3 060-079 196 196 196 356 356 356 F F F
211932024-02-28T22:21:17.045Z 4 080-099 195 195 195 354 354 354 F F F
211942024-02-28T22:21:17.045Z 5 100-119 192 192 192 351 351 351 F F F
211952024-02-28T22:21:17.045Z 6 120-139 196 196 196 357 357 357 F F F
211962024-02-28T22:21:17.045Z 7 140-159 197 197 197 359 359 359 F F F
211972024-02-28T22:21:17.045Z 8 160-179 194 194 194 353 353 353 F F F
211982024-02-28T22:21:17.048Z 9 180-199 196 196 196 356 356 356 F F F
211992024-02-28T22:21:17.048Z 10 200-219 194 194 194 353 353 353 F F F
212002024-02-28T22:21:17.048Z 11 220-239 195 195 195 354 354 354 F F F
212012024-02-28T22:21:17.048Z 12 240-259 189 189 189 347 347 347 F F F
212022024-02-28T22:21:17.049Z 13 260-279 194 194 194 353 353 353 F F F
212032024-02-28T22:21:17.049Z 14 280-299 190 190 190 349 349 349 F F F
212042024-02-28T22:21:17.049Z 15 300-319 192 192 192 351 351 351 F F F
212052024-02-28T22:21:17.049Z 16 320-339 194 194 194 353 353 353 F F F
212062024-02-28T22:21:17.049Z 17 340-359 196 196 196 358 358 358 F F F
212072024-02-28T22:21:17.049Z 18 360-379 194 194 194 353 353 353 F F F
212082024-02-28T22:21:17.049Z 19 380-399 186 186 186 343 343 343 F F F
212092024-02-28T22:21:17.049Z 20 400-419 192 192 192 351 351 351 F F F
212102024-02-28T22:21:17.049Z 21 420-439 195 195 195 354 354 354 F F F
212112024-02-28T22:21:17.049Z 22 440-459 196 196 196 356 356 356 F F F
212122024-02-28T22:21:17.049Z 23 460-479 197 197 197 359 359 359 F F F
212132024-02-28T22:21:17.049Z 24 480-499 197 197 197 359 359 359 F F F
212142024-02-28T22:21:17.049Z 25 500-519 196 196 196 356 356 356 F F F
212152024-02-28T22:21:17.049Z 26 520-539 193 193 193 352 352 352 F F F
212162024-02-28T22:21:17.049Z 27 540-559 191 191 191 350 350 350 F F F
212172024-02-28T22:21:17.049Z 28 560-579 194 194 194 353 353 353 F F F
212182024-02-28T22:21:17.049Z 29 580-599 186 186 186 343 343 343 F F F
212192024-02-28T22:21:17.049ZMax gen: 197, Max flush: 359
212202024-02-28T22:21:17.049Z
212212024-02-28T22:21:17.049ZBegin loop 99
212222024-02-28T22:21:17.049ZDownstairs to restart: 2
212232024-02-28T22:21:17.066Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943175154Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
212242024-02-28T22:21:17.067Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\",\n git_commit_timestamp: \"2024-02-28T21:51:33.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943454657Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
212252024-02-28T22:21:17.067ZCrucible runtime is spawned
212262024-02-28T22:21:17.072Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943498606Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
212272024-02-28T22:21:17.072Z{"msg":"Crucible e1bf0fbf-86df-4b55-85a2-26bd52535d2f has session id: 4ba21e89-cb3d-4a56-9cec-8aba76bf65bc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943532855Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212282024-02-28T22:21:17.072Z{"msg":"Upstairs opts: Upstairs UUID: e1bf0fbf-86df-4b55-85a2-26bd52535d2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943582013Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212292024-02-28T22:21:17.072Z{"msg":"Crucible stats registered with UUID: e1bf0fbf-86df-4b55-85a2-26bd52535d2f","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943642712Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212302024-02-28T22:21:17.072Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943668381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
212312024-02-28T22:21:17.072Z{"msg":"e1bf0fbf-86df-4b55-85a2-26bd52535d2f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943681141Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212322024-02-28T22:21:17.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94370138Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212332024-02-28T22:21:17.072Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943722379Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212342024-02-28T22:21:17.072Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943751959Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212352024-02-28T22:21:17.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943786718Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212362024-02-28T22:21:17.073Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943826607Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212372024-02-28T22:21:17.073Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943870845Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212382024-02-28T22:21:17.073Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943913094Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212392024-02-28T22:21:17.073Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943956533Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212402024-02-28T22:21:17.073Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.943984232Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212412024-02-28T22:21:17.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944017261Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212422024-02-28T22:21:17.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944039381Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212432024-02-28T22:21:17.073Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94406395Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212442024-02-28T22:21:17.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944746311Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212452024-02-28T22:21:17.073Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94479145Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212462024-02-28T22:21:17.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944831419Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212472024-02-28T22:21:17.073Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944864238Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212482024-02-28T22:21:17.073Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944906917Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212492024-02-28T22:21:17.073Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.944948486Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212502024-02-28T22:21:17.073Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.945328085Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212512024-02-28T22:21:17.073Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.945447132Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212522024-02-28T22:21:17.073Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.945662036Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212532024-02-28T22:21:17.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94587572Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212542024-02-28T22:21:17.073Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.945953708Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212552024-02-28T22:21:17.073Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946016956Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212562024-02-28T22:21:17.073Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946058775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212572024-02-28T22:21:17.074Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946082625Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212582024-02-28T22:21:17.074Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946129203Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212592024-02-28T22:21:17.074Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946157462Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212602024-02-28T22:21:17.074Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946182462Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212612024-02-28T22:21:17.074Z{"msg":"[0]R flush_numbers[0..12]: [356, 350, 347, 356, 354, 351, 357, 359, 353, 356, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946218321Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212622024-02-28T22:21:17.074Z{"msg":"[0]R generation[0..12]: [196, 191, 189, 196, 195, 192, 196, 197, 194, 196, 194, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94625543Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212632024-02-28T22:21:17.074Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946290609Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212642024-02-28T22:21:17.074Z{"msg":"[1]R flush_numbers[0..12]: [356, 350, 347, 356, 354, 351, 357, 359, 353, 356, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946326818Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212652024-02-28T22:21:17.074Z{"msg":"[1]R generation[0..12]: [196, 191, 189, 196, 195, 192, 196, 197, 194, 196, 194, 195]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
212662024-02-28T22:21:17.074Z,"time":"2024-02-28T22:21:15.946362847Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212672024-02-28T22:21:17.074Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946404996Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212682024-02-28T22:21:17.074Z{"msg":"[2]R flush_numbers[0..12]: [356, 350, 347, 356, 354, 351, 357, 359, 353, 356, 353, 354]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946442775Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212692024-02-28T22:21:17.074Z{"msg":"[2]R generation[0..12]: [196, 191, 189, 196, 195, 192, 196, 197, 194, 196, 194, 195]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946481514Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212702024-02-28T22:21:17.074Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946520582Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212712024-02-28T22:21:17.074Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946556501Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212722024-02-28T22:21:17.074Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.94659314Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212732024-02-28T22:21:17.074Z{"msg":"Next flush: 360","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946629219Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212742024-02-28T22:21:17.074Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946664129Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212752024-02-28T22:21:17.074Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946679928Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212762024-02-28T22:21:17.074ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
212772024-02-28T22:21:17.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946703927Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212782024-02-28T22:21:17.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946718207Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212792024-02-28T22:21:17.074Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946739366Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212802024-02-28T22:21:17.075Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946753206Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212812024-02-28T22:21:17.075Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946780695Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212822024-02-28T22:21:17.075Z{"msg":"e1bf0fbf-86df-4b55-85a2-26bd52535d2f is now active with session: 4ba21e89-cb3d-4a56-9cec-8aba76bf65bc","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946794825Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212832024-02-28T22:21:17.075Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946814994Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927,"session_id":"4ba21e89-cb3d-4a56-9cec-8aba76bf65bc"}
212842024-02-28T22:21:17.075Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:21:15.946827744Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
212852024-02-28T22:21:17.075ZLoading write count information from file "/tmp/test_repair_verify.data"
212862024-02-28T22:21:17.075ZRun Repair workload
212872024-02-28T22:21:17.079Z01/30 Write block 120 len 16384 data: 13 14 15 15
212882024-02-28T22:21:17.079Z02/30 Write block 222 len 32768 data: 13 11 11 14 15 12 15 19
212892024-02-28T22:21:17.079Z03/30 Read block 581 len 8192
212902024-02-28T22:21:17.079Z04/30 Write block 469 len 24576 data: 12 16 19 20 16 17
212912024-02-28T22:21:17.079Z05/30 Read block 476 len 32768
212922024-02-28T22:21:17.079Z06/30 Read block 517 len 32768
212932024-02-28T22:21:17.079Z07/30 Read block 593 len 8192
212942024-02-28T22:21:17.079Z08/30 Write block 291 len 32768 data: 9 9 8 6 11 13 11 10
212952024-02-28T22:21:17.079Z09/30 Write block 386 len 16384 data: 25 22 17 17
212962024-02-28T22:21:17.079Z10/30 Flush
212972024-02-28T22:21:17.084Z11/30 Write block 403 len 12288 data: 12 15 15
212982024-02-28T22:21:17.084Z12/30 Read block 146 len 16384
212992024-02-28T22:21:17.088Z13/30 Write block 544 len 40960 data: 13 14 15 13 9 11 9 8 8 8
213002024-02-28T22:21:17.088Z14/30 Write block 183 len 16384 data: 15 15 14 11
213012024-02-28T22:21:17.088Z15/30 Write block 382 len 4096 data: 21
213022024-02-28T22:21:17.088Z16/30 Write block 233 len 24576 data: 16 17 14 14 15 14
213032024-02-28T22:21:17.088Z17/30 Read block 167 len 4096
213042024-02-28T22:21:17.088Z18/30 Write block 302 len 32768 data: 11 14 14 14 13 18 17 17
213052024-02-28T22:21:17.088Z19/30 Read block 205 len 32768
213062024-02-28T22:21:17.091Z20/30 Read block 421 len 16384
213072024-02-28T22:21:17.091Z21/30 Write block 444 len 12288 data: 17 17 18
213082024-02-28T22:21:17.091Z22/30 Write block 164 len 40960 data: 13 15 15 19 20 18 18 16 15 17
213092024-02-28T22:21:17.091Z23/30 Read block 220 len 24576
213102024-02-28T22:21:17.095Z24/30 Write block 333 len 28672 data: 15 14 16 15 13 12 13
213112024-02-28T22:21:17.095Z25/30 Write block 412 len 8192 data: 12 14
213122024-02-28T22:21:17.095Z26/30 Write block 138 len 32768 data: 16 14 14 13 14 16 15 21
213132024-02-28T22:21:17.095Z27/30 Write block 518 len 28672 data: 20 21 21 19 17 12 11
213142024-02-28T22:21:17.102Z28/30 Read block 429 len 12288
213152024-02-28T22:21:17.102Z29/30 Write block 291 len 12288 data: 10 10 9
213162024-02-28T22:21:17.102Z30/30 Read block 104 len 20480
213172024-02-28T22:21:17.103Z----------------------------------------------------------------
213182024-02-28T22:21:17.103Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213192024-02-28T22:21:17.103ZStates: Active Active Active
213202024-02-28T22:21:17.103ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213212024-02-28T22:21:17.103Z 1 Acked 1000 Write 4 Done Done Sent false
213222024-02-28T22:21:17.103Z 2 Acked 1001 Write 8 Done Done Sent false
213232024-02-28T22:21:17.103Z 3 Acked 1002 Read 2 Done Done Sent false
213242024-02-28T22:21:17.103Z 4 Acked 1003 Write 6 Done Done Sent false
213252024-02-28T22:21:17.103Z 5 Acked 1004 Read 8 Done Done Sent false
213262024-02-28T22:21:17.103Z 6 Acked 1005 Read 8 Done Done Sent false
213272024-02-28T22:21:17.103Z 7 Acked 1006 Read 2 Done Done Sent false
213282024-02-28T22:21:17.103Z 8 Acked 1007 Write 8 Done Done Sent false
213292024-02-28T22:21:17.103Z 9 Acked 1008 Write 4 Done Done Sent false
213302024-02-28T22:21:17.103Z 10 Acked 1009 Flush 0 Done Done Sent false
213312024-02-28T22:21:17.103Z 11 Acked 1010 Write 3 Done Done Sent false
213322024-02-28T22:21:17.103Z 12 Acked 1011 Read 4 Done Done Sent false
213332024-02-28T22:21:17.103Z 13 Acked 1012 Write 10 Done Done Sent false
213342024-02-28T22:21:17.103Z 14 Acked 1013 Write 4 Done Done Sent false
213352024-02-28T22:21:17.103Z 15 Acked 1014 Write 1 Done Done Sent false
213362024-02-28T22:21:17.103Z 16 Acked 1015 Write 6 Done Done Sent false
213372024-02-28T22:21:17.104Z 17 Acked 1016 Read 1 Done Done Sent false
213382024-02-28T22:21:17.104Z 18 Acked 1017 Write 8 Done Done Sent false
213392024-02-28T22:21:17.104Z 19 Acked 1018 Read 8 Done Done Sent false
213402024-02-28T22:21:17.104Z 20 Acked 1019 Read 4 Done Done Sent false
213412024-02-28T22:21:17.104Z 21 Acked 1020 Write 3 Done Done Sent false
213422024-02-28T22:21:17.104Z 22 Acked 1021 Write 10 Done Done Sent false
213432024-02-28T22:21:17.104Z 23 Acked 1022 Read 6 Done Done Sent false
213442024-02-28T22:21:17.104Z 24 Acked 1023 Write 7 Done Done Sent false
213452024-02-28T22:21:17.104Z 25 Acked 1024 Write 2 Done Done Sent false
213462024-02-28T22:21:17.104Z 26 Acked 1025 Write 8 Done Done Sent false
213472024-02-28T22:21:17.104Z 27 Acked 1026 Write 7 Done Done Sent false
213482024-02-28T22:21:17.104Z 28 Acked 1027 Read 3 Done Done Sent false
213492024-02-28T22:21:17.104Z 29 Acked 1028 Write 3 Done Done Sent false
213502024-02-28T22:21:17.104Z 30 Acked 1029 Read 5 Done Done Sent false
213512024-02-28T22:21:17.104Z STATES DS:0 DS:1 DS:2 TOTAL
213522024-02-28T22:21:17.104Z New 0 0 0 0
213532024-02-28T22:21:17.104Z Sent 0 0 30 30
213542024-02-28T22:21:17.104Z Done 30 30 0 60
213552024-02-28T22:21:17.104Z Skipped 0 0 0 0
213562024-02-28T22:21:17.104Z Error 0 0 0 0
213572024-02-28T22:21:17.104ZLast Flush: 1009 1009 0
213582024-02-28T22:21:17.104ZDownstairs last five completed:
213592024-02-28T22:21:17.104ZUpstairs last five completed: 30 29 28 27 26
213602024-02-28T22:21:17.105Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-28T22:21:15.975055356Z","hostname":"ip-10-150-1-47.us-west-2.compute.internal","pid":1927}
213612024-02-28T22:21:17.105ZWrote out file "/tmp/test_repair_verify.data"
213622024-02-28T22:21:17.109Z
213632024-02-28T22:21:17.109ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213642024-02-28T22:21:17.128ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
213652024-02-28T22:21:17.129Z 0 000-019 196 196 196 356 356 356 F F F
213662024-02-28T22:21:17.129Z 1 020-039 191 191 191 350 350 350 F F F
213672024-02-28T22:21:17.129Z 2 040-059 189 189 189 347 347 347 F F F
213682024-02-28T22:21:17.129Z 3 060-079 196 196 196 356 356 356 F F F
213692024-02-28T22:21:17.129Z 4 080-099 195 195 195 354 354 354 F F F
213702024-02-28T22:21:17.129Z 5 100-119 192 192 192 351 351 351 F F F
213712024-02-28T22:21:17.129Z 6 120-139 198 198 196 360 360 357 T T F <---
213722024-02-28T22:21:17.129Z 7 140-159 197 197 197 359 359 359 T T F <---
213732024-02-28T22:21:17.129Z 8 160-179 194 194 194 353 353 353 T T F <---
213742024-02-28T22:21:17.129Z 9 180-199 196 196 196 356 356 356 T T F <---
213752024-02-28T22:21:17.138Z 10 200-219 194 194 194 353 353 353 F F F
213762024-02-28T22:21:17.138Z 11 220-239 198 198 195 360 360 354 T T F <---
213772024-02-28T22:21:17.138Z 12 240-259 189 189 189 347 347 347 F F F
213782024-02-28T22:21:17.138Z 13 260-279 194 194 194 353 353 353 F F F
213792024-02-28T22:21:17.138Z 14 280-299 198 198 190 360 360 349 T T F <---
213802024-02-28T22:21:17.138Z 15 300-319 192 192 192 351 351 351 T T F <---
213812024-02-28T22:21:17.138Z 16 320-339 194 194 194 353 353 353 T T F <---
213822024-02-28T22:21:17.139Z 17 340-359 196 196 196 358 358 358 F F F
213832024-02-28T22:21:17.139Z 18 360-379 194 194 194 353 353 353 F F F
213842024-02-28T22:21:17.139Z 19 380-399 198 198 186 360 360 343 T T F <---
213852024-02-28T22:21:17.139Z 20 400-419 192 192 192 351 351 351 T T F <---
213862024-02-28T22:21:17.139Z 21 420-439 195 195 195 354 354 354 F F F
213872024-02-28T22:21:17.139Z 22 440-459 196 196 196 356 356 356 T T F <---
213882024-02-28T22:21:17.139Z 23 460-479 198 198 197 360 360 359 F F F <---
213892024-02-28T22:21:17.139Z 24 480-499 197 197 197 359 359 359 F F F
213902024-02-28T22:21:17.139Z 25 500-519 196 196 196 356 356 356 T T F <---
213912024-02-28T22:21:17.139Z 26 520-539 193 193 193 352 352 352 T T F <---
213922024-02-28T22:21:17.139Z 27 540-559 191 191 191 350 350 350 T T F <---
213932024-02-28T22:21:17.139Z 28 560-579 194 194 194 353 353 353 F F F
213942024-02-28T22:21:17.139Z 29 580-599 186 186 186 343 343 343 F F F
213952024-02-28T22:21:17.139ZMax gen: 198, Max flush: 360
213962024-02-28T22:21:17.139ZError: Difference in extent metadata found!
213972024-02-28T22:21:17.139ZA Difference in extent metadata is expected here
213982024-02-28T22:21:17.139ZOn loop 99
213992024-02-28T22:21:17.139Z
214002024-02-28T22:21:17.139ZVerifying data now
214012024-02-28T22:21:21.905ZNo change after verify
214022024-02-28T22:21:21.905ZLoop: 99 Downstairs dump after verify (and repair):
214032024-02-28T22:21:21.920ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214042024-02-28T22:21:21.920Z 0 000-019 196 196 196 356 356 356 F F F
214052024-02-28T22:21:21.920Z 1 020-039 191 191 191 350 350 350 F F F
214062024-02-28T22:21:21.920Z 2 040-059 189 189 189 347 347 347 F F F
214072024-02-28T22:21:21.920Z 3 060-079 196 196 196 356 356 356 F F F
214082024-02-28T22:21:21.920Z 4 080-099 195 195 195 354 354 354 F F F
214092024-02-28T22:21:21.920Z 5 100-119 192 192 192 351 351 351 F F F
214102024-02-28T22:21:21.920Z 6 120-139 199 199 199 361 361 361 F F F
214112024-02-28T22:21:21.920Z 7 140-159 199 199 199 361 361 361 F F F
214122024-02-28T22:21:21.920Z 8 160-179 199 199 199 361 361 361 F F F
214132024-02-28T22:21:21.920Z 9 180-199 199 199 199 361 361 361 F F F
214142024-02-28T22:21:21.920Z 10 200-219 194 194 194 353 353 353 F F F
214152024-02-28T22:21:21.924Z 11 220-239 199 199 199 361 361 361 F F F
214162024-02-28T22:21:21.925Z 12 240-259 189 189 189 347 347 347 F F F
214172024-02-28T22:21:21.925Z 13 260-279 194 194 194 353 353 353 F F F
214182024-02-28T22:21:21.925Z 14 280-299 199 199 199 361 361 361 F F F
214192024-02-28T22:21:21.925Z 15 300-319 199 199 199 361 361 361 F F F
214202024-02-28T22:21:21.925Z 16 320-339 199 199 199 361 361 361 F F F
214212024-02-28T22:21:21.925Z 17 340-359 196 196 196 358 358 358 F F F
214222024-02-28T22:21:21.925Z 18 360-379 194 194 194 353 353 353 F F F
214232024-02-28T22:21:21.925Z 19 380-399 199 199 199 361 361 361 F F F
214242024-02-28T22:21:21.925Z 20 400-419 199 199 199 361 361 361 F F F
214252024-02-28T22:21:21.926Z 21 420-439 195 195 195 354 354 354 F F F
214262024-02-28T22:21:21.926Z 22 440-459 199 199 199 361 361 361 F F F
214272024-02-28T22:21:21.926Z 23 460-479 198 198 198 360 360 360 F F F
214282024-02-28T22:21:21.926Z 24 480-499 197 197 197 359 359 359 F F F
214292024-02-28T22:21:21.926Z 25 500-519 199 199 199 361 361 361 F F F
214302024-02-28T22:21:21.926Z 26 520-539 199 199 199 361 361 361 F F F
214312024-02-28T22:21:21.926Z 27 540-559 199 199 199 361 361 361 F F F
214322024-02-28T22:21:21.926Z 28 560-579 194 194 194 353 353 353 F F F
214332024-02-28T22:21:21.926Z 29 580-599 186 186 186 343 343 343 F F F
214342024-02-28T22:21:21.926ZMax gen: 199, Max flush: 361
214352024-02-28T22:21:21.927Z8:27 Test duration
214362024-02-28T22:21:21.927ZTest completed
214372024-02-28T22:21:21.932Z
214382024-02-28T22:21:21.932Zreal 8:27.673773444
214392024-02-28T22:21:21.932Zuser 1:31.922052355
214402024-02-28T22:21:21.932Zsys 17.927367616
214412024-02-28T22:21:21.933Ztrap 0.082373251
214422024-02-28T22:21:21.933Ztflt 0.051172785
214432024-02-28T22:21:21.933Zdflt 0.049829660
214442024-02-28T22:21:21.933Zkflt 0.003438105
214452024-02-28T22:21:21.933Zlock 9:54:46.058804595
214462024-02-28T22:21:21.933Zslp 40:47.321445442
214472024-02-28T22:21:21.933Zlat 6.494897001
214482024-02-28T22:21:21.934Zstop 0.049972400
214492024-02-28T22:21:21.934Zprocess exited: duration 516634 ms, exit code 0
 
214502024-02-28T22:21:21.966Zfound 4 output files
214512024-02-28T22:21:21.966Zuploading: /tmp/test_repair_ds8810.txt (116037 bytes)
214522024-02-28T22:21:23.426Zuploaded: /tmp/test_repair_ds8810.txt
214532024-02-28T22:21:23.426Zuploading: /tmp/test_repair_ds8820.txt (43608 bytes)
214542024-02-28T22:21:24.464Zuploaded: /tmp/test_repair_ds8820.txt
214552024-02-28T22:21:24.464Zuploading: /tmp/test_repair_ds8830.txt (26137 bytes)
214562024-02-28T22:21:25.486Zuploaded: /tmp/test_repair_ds8830.txt
214572024-02-28T22:21:25.904Zuploading: /tmp/test_repair_out.txt (97304 bytes)
214582024-02-28T22:21:26.994Zuploaded: /tmp/test_repair_out.txt