01HNJS51RXA6W02N9RQAS6SC0C: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HNJS5GDHA7WVP2BA45X7M54E

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-01T17:24:39.705Zjob dependencies complete; ready to run (waiting for 35 m 46 s)
22024-02-01T17:26:30.360Zjob assigned to worker 01HNJV783D5Z8B8P7RJA5ZJE0X (queued for 1 m 50 s)
32024-02-01T17:26:31.204Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-01T17:26:47.427Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-01T17:26:47.428Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-01T17:26:59.200Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-01T17:26:59.200Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-01T17:27:12.267Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-01T17:27:12.267Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-01T17:27:20.490Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-01T17:27:20.507Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-01T17:27:20.523Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-01T17:27:20.524Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-01T17:27:20.543Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-01T17:27:20.543Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-01T17:27:20.564Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-01T17:27:20.564Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-01T17:27:20.591Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-01T17:27:20.591Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-01T17:27:20.615Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-01T17:27:20.615Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-01T17:27:20.638Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-01T17:27:20.638Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-01T17:27:20.651Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-01T17:27:20.654Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-01T17:27:40.678Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-01T17:27:40.678Zstarting task 0: "setup"
282024-02-01T17:27:40.696Z++ uname -s
292024-02-01T17:27:40.698Z+ kern=SunOS
302024-02-01T17:27:40.698Z+ case "$kern" in
312024-02-01T17:27:40.698Z+ groupadd -g 12345 build
322024-02-01T17:27:40.698Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-01T17:27:42.794Z+ zfs create -o mountpoint=/work rpool/work
342024-02-01T17:27:43.044Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-01T17:27:43.057Z+ home_fs=zfs
362024-02-01T17:27:43.057Z+ [[ zfs == autofs ]]
372024-02-01T17:27:43.057Z+ mkdir -p /home/build
382024-02-01T17:27:43.057Z+ chown build:build /home/build /work
392024-02-01T17:27:45.033Z+ chmod 0700 /home/build /work
402024-02-01T17:27:45.036Zprocess exited: duration 4379 ms, exit code 0
 
412024-02-01T17:27:45.047Zstarting task 1: "authentication"
422024-02-01T17:27:45.082Zprocess exited: duration 28 ms, exit code 0
 
432024-02-01T17:27:45.097Zstarting task 2: "build"
442024-02-01T17:27:45.104Z+ banner cores
452024-02-01T17:27:45.173Z
462024-02-01T17:27:45.174Z #### #### ##### ###### ####
472024-02-01T17:27:45.174Z # # # # # # # #
482024-02-01T17:27:45.174Z # # # # # ##### ####
492024-02-01T17:27:45.174Z # # # ##### # #
502024-02-01T17:27:45.174Z # # # # # # # # #
512024-02-01T17:27:45.174Z #### #### # # ###### ####
522024-02-01T17:27:45.174Z
532024-02-01T17:27:45.177Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-01T17:27:45.190Z+ echo 'input bins dir contains:'
552024-02-01T17:27:45.190Zinput bins dir contains:
562024-02-01T17:27:45.194Z+ ls -ltr /input/build/work/bins
572024-02-01T17:27:45.194Ztotal 762468
582024-02-01T17:27:45.194Z-rw-r--r-- 1 root root 121626818 Feb 1 17:26 crucible-downstairs.gz
592024-02-01T17:27:45.194Z-rw-r--r-- 1 root root 91386942 Feb 1 17:26 crucible-hammer.gz
602024-02-01T17:27:45.194Z-rw-r--r-- 1 root root 105088684 Feb 1 17:27 crutest.gz
612024-02-01T17:27:45.194Z-rw-r--r-- 1 root root 72023915 Feb 1 17:27 dsc.gz
622024-02-01T17:27:45.194Z+ echo 'input script dir contains:'
632024-02-01T17:27:45.194Zinput script dir contains:
642024-02-01T17:27:45.195Z+ ls -ltr /input/build/work/scripts
652024-02-01T17:27:45.195Ztotal 38
662024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 3056 Feb 1 17:27 perf-downstairs-tick.d
672024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 939 Feb 1 17:27 test_ds.sh
682024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 3746 Feb 1 17:27 test_live_repair.sh
692024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 6416 Feb 1 17:27 test_repair.sh
702024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 2969 Feb 1 17:27 test_replay.sh
712024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 9465 Feb 1 17:27 test_up.sh
722024-02-01T17:27:45.195Z-rw-r--r-- 1 root root 2824 Feb 1 17:27 upstairs_info.d
732024-02-01T17:27:45.195Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742024-02-01T17:27:45.198Z+ echo ' chmod input script dir contains:'
752024-02-01T17:27:45.198Z chmod input script dir contains:
762024-02-01T17:27:45.198Z+ ls -ltr /input/build/work/scripts
772024-02-01T17:27:45.202Ztotal 38
782024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 3056 Feb 1 17:27 perf-downstairs-tick.d
792024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 939 Feb 1 17:27 test_ds.sh
802024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 3746 Feb 1 17:27 test_live_repair.sh
812024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 6416 Feb 1 17:27 test_repair.sh
822024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 2969 Feb 1 17:27 test_replay.sh
832024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 9465 Feb 1 17:27 test_up.sh
842024-02-01T17:27:45.202Z-rwxr-xr-x 1 root root 2824 Feb 1 17:27 upstairs_info.d
852024-02-01T17:27:45.202Z+ banner unpack
862024-02-01T17:27:45.202Z
872024-02-01T17:27:45.202Z # # # # ##### ## #### # #
882024-02-01T17:27:45.202Z # # ## # # # # # # # # #
892024-02-01T17:27:45.202Z # # # # # # # # # # ####
902024-02-01T17:27:45.202Z # # # # # ##### ###### # # #
912024-02-01T17:27:45.202Z # # # ## # # # # # # #
922024-02-01T17:27:45.202Z #### # # # # # #### # #
932024-02-01T17:27:45.202Z
942024-02-01T17:27:45.206Z+ mkdir -p /var/tmp/bins
952024-02-01T17:27:45.206Z+ for t in "$input/bins/"*.gz
962024-02-01T17:27:45.206Z++ basename /input/build/work/bins/crucible-downstairs.gz
972024-02-01T17:27:45.209Z+ b=crucible-downstairs.gz
982024-02-01T17:27:45.209Z+ b=crucible-downstairs
992024-02-01T17:27:45.209Z+ gunzip
1002024-02-01T17:27:49.594Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012024-02-01T17:27:49.606Z+ for t in "$input/bins/"*.gz
1022024-02-01T17:27:49.606Z++ basename /input/build/work/bins/crucible-hammer.gz
1032024-02-01T17:27:49.606Z+ b=crucible-hammer.gz
1042024-02-01T17:27:49.606Z+ b=crucible-hammer
1052024-02-01T17:27:49.606Z+ gunzip
1062024-02-01T17:27:52.815Z+ chmod +x /var/tmp/bins/crucible-hammer
1072024-02-01T17:27:52.827Z+ for t in "$input/bins/"*.gz
1082024-02-01T17:27:52.827Z++ basename /input/build/work/bins/crutest.gz
1092024-02-01T17:27:52.827Z+ b=crutest.gz
1102024-02-01T17:27:52.828Z+ b=crutest
1112024-02-01T17:27:52.828Z+ gunzip
1122024-02-01T17:27:56.423Z+ chmod +x /var/tmp/bins/crutest
1132024-02-01T17:27:56.428Z+ for t in "$input/bins/"*.gz
1142024-02-01T17:27:56.428Z++ basename /input/build/work/bins/dsc.gz
1152024-02-01T17:27:56.434Z+ b=dsc.gz
1162024-02-01T17:27:56.434Z+ b=dsc
1172024-02-01T17:27:56.434Z+ gunzip
1182024-02-01T17:27:58.966Z+ chmod +x /var/tmp/bins/dsc
1192024-02-01T17:27:58.971Z+ export BINDIR=/var/tmp/bins
1202024-02-01T17:27:58.971Z+ BINDIR=/var/tmp/bins
1212024-02-01T17:27:58.971Z+ banner repair
1222024-02-01T17:27:58.971Z
1232024-02-01T17:27:58.971Z ##### ###### ##### ## # #####
1242024-02-01T17:27:58.971Z # # # # # # # # # #
1252024-02-01T17:27:58.971Z # # ##### # # # # # # #
1262024-02-01T17:27:58.971Z ##### # ##### ###### # #####
1272024-02-01T17:27:58.971Z # # # # # # # # #
1282024-02-01T17:27:58.971Z # # ###### # # # # # #
1292024-02-01T17:27:58.971Z
1302024-02-01T17:27:58.971Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312024-02-01T17:27:59.111ZTurn off color for downstairs dump
1322024-02-01T17:27:59.129ZCreating region directory at: /var/tmp/test_repair
1332024-02-01T17:27:59.132ZCreating dsc directory at: /tmp/dsc
1342024-02-01T17:27:59.773ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.40418395
1352024-02-01T17:27:59.809ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.05716667
1362024-02-01T17:27:59.809ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.057120573
1372024-02-01T17:27:59.809ZRegion set with 3 regions was created
1382024-02-01T17:27:59.809Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1131 1132 1133
1392024-02-01T17:27:59.809ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402024-02-01T17:27:59.809Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.973260479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
1412024-02-01T17:27:59.810Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.97444834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
1422024-02-01T17:27:59.810Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.974501636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
1432024-02-01T17:27:59.810Z{"msg":"Crucible e6e5014e-daf3-4cb4-bb9a-3c0abc5959e2 has session id: b2306837-845d-4622-95f2-b9b7f89e81b3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.974525556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1442024-02-01T17:27:59.810Z{"msg":"Upstairs opts: Upstairs UUID: e6e5014e-daf3-4cb4-bb9a-3c0abc5959e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.974555687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1452024-02-01T17:27:59.810Z{"msg":"Crucible stats registered with UUID: e6e5014e-daf3-4cb4-bb9a-3c0abc5959e2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.974578178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1462024-02-01T17:27:59.812ZCrucible runtime is spawned
1472024-02-01T17:27:59.812Z{"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.975424456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
1482024-02-01T17:27:59.812Z{"msg":"e6e5014e-daf3-4cb4-bb9a-3c0abc5959e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.975905467Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1492024-02-01T17:27:59.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.975969614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1502024-02-01T17:27:59.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.975999783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1512024-02-01T17:27:59.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.97604506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1522024-02-01T17:27:59.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.976077922Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1532024-02-01T17:27:59.813Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.976122469Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1542024-02-01T17:27:59.813Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.976147661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1552024-02-01T17:27:59.814Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.97618922Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1562024-02-01T17:27:59.814Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.976303938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1572024-02-01T17:27:59.814Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.976411361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1582024-02-01T17:27:59.817Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.977009092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1592024-02-01T17:27:59.817Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.977078871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1602024-02-01T17:27:59.817Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.977107277Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"io task","client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1612024-02-01T17:27:59.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981187914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1622024-02-01T17:27:59.817Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981260583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1632024-02-01T17:27:59.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981307167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1642024-02-01T17:27:59.817Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981349339Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1652024-02-01T17:27:59.817Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981391201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1662024-02-01T17:27:59.818Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.981456286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1672024-02-01T17:27:59.818Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.983530768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1682024-02-01T17:27:59.819Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.983600024Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1692024-02-01T17:27:59.819Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.983724047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1702024-02-01T17:27:59.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.98522888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1712024-02-01T17:27:59.819Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985315767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1722024-02-01T17:27:59.819Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985390054Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1732024-02-01T17:27:59.819Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.98545291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1742024-02-01T17:27:59.820Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985479699Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1752024-02-01T17:27:59.820Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985521472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1762024-02-01T17:27:59.820Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985543926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1772024-02-01T17:27:59.820Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985585019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1782024-02-01T17:27:59.820Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985831016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1792024-02-01T17:27:59.821Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.985882695Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1802024-02-01T17:27:59.821Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986168068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1812024-02-01T17:27:59.821Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986254956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1822024-02-01T17:27:59.821Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.98630497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1832024-02-01T17:27:59.821Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986331807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1842024-02-01T17:27:59.822Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986374082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1852024-02-01T17:27:59.822Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986399201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1862024-02-01T17:27:59.822Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986441393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1872024-02-01T17:27:59.822Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986472131Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1882024-02-01T17:27:59.822Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986515246Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1892024-02-01T17:27:59.822Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986539004Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1902024-02-01T17:27:59.823Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986578991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"":"downstairs","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1912024-02-01T17:27:59.823Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986603016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1922024-02-01T17:27:59.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986661224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"0","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1932024-02-01T17:27:59.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
1942024-02-01T17:27:59.823Z,"time":"2024-02-01T17:27:58.986702138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"1","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1952024-02-01T17:27:59.823Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.98677226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"client":"2","":"downstairs","session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1962024-02-01T17:27:59.823Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986816564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1972024-02-01T17:27:59.824Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.98686998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1982024-02-01T17:27:59.824Z{"msg":"e6e5014e-daf3-4cb4-bb9a-3c0abc5959e2 is now active with session: b2306837-845d-4622-95f2-b9b7f89e81b3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986911377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
1992024-02-01T17:27:59.824Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.986964559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135,"session_id":"b2306837-845d-4622-95f2-b9b7f89e81b3"}
2002024-02-01T17:27:59.824Z{"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:27:58.987004483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
2012024-02-01T17:27:59.825ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2022024-02-01T17:27:59.825ZFill test
2032024-02-01T17:27:59.976ZRead and Verify all blocks (0..600 range:false)
2042024-02-01T17:28:00.089ZWrote out file "/tmp/test_repair_verify.data"
2052024-02-01T17:28:00.089ZCLIENT: Tests done. All submitted work has been ACK'd
2062024-02-01T17:28:00.093Z----------------------------------------------------------------
2072024-02-01T17:28:00.093Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2082024-02-01T17:28:00.093ZStates: Active Active Active
2092024-02-01T17:28:00.093ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2102024-02-01T17:28:00.093Z 8 Acked 1007 Read 100 Done Done Done false
2112024-02-01T17:28:00.093Z 9 Acked 1008 Read 100 Done Done Done false
2122024-02-01T17:28:00.093Z 10 Acked 1009 Read 100 Done Done Done false
2132024-02-01T17:28:00.093Z 11 Acked 1010 Read 100 Done Done Done false
2142024-02-01T17:28:00.093Z 12 Acked 1011 Read 100 Done Done Done false
2152024-02-01T17:28:00.093Z 13 Acked 1012 Read 100 Done Done Done false
2162024-02-01T17:28:00.093Z STATES DS:0 DS:1 DS:2 TOTAL
2172024-02-01T17:28:00.093Z New 0 0 0 0
2182024-02-01T17:28:00.094Z Sent 0 0 0 0
2192024-02-01T17:28:00.094Z Done 6 6 6 18
2202024-02-01T17:28:00.094Z Skipped 0 0 0 0
2212024-02-01T17:28:00.094Z Error 0 0 0 0
2222024-02-01T17:28:00.094ZLast Flush: 1006 1006 1006
2232024-02-01T17:28:00.094ZDownstairs last five completed: 1006 1005 1004 1003 1002
2242024-02-01T17:28:00.094ZUpstairs last five completed: 13 12 11 10 9
2252024-02-01T17:28:00.094ZCLIENT: Up:0 ds:6 act:3
2262024-02-01T17:28:04.091Z----------------------------------------------------------------
2272024-02-01T17:28:04.092Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2282024-02-01T17:28:04.092ZDownstairs last five completed: 1013 1012 1011 1010 1009
2292024-02-01T17:28:04.092ZUpstairs last five completed: 14 13 12 11 10
2302024-02-01T17:28:04.092ZCLIENT: Up:0 ds:0 act:3
2312024-02-01T17:28:04.092ZCLIENT: All crucible jobs finished, exiting program
2322024-02-01T17:28:04.095Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:03.351609792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1135}
2332024-02-01T17:28:04.102Z
2342024-02-01T17:28:04.103ZBegin loop 1
2352024-02-01T17:28:04.103ZDownstairs to restart: 2
2362024-02-01T17:28:04.130Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.389366091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
2372024-02-01T17:28:04.138Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.389827477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
2382024-02-01T17:28:04.138Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.38992384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
2392024-02-01T17:28:04.139Z{"msg":"Crucible e681c7ec-6f44-4673-b6f1-bd31b365e495 has session id: 4103d94f-d3ac-4985-9c44-25d806caf313","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.389977252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2402024-02-01T17:28:04.139Z{"msg":"Crucible runtime is spawned
2412024-02-01T17:28:04.139ZUpstairs opts: Upstairs UUID: e681c7ec-6f44-4673-b6f1-bd31b365e495, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390094631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2422024-02-01T17:28:04.139Z{"msg":"Crucible stats registered with UUID: e681c7ec-6f44-4673-b6f1-bd31b365e495","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390164475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2432024-02-01T17:28:04.139Z{"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390203319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
2442024-02-01T17:28:04.139Z{"msg":"e681c7ec-6f44-4673-b6f1-bd31b365e495 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390258596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2452024-02-01T17:28:04.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390284712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2462024-02-01T17:28:04.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390380744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2472024-02-01T17:28:04.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390428775Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2482024-02-01T17:28:04.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390470078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2492024-02-01T17:28:04.139Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390574306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2502024-02-01T17:28:04.139Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390647966Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2512024-02-01T17:28:04.139Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.39069582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2522024-02-01T17:28:04.139Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390741911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2532024-02-01T17:28:04.139Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390916768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2542024-02-01T17:28:04.139Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.390998389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2552024-02-01T17:28:04.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.391074258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2562024-02-01T17:28:04.140Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.391157689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"io task","client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2572024-02-01T17:28:04.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.391871102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2582024-02-01T17:28:04.140Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.391964459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2592024-02-01T17:28:04.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.392121137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2602024-02-01T17:28:04.140Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.392206624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2612024-02-01T17:28:04.140Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.392824733Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2622024-02-01T17:28:04.140Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.392986259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2632024-02-01T17:28:04.140Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.39307128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2642024-02-01T17:28:04.140Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.393120896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2652024-02-01T17:28:04.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.393851183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2662024-02-01T17:28:04.142Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.393958111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2672024-02-01T17:28:04.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.394058798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2682024-02-01T17:28:04.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.394093948Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2692024-02-01T17:28:04.142Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.394123602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2702024-02-01T17:28:04.142Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.39418327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2712024-02-01T17:28:04.142Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.394368083Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2722024-02-01T17:28:04.142Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395067956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2732024-02-01T17:28:04.142Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395156989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2742024-02-01T17:28:04.142Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395234466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2752024-02-01T17:28:04.142Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395308605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2762024-02-01T17:28:04.142Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395382716Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2772024-02-01T17:28:04.142Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
2782024-02-01T17:28:04.142Z,"time":"2024-02-01T17:28:03.395456235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2792024-02-01T17:28:04.146Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395506249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2802024-02-01T17:28:04.147Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395583168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2812024-02-01T17:28:04.147Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395631561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2822024-02-01T17:28:04.147Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395706357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2832024-02-01T17:28:04.147Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395783119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2842024-02-01T17:28:04.147Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395827724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2852024-02-01T17:28:04.147Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.395897553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2862024-02-01T17:28:04.147Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.39596909Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2872024-02-01T17:28:04.147Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396017739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"":"downstairs","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2882024-02-01T17:28:04.147Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396086683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2892024-02-01T17:28:04.147Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396155216Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"0","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2902024-02-01T17:28:04.147Z{"msg"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2912024-02-01T17:28:04.147Z:"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396209791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"1","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2922024-02-01T17:28:04.147Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396272553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"client":"2","":"downstairs","session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2932024-02-01T17:28:04.147Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396300051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2942024-02-01T17:28:04.147Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396321506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2952024-02-01T17:28:04.147Z{"msg":"e681c7ec-6f44-4673-b6f1-bd31b365e495 is now active with session: 4103d94f-d3ac-4985-9c44-25d806caf313","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.39636222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2962024-02-01T17:28:04.147Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396384416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137,"session_id":"4103d94f-d3ac-4985-9c44-25d806caf313"}
2972024-02-01T17:28:04.147Z{"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:03.396405518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
2982024-02-01T17:28:04.147ZLoading write count information from file "/tmp/test_repair_verify.data"
2992024-02-01T17:28:04.147ZRun Repair workload
3002024-02-01T17:28:04.147Z01/30 Write block 428 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3012024-02-01T17:28:04.147Z02/30 Read block 18 len 12288
3022024-02-01T17:28:04.148Z03/30 Read block 232 len 8192
3032024-02-01T17:28:04.148Z04/30 Write block 540 len 24576 data: 2 2 2 2 2 2
3042024-02-01T17:28:04.148Z05/30 Write block 107 len 4096 data: 2
3052024-02-01T17:28:04.148Z06/30 Write block 374 len 36864 data: 2 2 2 2 2 2 2 2 2
3062024-02-01T17:28:04.154Z07/30 Read block 32 len 36864
3072024-02-01T17:28:04.154Z08/30 Read block 295 len 12288
3082024-02-01T17:28:04.154Z09/30 Write block 556 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3092024-02-01T17:28:04.157Z10/30 Flush
3102024-02-01T17:28:04.160Z11/30 Write block 287 len 20480 data: 2 2 2 2 2
3112024-02-01T17:28:04.161Z12/30 Flush
3122024-02-01T17:28:04.164Z13/30 Write block 458 len 12288 data: 2 2 2
3132024-02-01T17:28:04.164Z14/30 Write block 34 len 24576 data: 2 2 2 2 2 2
3142024-02-01T17:28:04.168Z15/30 Write block 532 len 32768 data: 2 2 2 2 2 2 2 2
3152024-02-01T17:28:04.168Z16/30 Read block 473 len 12288
3162024-02-01T17:28:04.168Z17/30 Write block 204 len 20480 data: 2 2 2 2 2
3172024-02-01T17:28:04.172Z18/30 Write block 387 len 28672 data: 2 2 2 2 2 2 2
3182024-02-01T17:28:04.172Z19/30 Read block 382 len 4096
3192024-02-01T17:28:04.172Z20/30 Read block 322 len 20480
3202024-02-01T17:28:04.175Z21/30 Read block 465 len 16384
3212024-02-01T17:28:04.175Z22/30 Read block 257 len 4096
3222024-02-01T17:28:04.175Z23/30 Read block 290 len 12288
3232024-02-01T17:28:04.178Z24/30 Read block 547 len 28672
3242024-02-01T17:28:04.178Z25/30 Flush
3252024-02-01T17:28:04.183Z26/30 Write block 10 len 36864 data: 2 2 2 2 2 2 2 2 2
3262024-02-01T17:28:04.186Z27/30 Read block 36 len 4096
3272024-02-01T17:28:04.186Z28/30 Read block 412 len 32768
3282024-02-01T17:28:04.186Z29/30 Read block 48 len 16384
3292024-02-01T17:28:04.200Z30/30 Write block 20 len 32768 data: 2 2 2 2 2 2 2 2
3302024-02-01T17:28:04.200Z----------------------------------------------------------------
3312024-02-01T17:28:04.200Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3322024-02-01T17:28:04.200ZStates: Active Active Active
3332024-02-01T17:28:04.200ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3342024-02-01T17:28:04.200Z 1 Acked 1000 Write 10 Done Done Sent false
3352024-02-01T17:28:04.200Z 2 Acked 1001 Read 3 Done Done Sent false
3362024-02-01T17:28:04.200Z 3 Acked 1002 Read 2 Done Done Sent false
3372024-02-01T17:28:04.200Z 4 Acked 1003 Write 6 Done Done Sent false
3382024-02-01T17:28:04.200Z 5 Acked 1004 Write 1 Done Done Sent false
3392024-02-01T17:28:04.200Z 6 Acked 1005 Write 9 Done Done Sent false
3402024-02-01T17:28:04.200Z 7 Acked 1006 Read 9 Done Done Sent false
3412024-02-01T17:28:04.200Z 8 Acked 1007 Read 3 Done Done Sent false
3422024-02-01T17:28:04.200Z 9 Acked 1008 Write 10 Done Done Sent false
3432024-02-01T17:28:04.200Z 10 Acked 1009 Flush 0 Done Done Sent false
3442024-02-01T17:28:04.200Z 11 Acked 1010 Write 5 Done Done Sent false
3452024-02-01T17:28:04.200Z 12 Acked 1011 Flush 0 Done Done Sent false
3462024-02-01T17:28:04.200Z 13 Acked 1012 Write 3 Done Done Sent false
3472024-02-01T17:28:04.200Z 14 Acked 1013 Write 6 Done Done Sent false
3482024-02-01T17:28:04.201Z 15 Acked 1014 Write 8 Done Done Sent false
3492024-02-01T17:28:04.201Z 16 Acked 1015 Read 3 Done Done Sent false
3502024-02-01T17:28:04.201Z 17 Acked 1016 Write 5 Done Done Sent false
3512024-02-01T17:28:04.201Z 18 Acked 1017 Write 7 Done Done Sent false
3522024-02-01T17:28:04.201Z 19 Acked 1018 Read 1 Done Done Sent false
3532024-02-01T17:28:04.201Z 20 Acked 1019 Read 5 Done Done Sent false
3542024-02-01T17:28:04.201Z 21 Acked 1020 Read 4 Done Done Sent false
3552024-02-01T17:28:04.201Z 22 Acked 1021 Read 1 Done Done Sent false
3562024-02-01T17:28:04.201Z 23 Acked 1022 Read 3 Done Done Sent false
3572024-02-01T17:28:04.201Z 24 Acked 1023 Read 7 Done Done Sent false
3582024-02-01T17:28:04.201Z 25 Acked 1024 Flush 0 Done Done Sent false
3592024-02-01T17:28:04.201Z 26 Acked 1025 Write 9 Done Done Sent false
3602024-02-01T17:28:04.201Z 27 Acked 1026 Read 1 Done Done Sent false
3612024-02-01T17:28:04.201Z 28 Acked 1027 Read 8 Done Done Sent false
3622024-02-01T17:28:04.202Z 29 Acked 1028 Read 4 Done Done Sent false
3632024-02-01T17:28:04.202Z 30 Acked 1029 Write 8 Sent Sent Sent false
3642024-02-01T17:28:04.202Z STATES DS:0 DS:1 DS:2 TOTAL
3652024-02-01T17:28:04.202Z New 0 0 0 0
3662024-02-01T17:28:04.202Z Sent 1 1 30 32
3672024-02-01T17:28:04.202Z Done 29 29 0 58
3682024-02-01T17:28:04.202Z Skipped 0 0 0 0
3692024-02-01T17:28:04.202Z Error 0 0 0 0
3702024-02-01T17:28:04.202ZLast Flush: 1024 1024 0
3712024-02-01T17:28:04.202ZDownstairs last five completed:
3722024-02-01T17:28:04.202ZUpstairs last five completed: 30 29 28 27 26
3732024-02-01T17:28:04.202Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:03.448313308Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1137}
3742024-02-01T17:28:04.202ZWrote out file "/tmp/test_repair_verify.data"
3752024-02-01T17:28:04.206Z
3762024-02-01T17:28:04.206ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
3772024-02-01T17:28:04.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
3782024-02-01T17:28:04.224Z 0 000-019 1 1 1 1 1 1 T T F <---
3792024-02-01T17:28:04.228Z 1 020-039 2 2 1 4 4 1 T T F <---
3802024-02-01T17:28:04.228Z 2 040-059 1 1 1 1 1 1 F F F
3812024-02-01T17:28:04.228Z 3 060-079 1 1 1 1 1 1 F F F
3822024-02-01T17:28:04.228Z 4 080-099 1 1 1 1 1 1 F F F
3832024-02-01T17:28:04.228Z 5 100-119 2 2 1 2 2 1 F F F <---
3842024-02-01T17:28:04.228Z 6 120-139 1 1 1 1 1 1 F F F
3852024-02-01T17:28:04.228Z 7 140-159 1 1 1 1 1 1 F F F
3862024-02-01T17:28:04.228Z 8 160-179 1 1 1 1 1 1 F F F
3872024-02-01T17:28:04.228Z 9 180-199 1 1 1 1 1 1 F F F
3882024-02-01T17:28:04.228Z 10 200-219 2 2 1 4 4 1 F F F <---
3892024-02-01T17:28:04.228Z 11 220-239 1 1 1 1 1 1 F F F
3902024-02-01T17:28:04.228Z 12 240-259 1 1 1 1 1 1 F F F
3912024-02-01T17:28:04.228Z 13 260-279 1 1 1 1 1 1 F F F
3922024-02-01T17:28:04.228Z 14 280-299 2 2 1 3 3 1 F F F <---
3932024-02-01T17:28:04.228Z 15 300-319 1 1 1 1 1 1 F F F
3942024-02-01T17:28:04.228Z 16 320-339 1 1 1 1 1 1 F F F
3952024-02-01T17:28:04.228Z 17 340-359 1 1 1 1 1 1 F F F
3962024-02-01T17:28:04.228Z 18 360-379 2 2 1 2 2 1 F F F <---
3972024-02-01T17:28:04.229Z 19 380-399 2 2 1 4 4 1 F F F <---
3982024-02-01T17:28:04.229Z 20 400-419 1 1 1 1 1 1 F F F
3992024-02-01T17:28:04.229Z 21 420-439 2 2 1 2 2 1 F F F <---
4002024-02-01T17:28:04.229Z 22 440-459 2 2 1 4 4 1 F F F <---
4012024-02-01T17:28:04.229Z 23 460-479 2 2 1 4 4 1 F F F <---
4022024-02-01T17:28:04.229Z 24 480-499 1 1 1 1 1 1 F F F
4032024-02-01T17:28:04.229Z 25 500-519 1 1 1 1 1 1 F F F
4042024-02-01T17:28:04.229Z 26 520-539 2 2 1 4 4 1 F F F <---
4052024-02-01T17:28:04.229Z 27 540-559 2 2 1 2 2 1 F F F <---
4062024-02-01T17:28:04.229Z 28 560-579 2 2 1 2 2 1 F F F <---
4072024-02-01T17:28:04.239Z 29 580-599 1 1 1 1 1 1 F F F
4082024-02-01T17:28:04.239ZMax gen: 2, Max flush: 4
4092024-02-01T17:28:04.239ZError: Difference in extent metadata found!
4102024-02-01T17:28:04.243ZA Difference in extent metadata is expected here
4112024-02-01T17:28:04.243ZOn loop 1
4122024-02-01T17:28:04.243Z
4132024-02-01T17:28:04.243ZVerifying data now
4142024-02-01T17:28:09.350ZNo change after verify
4152024-02-01T17:28:09.350ZLoop: 1 Downstairs dump after verify (and repair):
4162024-02-01T17:28:09.782ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4172024-02-01T17:28:09.782Z 0 000-019 3 3 3 5 5 5 F F F
4182024-02-01T17:28:09.782Z 1 020-039 3 3 3 5 5 5 F F F
4192024-02-01T17:28:09.782Z 2 040-059 1 1 1 1 1 1 F F F
4202024-02-01T17:28:09.791Z 3 060-079 1 1 1 1 1 1 F F F
4212024-02-01T17:28:09.791Z 4 080-099 1 1 1 1 1 1 F F F
4222024-02-01T17:28:09.791Z 5 100-119 2 2 2 2 2 2 F F F
4232024-02-01T17:28:09.791Z 6 120-139 1 1 1 1 1 1 F F F
4242024-02-01T17:28:09.791Z 7 140-159 1 1 1 1 1 1 F F F
4252024-02-01T17:28:09.791Z 8 160-179 1 1 1 1 1 1 F F F
4262024-02-01T17:28:09.791Z 9 180-199 1 1 1 1 1 1 F F F
4272024-02-01T17:28:09.791Z 10 200-219 2 2 2 4 4 4 F F F
4282024-02-01T17:28:09.793Z 11 220-239 1 1 1 1 1 1 F F F
4292024-02-01T17:28:09.793Z 12 240-259 1 1 1 1 1 1 F F F
4302024-02-01T17:28:09.793Z 13 260-279 1 1 1 1 1 1 F F F
4312024-02-01T17:28:09.793Z 14 280-299 2 2 2 3 3 3 F F F
4322024-02-01T17:28:09.793Z 15 300-319 1 1 1 1 1 1 F F F
4332024-02-01T17:28:09.793Z 16 320-339 1 1 1 1 1 1 F F F
4342024-02-01T17:28:09.793Z 17 340-359 1 1 1 1 1 1 F F F
4352024-02-01T17:28:09.793Z 18 360-379 2 2 2 2 2 2 F F F
4362024-02-01T17:28:09.793Z 19 380-399 2 2 2 4 4 4 F F F
4372024-02-01T17:28:09.793Z 20 400-419 1 1 1 1 1 1 F F F
4382024-02-01T17:28:09.793Z 21 420-439 2 2 2 2 2 2 F F F
4392024-02-01T17:28:09.793Z 22 440-459 2 2 2 4 4 4 F F F
4402024-02-01T17:28:09.793Z 23 460-479 2 2 2 4 4 4 F F F
4412024-02-01T17:28:09.793Z 24 480-499 1 1 1 1 1 1 F F F
4422024-02-01T17:28:09.793Z 25 500-519 1 1 1 1 1 1 F F F
4432024-02-01T17:28:09.793Z 26 520-539 2 2 2 4 4 4 F F F
4442024-02-01T17:28:09.793Z 27 540-559 2 2 2 2 2 2 F F F
4452024-02-01T17:28:09.793Z 28 560-579 2 2 2 2 2 2 F F F
4462024-02-01T17:28:09.793Z 29 580-599 1 1 1 1 1 1 F F F
4472024-02-01T17:28:09.794ZMax gen: 3, Max flush: 5
4482024-02-01T17:28:09.794Z
4492024-02-01T17:28:09.794ZBegin loop 2
4502024-02-01T17:28:09.794ZDownstairs to restart: 1
4512024-02-01T17:28:09.794Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.662362285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
4522024-02-01T17:28:09.794Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.662862872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
4532024-02-01T17:28:09.794Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66300685Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
4542024-02-01T17:28:09.794Z{"msg":"Crucible 35633d53-ad69-4353-848f-01c41680867c has session id: 11efb7a7-7f3b-47d7-b21d-f3c999257375","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663050485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4552024-02-01T17:28:09.794Z{"msg":"Upstairs opts: Upstairs UUID: 35633d53-ad69-4353-848f-01c41680867c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663090767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4562024-02-01T17:28:09.794Z{"msg":"Crucible stats registered with UUID: 35633d53-ad69-4353-848f-01c41680867c","v":0,"name":"crucible","level":30Crucible runtime is spawned
4572024-02-01T17:28:09.794Z,"time":"2024-02-01T17:28:08.663165637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4582024-02-01T17:28:09.794Z{"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663423866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
4592024-02-01T17:28:09.794Z{"msg":"35633d53-ad69-4353-848f-01c41680867c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663493801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4602024-02-01T17:28:09.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663538299Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4612024-02-01T17:28:09.794Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663582974Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4622024-02-01T17:28:09.794Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663659136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4632024-02-01T17:28:09.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663702255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4642024-02-01T17:28:09.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663774733Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4652024-02-01T17:28:09.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663818431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4662024-02-01T17:28:09.802Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66384317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4672024-02-01T17:28:09.802Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663866671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4682024-02-01T17:28:09.802Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.663897175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4692024-02-01T17:28:09.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.664111346Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4702024-02-01T17:28:09.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.664167059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4712024-02-01T17:28:09.802Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.664338285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"io task","client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4722024-02-01T17:28:09.802Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.665671579Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4732024-02-01T17:28:09.803Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.665779193Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4742024-02-01T17:28:09.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.665940907Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4752024-02-01T17:28:09.803Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.666007122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4762024-02-01T17:28:09.803Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.666032535Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4772024-02-01T17:28:09.804Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.666055082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4782024-02-01T17:28:09.804Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.666797996Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4792024-02-01T17:28:09.804Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.666878034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4802024-02-01T17:28:09.804Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.667363273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4812024-02-01T17:28:09.804Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.667844184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4822024-02-01T17:28:09.804Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.667917788Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4832024-02-01T17:28:09.805Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.667947385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4842024-02-01T17:28:09.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668057988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4852024-02-01T17:28:09.805Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668136031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4862024-02-01T17:28:09.805Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668197893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4872024-02-01T17:28:09.805Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668223956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4882024-02-01T17:28:09.805Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668265815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4892024-02-01T17:28:09.805Z{"msg":"[0]R flush_numbers[0..12]: [5, 5, 1, 1, 1, 2, 1, 1, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668291137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4902024-02-01T17:28:09.805Z{"msg":"[0]R generation[0..12]: [3, 3, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66833255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4912024-02-01T17:28:09.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668359105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4922024-02-01T17:28:09.805Z{"msg":"[1]R flush_numbers[0..12]: [5, 5, 1, 1, 1, 2, 1, 1, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668400689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4932024-02-01T17:28:09.806Z{"msg":"[1]R generation[0..12]: [3, 3, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668425241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4942024-02-01T17:28:09.806Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
4952024-02-01T17:28:09.806Z,"time":"2024-02-01T17:28:08.668451977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4962024-02-01T17:28:09.806Z{"msg":"[2]R flush_numbers[0..12]: [5, 5, 1, 1, 1, 2, 1, 1, 1, 1, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66847568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4972024-02-01T17:28:09.806Z{"msg":"[2]R generation[0..12]: [3, 3, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668518051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4982024-02-01T17:28:09.806Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668543663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
4992024-02-01T17:28:09.806Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668595754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5002024-02-01T17:28:09.806Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668621901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5012024-02-01T17:28:09.806Z{"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66866187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5022024-02-01T17:28:09.806Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668686545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"":"downstairs","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5032024-02-01T17:28:09.807Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668727436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5042024-02-01T17:28:09.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668749387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"0","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5052024-02-01T17:28:09.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668790323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"1","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5062024-02-01T17:28:09.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.66881438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"client":"2","":"downstairs","session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5072024-02-01T17:28:09.807Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668854964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5082024-02-01T17:28:09.807Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668877393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5092024-02-01T17:28:09.807Z{"msg":"35633d53-ad69-4353-848f-01c41680867c is now active with session: 11efb7a7-7f3b-47d7-b21d-f3c999257375","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668919197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5102024-02-01T17:28:09.807Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668941182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145,"session_id":"11efb7a7-7f3b-47d7-b21d-f3c999257375"}
5112024-02-01T17:28:09.808Z{"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:08.668984666Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
5122024-02-01T17:28:09.808ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5132024-02-01T17:28:09.808ZLoading write count information from file "/tmp/test_repair_verify.data"
5142024-02-01T17:28:09.808ZRun Repair workload
5152024-02-01T17:28:09.808Z01/30 Write block 152 len 16384 data: 2 2 2 2
5162024-02-01T17:28:09.808Z02/30 Write block 487 len 28672 data: 2 2 2 2 2 2 2
5172024-02-01T17:28:09.808Z03/30 Read block 290 len 20480
5182024-02-01T17:28:09.808Z04/30 Write block 181 len 24576 data: 2 2 2 2 2 2
5192024-02-01T17:28:09.808Z05/30 Write block 200 len 36864 data: 2 2 2 2 3 3 3 3 3
5202024-02-01T17:28:09.814Z06/30 Write block 373 len 36864 data: 2 3 3 3 3 3 3 3 3
5212024-02-01T17:28:09.814Z07/30 Read block 112 len 40960
5222024-02-01T17:28:09.815Z08/30 Write block 430 len 40960 data: 3 3 3 3 3 3 3 3 2 2
5232024-02-01T17:28:09.815Z09/30 Read block 92 len 8192
5242024-02-01T17:28:09.815Z10/30 Flush
5252024-02-01T17:28:09.815Z11/30 Write block 416 len 4096 data: 2
5262024-02-01T17:28:09.815Z12/30 Write block 57 len 4096 data: 2
5272024-02-01T17:28:09.815Z13/30 Write block 269 len 20480 data: 2 2 2 2 2
5282024-02-01T17:28:09.815Z14/30 Write block 242 len 8192 data: 2 2
5292024-02-01T17:28:09.815Z15/30 Write block 588 len 16384 data: 2 2 2 2
5302024-02-01T17:28:09.815Z16/30 Read block 6 len 32768
5312024-02-01T17:28:09.815Z17/30 Read block 272 len 16384
5322024-02-01T17:28:09.815Z18/30 Read block 334 len 28672
5332024-02-01T17:28:09.815Z19/30 Write block 211 len 4096 data: 2
5342024-02-01T17:28:09.815Z20/30 Write block 80 len 28672 data: 2 2 2 2 2 2 2
5352024-02-01T17:28:09.815Z21/30 Flush
5362024-02-01T17:28:09.815Z22/30 Write block 254 len 28672 data: 2 2 2 2 2 2 2
5372024-02-01T17:28:09.815Z23/30 Read block 17 len 20480
5382024-02-01T17:28:09.815Z24/30 Write block 449 len 36864 data: 2 2 2 2 2 2 2 2 2
5392024-02-01T17:28:09.815Z25/30 Read block 244 len 32768
5402024-02-01T17:28:09.818Z26/30 Read block 560 len 24576
5412024-02-01T17:28:09.818Z27/30 Write block 546 len 4096 data: 2
5422024-02-01T17:28:09.818Z28/30 Read block 582 len 4096
5432024-02-01T17:28:09.818Z29/30 Read block 193 len 4096
5442024-02-01T17:28:09.819Z30/30 Read block 100 len 16384
5452024-02-01T17:28:09.819Z----------------------------------------------------------------
5462024-02-01T17:28:09.819Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
5472024-02-01T17:28:09.819ZStates: Active Active Active
5482024-02-01T17:28:09.819ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
5492024-02-01T17:28:09.819Z 1 Acked 1000 Write 4 Done Done Done false
5502024-02-01T17:28:09.819Z 2 Acked 1001 Write 7 Done Done Done false
5512024-02-01T17:28:09.819Z 3 Acked 1002 Read 5 Done Sent Done false
5522024-02-01T17:28:09.819Z 4 Acked 1003 Write 6 Done Sent Done false
5532024-02-01T17:28:09.819Z 5 Acked 1004 Write 9 Done Sent Done false
5542024-02-01T17:28:09.819Z 6 Acked 1005 Write 9 Done Sent Done false
5552024-02-01T17:28:09.819Z 7 Acked 1006 Read 10 Done Sent Done false
5562024-02-01T17:28:09.819Z 8 Acked 1007 Write 10 Done Sent Done false
5572024-02-01T17:28:09.819Z 9 Acked 1008 Read 2 Done Sent Done false
5582024-02-01T17:28:09.819Z 10 Acked 1009 Flush 0 Done Sent Done false
5592024-02-01T17:28:09.819Z 11 Acked 1010 Write 1 Done Sent Done false
5602024-02-01T17:28:09.819Z 12 Acked 1011 Write 1 Done Sent Done false
5612024-02-01T17:28:09.819Z 13 Acked 1012 Write 5 Done Sent Done false
5622024-02-01T17:28:09.819Z 14 Acked 1013 Write 2 Done Sent Done false
5632024-02-01T17:28:09.819Z 15 Acked 1014 Write 4 Done Sent Done false
5642024-02-01T17:28:09.819Z 16 Acked 1015 Read 8 Done Sent Done false
5652024-02-01T17:28:09.819Z 17 Acked 1016 Read 4 Done Sent Done false
5662024-02-01T17:28:09.819Z 18 Acked 1017 Read 7 Done Sent Done false
5672024-02-01T17:28:09.819Z 19 Acked 1018 Write 1 Done Sent Done false
5682024-02-01T17:28:09.820Z 20 Acked 1019 Write 7 Done Sent Done false
5692024-02-01T17:28:09.820Z 21 Acked 1020 Flush 0 Done Sent Done false
5702024-02-01T17:28:09.820Z 22 Acked 1021 Write 7 Done Sent Done false
5712024-02-01T17:28:09.820Z 23 Acked 1022 Read 5 Done Sent Done false
5722024-02-01T17:28:09.820Z 24 Acked 1023 Write 9 Done Sent Done false
5732024-02-01T17:28:09.820Z 25 Acked 1024 Read 8 Done Sent Done false
5742024-02-01T17:28:09.820Z 26 Acked 1025 Read 6 Done Sent Done false
5752024-02-01T17:28:09.820Z 27 Acked 1026 Write 1 Done Sent Done false
5762024-02-01T17:28:09.820Z 28 Acked 1027 Read 1 Done Sent Done false
5772024-02-01T17:28:09.820Z 29 Acked 1028 Read 1 Done Sent Done false
5782024-02-01T17:28:09.820Z 30 Acked 1029 Read 4 Done Sent Done false
5792024-02-01T17:28:09.820Z STATES DS:0 DS:1 DS:2 TOTAL
5802024-02-01T17:28:09.820Z New 0 0 0 0
5812024-02-01T17:28:09.820Z Sent 0 28 0 28
5822024-02-01T17:28:09.820Z Done 30 2 30 62
5832024-02-01T17:28:09.820Z Skipped 0 0 0 0
5842024-02-01T17:28:09.820Z Error 0 0 0 0
5852024-02-01T17:28:09.820ZLast Flush: 1020 0 1020
5862024-02-01T17:28:09.820ZDownstairs last five completed:
5872024-02-01T17:28:09.820ZUpstairs last five completed: 30 29 28 27 26
5882024-02-01T17:28:09.822Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:08.715721453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1145}
5892024-02-01T17:28:09.822ZWrote out file "/tmp/test_repair_verify.data"
5902024-02-01T17:28:09.822Z
5912024-02-01T17:28:09.822ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
5922024-02-01T17:28:09.822ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
5932024-02-01T17:28:09.822Z 0 000-019 3 3 3 5 5 5 F F F
5942024-02-01T17:28:09.822Z 1 020-039 3 3 3 5 5 5 F F F
5952024-02-01T17:28:09.822Z 2 040-059 4 1 4 7 1 7 F F F <---
5962024-02-01T17:28:09.822Z 3 060-079 1 1 1 1 1 1 F F F
5972024-02-01T17:28:09.822Z 4 080-099 4 1 4 7 1 7 F F F <---
5982024-02-01T17:28:09.822Z 5 100-119 2 2 2 2 2 2 F F F
5992024-02-01T17:28:09.822Z 6 120-139 1 1 1 1 1 1 F F F
6002024-02-01T17:28:09.823Z 7 140-159 4 1 4 6 1 6 F T F <---
6012024-02-01T17:28:09.823Z 8 160-179 1 1 1 1 1 1 F F F
6022024-02-01T17:28:09.823Z 9 180-199 4 1 4 6 1 6 F F F <---
6032024-02-01T17:28:09.823Z 10 200-219 4 2 4 7 4 7 F F F <---
6042024-02-01T17:28:09.823Z 11 220-239 1 1 1 1 1 1 F F F
6052024-02-01T17:28:09.823Z 12 240-259 4 1 4 7 1 7 T F T <---
6062024-02-01T17:28:09.823Z 13 260-279 4 1 4 7 1 7 T F T <---
6072024-02-01T17:28:09.823Z 14 280-299 2 2 2 3 3 3 F F F
6082024-02-01T17:28:09.823Z 15 300-319 1 1 1 1 1 1 F F F
6092024-02-01T17:28:09.823Z 16 320-339 1 1 1 1 1 1 F F F
6102024-02-01T17:28:09.823Z 17 340-359 1 1 1 1 1 1 F F F
6112024-02-01T17:28:09.823Z 18 360-379 4 2 4 6 2 6 F F F <---
6122024-02-01T17:28:09.826Z 19 380-399 4 2 4 6 4 6 F F F <---
6132024-02-01T17:28:09.826Z 20 400-419 4 1 4 7 1 7 F F F <---
6142024-02-01T17:28:09.826Z 21 420-439 4 2 4 6 2 6 F F F <---
6152024-02-01T17:28:09.827Z 22 440-459 2 2 2 4 4 4 T F T <---
6162024-02-01T17:28:09.827Z 23 460-479 2 2 2 4 4 4 F F F
6172024-02-01T17:28:09.827Z 24 480-499 4 1 4 6 1 6 F T F <---
6182024-02-01T17:28:09.827Z 25 500-519 1 1 1 1 1 1 F F F
6192024-02-01T17:28:09.827Z 26 520-539 2 2 2 4 4 4 F F F
6202024-02-01T17:28:09.830Z 27 540-559 2 2 2 2 2 2 T F T <---
6212024-02-01T17:28:09.830Z 28 560-579 2 2 2 2 2 2 F F F
6222024-02-01T17:28:09.830Z 29 580-599 4 1 4 7 1 7 F F F <---
6232024-02-01T17:28:09.830ZMax gen: 4, Max flush: 7
6242024-02-01T17:28:09.830ZError: Difference in extent metadata found!
6252024-02-01T17:28:09.830ZA Difference in extent metadata is expected here
6262024-02-01T17:28:09.830ZOn loop 2
6272024-02-01T17:28:09.830Z
6282024-02-01T17:28:09.830ZVerifying data now
6292024-02-01T17:28:15.758ZNo change after verify
6302024-02-01T17:28:15.758ZLoop: 2 Downstairs dump after verify (and repair):
6312024-02-01T17:28:16.199ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6322024-02-01T17:28:16.199Z 0 000-019 3 3 3 5 5 5 F F F
6332024-02-01T17:28:16.207Z 1 020-039 3 3 3 5 5 5 F F F
6342024-02-01T17:28:16.207Z 2 040-059 4 4 4 7 7 7 F F F
6352024-02-01T17:28:16.207Z 3 060-079 1 1 1 1 1 1 F F F
6362024-02-01T17:28:16.207Z 4 080-099 4 4 4 7 7 7 F F F
6372024-02-01T17:28:16.207Z 5 100-119 2 2 2 2 2 2 F F F
6382024-02-01T17:28:16.207Z 6 120-139 1 1 1 1 1 1 F F F
6392024-02-01T17:28:16.207Z 7 140-159 4 4 4 6 6 6 F F F
6402024-02-01T17:28:16.207Z 8 160-179 1 1 1 1 1 1 F F F
6412024-02-01T17:28:16.208Z 9 180-199 4 4 4 6 6 6 F F F
6422024-02-01T17:28:16.208Z 10 200-219 4 4 4 7 7 7 F F F
6432024-02-01T17:28:16.208Z 11 220-239 1 1 1 1 1 1 F F F
6442024-02-01T17:28:16.208Z 12 240-259 5 5 5 8 8 8 F F F
6452024-02-01T17:28:16.208Z 13 260-279 5 5 5 8 8 8 F F F
6462024-02-01T17:28:16.208Z 14 280-299 2 2 2 3 3 3 F F F
6472024-02-01T17:28:16.208Z 15 300-319 1 1 1 1 1 1 F F F
6482024-02-01T17:28:16.208Z 16 320-339 1 1 1 1 1 1 F F F
6492024-02-01T17:28:16.208Z 17 340-359 1 1 1 1 1 1 F F F
6502024-02-01T17:28:16.208Z 18 360-379 4 4 4 6 6 6 F F F
6512024-02-01T17:28:16.208Z 19 380-399 4 4 4 6 6 6 F F F
6522024-02-01T17:28:16.208Z 20 400-419 4 4 4 7 7 7 F F F
6532024-02-01T17:28:16.208Z 21 420-439 4 4 4 6 6 6 F F F
6542024-02-01T17:28:16.209Z 22 440-459 5 5 5 8 8 8 F F F
6552024-02-01T17:28:16.209Z 23 460-479 2 2 2 4 4 4 F F F
6562024-02-01T17:28:16.209Z 24 480-499 4 4 4 6 6 6 F F F
6572024-02-01T17:28:16.209Z 25 500-519 1 1 1 1 1 1 F F F
6582024-02-01T17:28:16.209Z 26 520-539 2 2 2 4 4 4 F F F
6592024-02-01T17:28:16.209Z 27 540-559 5 5 5 8 8 8 F F F
6602024-02-01T17:28:16.209Z 28 560-579 2 2 2 2 2 2 F F F
6612024-02-01T17:28:16.211Z 29 580-599 4 4 4 7 7 7 F F F
6622024-02-01T17:28:16.211ZMax gen: 5, Max flush: 8
6632024-02-01T17:28:16.211Z
6642024-02-01T17:28:16.211ZBegin loop 3
6652024-02-01T17:28:16.211ZDownstairs to restart: 2
6662024-02-01T17:28:16.211Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.075316285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
6672024-02-01T17:28:16.211Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.075782733Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
6682024-02-01T17:28:16.212Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.075867975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
6692024-02-01T17:28:16.212Z{"msg":"Crucible 2fdf93c9-314d-4ef2-99e6-a589a7a1e88a has session id: 8c303732-33a5-45aa-9e29-dcb17d50164f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.075905161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6702024-02-01T17:28:16.212Z{Crucible runtime is spawned
6712024-02-01T17:28:16.212Z"msg":"Upstairs opts: Upstairs UUID: 2fdf93c9-314d-4ef2-99e6-a589a7a1e88a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076029184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6722024-02-01T17:28:16.212Z{"msg":"Crucible stats registered with UUID: 2fdf93c9-314d-4ef2-99e6-a589a7a1e88a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076123714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6732024-02-01T17:28:16.212Z{"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076152787Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
6742024-02-01T17:28:16.212Z{"msg":"2fdf93c9-314d-4ef2-99e6-a589a7a1e88a active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076326351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6752024-02-01T17:28:16.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.07639832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6762024-02-01T17:28:16.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.07649255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6772024-02-01T17:28:16.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076538801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6782024-02-01T17:28:16.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076585201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6792024-02-01T17:28:16.213Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076627055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6802024-02-01T17:28:16.213Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076733088Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6812024-02-01T17:28:16.213Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076901281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6822024-02-01T17:28:16.213Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.076950095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6832024-02-01T17:28:16.214Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.077106426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6842024-02-01T17:28:16.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.077170508Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6852024-02-01T17:28:16.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.077197746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6862024-02-01T17:28:16.214Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.077225409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"io task","client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6872024-02-01T17:28:16.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078229626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6882024-02-01T17:28:16.214Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078345832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6892024-02-01T17:28:16.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078392843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6902024-02-01T17:28:16.214Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078473575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6912024-02-01T17:28:16.214Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078507216Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6922024-02-01T17:28:16.214Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.078581711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6932024-02-01T17:28:16.214Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.079212793Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6942024-02-01T17:28:16.214Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.079415124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6952024-02-01T17:28:16.214Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.079560323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6962024-02-01T17:28:16.214Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080107721Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6972024-02-01T17:28:16.215Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080166835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6982024-02-01T17:28:16.215Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080341071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
6992024-02-01T17:28:16.215Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.08052738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7002024-02-01T17:28:16.215Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080606122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7012024-02-01T17:28:16.215Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.08070886Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7022024-02-01T17:28:16.216Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080760935Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7032024-02-01T17:28:16.216Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080804919Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7042024-02-01T17:28:16.216Z{"msg":"[0]R flush_numbers[0..12]: [5, 5, 7, 1, 7, 2, 1, 6, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080862833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7052024-02-01T17:28:16.216Z{Wait for a query_work_queue command to finish before sending IO
7062024-02-01T17:28:16.216Z"msg":"[0]R generation[0..12]: [3, 3, 4, 1, 4, 2, 1, 4, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.080983023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7072024-02-01T17:28:16.216Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081060944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7082024-02-01T17:28:16.216Z{"msg":"[1]R flush_numbers[0..12]: [5, 5, 7, 1, 7, 2, 1, 6, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081139711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7092024-02-01T17:28:16.216Z{"msg":"[1]R generation[0..12]: [3, 3, 4, 1, 4, 2, 1, 4, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081184473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7102024-02-01T17:28:16.216Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081241938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7112024-02-01T17:28:16.216Z{"msg":"[2]R flush_numbers[0..12]: [5, 5, 7, 1, 7, 2, 1, 6, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081318301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7122024-02-01T17:28:16.216Z{"msg":"[2]R generation[0..12]: [3, 3, 4, 1, 4, 2, 1, 4, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081374387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7132024-02-01T17:28:16.216Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.08141986Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7142024-02-01T17:28:16.216Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081483852Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7152024-02-01T17:28:16.218Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081528185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7162024-02-01T17:28:16.218Z{"msg":"Next flush: 9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081568531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7172024-02-01T17:28:16.218Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081636506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"":"downstairs","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7182024-02-01T17:28:16.218Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081680405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7192024-02-01T17:28:16.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081732307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
7202024-02-01T17:28:16.218Z,"pid":1153,"client":"0","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7212024-02-01T17:28:16.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081779728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"1","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7222024-02-01T17:28:16.218Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081832139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"client":"2","":"downstairs","session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7232024-02-01T17:28:16.218Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081875499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7242024-02-01T17:28:16.218Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081926276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7252024-02-01T17:28:16.218Z{"msg":"2fdf93c9-314d-4ef2-99e6-a589a7a1e88a is now active with session: 8c303732-33a5-45aa-9e29-dcb17d50164f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.081967509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7262024-02-01T17:28:16.218Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.082022195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153,"session_id":"8c303732-33a5-45aa-9e29-dcb17d50164f"}
7272024-02-01T17:28:16.218Z{"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:15.082062293Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
7282024-02-01T17:28:16.218ZLoading write count information from file "/tmp/test_repair_verify.data"
7292024-02-01T17:28:16.218ZRun Repair workload
7302024-02-01T17:28:16.218Z01/30 Write block 541 len 24576 data: 3 3 3 3 3 3
7312024-02-01T17:28:16.218Z02/30 Read block 489 len 28672
7322024-02-01T17:28:16.219Z03/30 Read block 478 len 24576
7332024-02-01T17:28:16.225Z04/30 Write block 389 len 20480 data: 3 3 3 3 3
7342024-02-01T17:28:16.225Z05/30 Read block 214 len 4096
7352024-02-01T17:28:16.225Z06/30 Write block 146 len 28672 data: 2 2 2 2 2 2 3
7362024-02-01T17:28:16.225Z07/30 Read block 498 len 16384
7372024-02-01T17:28:16.225Z08/30 Write block 354 len 4096 data: 2
7382024-02-01T17:28:16.225Z09/30 Read block 249 len 32768
7392024-02-01T17:28:16.225Z10/30 Write block 153 len 4096 data: 3
7402024-02-01T17:28:16.225Z11/30 Write block 122 len 40960 data: 2 2 2 2 2 2 2 2 2 2
7412024-02-01T17:28:16.225Z12/30 Read block 176 len 28672
7422024-02-01T17:28:16.225Z13/30 Flush
7432024-02-01T17:28:16.225Z14/30 Flush
7442024-02-01T17:28:16.225Z15/30 Write block 331 len 24576 data: 2 2 2 2 2 2
7452024-02-01T17:28:16.225Z16/30 Write block 530 len 12288 data: 2 2 3
7462024-02-01T17:28:16.226Z17/30 Write block 148 len 8192 data: 3 3
7472024-02-01T17:28:16.226Z18/30 Read block 320 len 8192
7482024-02-01T17:28:16.228Z19/30 Write block 358 len 8192 data: 2 2
7492024-02-01T17:28:16.228Z20/30 Read block 114 len 12288
7502024-02-01T17:28:16.228Z21/30 Read block 479 len 4096
7512024-02-01T17:28:16.228Z22/30 Read block 120 len 32768
7522024-02-01T17:28:16.228Z23/30 Write block 30 len 12288 data: 2 2 2
7532024-02-01T17:28:16.228Z24/30 Write block 334 len 12288 data: 3 3 3
7542024-02-01T17:28:16.228Z25/30 Read block 468 len 20480
7552024-02-01T17:28:16.228Z26/30 Write block 548 len 32768 data: 2 2 2 2 2 2 2 2
7562024-02-01T17:28:16.229Z27/30 Write block 432 len 24576 data: 4 4 4 4 4 4
7572024-02-01T17:28:16.229Z28/30 Read block 25 len 24576
7582024-02-01T17:28:16.229Z29/30 Write block 534 len 16384 data: 3 3 3 3
7592024-02-01T17:28:16.229Z30/30 Read block 570 len 36864
7602024-02-01T17:28:16.229Z----------------------------------------------------------------
7612024-02-01T17:28:16.229Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
7622024-02-01T17:28:16.229ZStates: Active Active Active
7632024-02-01T17:28:16.229ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
7642024-02-01T17:28:16.229Z 1 Acked 1000 Write 6 Done Done Done false
7652024-02-01T17:28:16.229Z 2 Acked 1001 Read 7 Done Done Done false
7662024-02-01T17:28:16.229Z 3 Acked 1002 Read 6 Done Done Done false
7672024-02-01T17:28:16.229Z 4 Acked 1003 Write 5 Done Done Done false
7682024-02-01T17:28:16.229Z 5 Acked 1004 Read 1 Done Done Done false
7692024-02-01T17:28:16.229Z 6 Acked 1005 Write 7 Done Done Done false
7702024-02-01T17:28:16.229Z 7 Acked 1006 Read 4 Done Done Done false
7712024-02-01T17:28:16.229Z 8 Acked 1007 Write 1 Done Done Sent false
7722024-02-01T17:28:16.229Z 9 Acked 1008 Read 8 Done Done Sent false
7732024-02-01T17:28:16.229Z 10 Acked 1009 Write 1 Done Done Sent false
7742024-02-01T17:28:16.229Z 11 Acked 1010 Write 10 Done Done Sent false
7752024-02-01T17:28:16.229Z 12 Acked 1011 Read 7 Done Done Sent false
7762024-02-01T17:28:16.229Z 13 Acked 1012 Flush 0 Done Done Sent false
7772024-02-01T17:28:16.229Z 14 Acked 1013 Flush 0 Done Done Sent false
7782024-02-01T17:28:16.230Z 15 Acked 1014 Write 6 Done Done Sent false
7792024-02-01T17:28:16.230Z 16 Acked 1015 Write 3 Done Done Sent false
7802024-02-01T17:28:16.230Z 17 Acked 1016 Write 2 Done Done Sent false
7812024-02-01T17:28:16.230Z 18 Acked 1017 Read 2 Done Done Sent false
7822024-02-01T17:28:16.230Z 19 Acked 1018 Write 2 Done Done Sent false
7832024-02-01T17:28:16.230Z 20 Acked 1019 Read 3 Done Done Sent false
7842024-02-01T17:28:16.230Z 21 Acked 1020 Read 1 Done Done Sent false
7852024-02-01T17:28:16.230Z 22 Acked 1021 Read 8 Done Done Sent false
7862024-02-01T17:28:16.230Z 23 Acked 1022 Write 3 Done Done Sent false
7872024-02-01T17:28:16.230Z 24 Acked 1023 Write 3 Done Done Sent false
7882024-02-01T17:28:16.230Z 25 Acked 1024 Read 5 Done Done Sent false
7892024-02-01T17:28:16.230Z 26 Acked 1025 Write 8 Done Done Sent false
7902024-02-01T17:28:16.230Z 27 Acked 1026 Write 6 Done Done Sent false
7912024-02-01T17:28:16.230Z 28 Acked 1027 Read 6 Done Done Sent false
7922024-02-01T17:28:16.230Z 29 Acked 1028 Write 4 Done Done Sent false
7932024-02-01T17:28:16.230Z 30 Acked 1029 Read 9 Sent Done Sent false
7942024-02-01T17:28:16.232Z STATES DS:0 DS:1 DS:2 TOTAL
7952024-02-01T17:28:16.232Z New 0 0 0 0
7962024-02-01T17:28:16.232Z Sent 1 0 23 24
7972024-02-01T17:28:16.232Z Done 29 30 7 66
7982024-02-01T17:28:16.233Z Skipped 0 0 0 0
7992024-02-01T17:28:16.233Z Error 0 0 0 0
8002024-02-01T17:28:16.233ZLast Flush: 1013 1013 0
8012024-02-01T17:28:16.233ZDownstairs last five completed:
8022024-02-01T17:28:16.233ZUpstairs last five completed: 30 29 28 27 26
8032024-02-01T17:28:16.233Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:15.123683944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1153}
8042024-02-01T17:28:16.233ZWrote out file "/tmp/test_repair_verify.data"
8052024-02-01T17:28:16.233Z
8062024-02-01T17:28:16.233ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
8072024-02-01T17:28:16.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8082024-02-01T17:28:16.233Z 0 000-019 3 3 3 5 5 5 F F F
8092024-02-01T17:28:16.233Z 1 020-039 3 3 3 5 5 5 T T F <---
8102024-02-01T17:28:16.233Z 2 040-059 4 4 4 7 7 7 F F F
8112024-02-01T17:28:16.233Z 3 060-079 1 1 1 1 1 1 F F F
8122024-02-01T17:28:16.233Z 4 080-099 4 4 4 7 7 7 F F F
8132024-02-01T17:28:16.233Z 5 100-119 2 2 2 2 2 2 F F F
8142024-02-01T17:28:16.233Z 6 120-139 6 6 1 9 9 1 F F F <---
8152024-02-01T17:28:16.233Z 7 140-159 6 6 4 9 9 6 T T T <---
8162024-02-01T17:28:16.233Z 8 160-179 1 1 1 1 1 1 F F F
8172024-02-01T17:28:16.233Z 9 180-199 4 4 4 6 6 6 F F F
8182024-02-01T17:28:16.233Z 10 200-219 4 4 4 7 7 7 F F F
8192024-02-01T17:28:16.233Z 11 220-239 1 1 1 1 1 1 F F F
8202024-02-01T17:28:16.233Z 12 240-259 5 5 5 8 8 8 F F F
8212024-02-01T17:28:16.234Z 13 260-279 5 5 5 8 8 8 F F F
8222024-02-01T17:28:16.234Z 14 280-299 2 2 2 3 3 3 F F F
8232024-02-01T17:28:16.234Z 15 300-319 1 1 1 1 1 1 F F F
8242024-02-01T17:28:16.234Z 16 320-339 1 1 1 1 1 1 T T F <---
8252024-02-01T17:28:16.234Z 17 340-359 6 6 1 9 9 1 T T F <---
8262024-02-01T17:28:16.234Z 18 360-379 4 4 4 6 6 6 F F F
8272024-02-01T17:28:16.234Z 19 380-399 6 6 4 9 9 6 F F T <---
8282024-02-01T17:28:16.234Z 20 400-419 4 4 4 7 7 7 F F F
8292024-02-01T17:28:16.234Z 21 420-439 4 4 4 6 6 6 T T F <---
8302024-02-01T17:28:16.234Z 22 440-459 5 5 5 8 8 8 F F F
8312024-02-01T17:28:16.234Z 23 460-479 2 2 2 4 4 4 F F F
8322024-02-01T17:28:16.234Z 24 480-499 4 4 4 6 6 6 F F F
8332024-02-01T17:28:16.238Z 25 500-519 1 1 1 1 1 1 F F F
8342024-02-01T17:28:16.238Z 26 520-539 2 2 2 4 4 4 T T F <---
8352024-02-01T17:28:16.238Z 27 540-559 6 6 5 9 9 8 T T T <---
8362024-02-01T17:28:16.238Z 28 560-579 2 2 2 2 2 2 F F F
8372024-02-01T17:28:16.238Z 29 580-599 4 4 4 7 7 7 F F F
8382024-02-01T17:28:16.238ZMax gen: 6, Max flush: 9
8392024-02-01T17:28:16.238ZError: Difference in extent metadata found!
8402024-02-01T17:28:16.238ZA Difference in extent metadata is expected here
8412024-02-01T17:28:16.238ZOn loop 3
8422024-02-01T17:28:16.238Z
8432024-02-01T17:28:16.238ZVerifying data now
8442024-02-01T17:28:21.748ZNo change after verify
8452024-02-01T17:28:21.748ZLoop: 3 Downstairs dump after verify (and repair):
8462024-02-01T17:28:21.757ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
8472024-02-01T17:28:21.758Z 0 000-019 3 3 3 5 5 5 F F F
8482024-02-01T17:28:21.765Z 1 020-039 7 7 7 10 10 10 F F F
8492024-02-01T17:28:21.765Z 2 040-059 4 4 4 7 7 7 F F F
8502024-02-01T17:28:21.766Z 3 060-079 1 1 1 1 1 1 F F F
8512024-02-01T17:28:21.766Z 4 080-099 4 4 4 7 7 7 F F F
8522024-02-01T17:28:21.766Z 5 100-119 2 2 2 2 2 2 F F F
8532024-02-01T17:28:21.766Z 6 120-139 6 6 6 9 9 9 F F F
8542024-02-01T17:28:21.766Z 7 140-159 7 7 7 10 10 10 F F F
8552024-02-01T17:28:21.766Z 8 160-179 1 1 1 1 1 1 F F F
8562024-02-01T17:28:21.768Z 9 180-199 4 4 4 6 6 6 F F F
8572024-02-01T17:28:21.768Z 10 200-219 4 4 4 7 7 7 F F F
8582024-02-01T17:28:21.768Z 11 220-239 1 1 1 1 1 1 F F F
8592024-02-01T17:28:21.768Z 12 240-259 5 5 5 8 8 8 F F F
8602024-02-01T17:28:21.768Z 13 260-279 5 5 5 8 8 8 F F F
8612024-02-01T17:28:21.768Z 14 280-299 2 2 2 3 3 3 F F F
8622024-02-01T17:28:21.768Z 15 300-319 1 1 1 1 1 1 F F F
8632024-02-01T17:28:21.768Z 16 320-339 7 7 7 10 10 10 F F F
8642024-02-01T17:28:21.768Z 17 340-359 7 7 7 10 10 10 F F F
8652024-02-01T17:28:21.768Z 18 360-379 4 4 4 6 6 6 F F F
8662024-02-01T17:28:21.768Z 19 380-399 6 6 6 9 9 9 F F F
8672024-02-01T17:28:21.769Z 20 400-419 4 4 4 7 7 7 F F F
8682024-02-01T17:28:21.769Z 21 420-439 7 7 7 10 10 10 F F F
8692024-02-01T17:28:21.769Z 22 440-459 5 5 5 8 8 8 F F F
8702024-02-01T17:28:21.769Z 23 460-479 2 2 2 4 4 4 F F F
8712024-02-01T17:28:21.769Z 24 480-499 4 4 4 6 6 6 F F F
8722024-02-01T17:28:21.769Z 25 500-519 1 1 1 1 1 1 F F F
8732024-02-01T17:28:21.769Z 26 520-539 7 7 7 10 10 10 F F F
8742024-02-01T17:28:21.769Z 27 540-559 7 7 7 10 10 10 F F F
8752024-02-01T17:28:21.769Z 28 560-579 2 2 2 2 2 2 F F F
8762024-02-01T17:28:21.769Z 29 580-599 4 4 4 7 7 7 F F F
8772024-02-01T17:28:21.769ZMax gen: 7, Max flush: 10
8782024-02-01T17:28:21.769Z
8792024-02-01T17:28:21.769ZBegin loop 4
8802024-02-01T17:28:21.769ZDownstairs to restart: 0
8812024-02-01T17:28:21.795Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.049400404Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
8822024-02-01T17:28:21.804Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.049970438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
8832024-02-01T17:28:21.804Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050040628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
8842024-02-01T17:28:21.804Z{"msg":"Crucible ddb82b6d-5a14-4e72-917e-5059a644723c has session id: 2c2ad4ff-0b85-4536-b998-69912ef4d8e6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050092491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8852024-02-01T17:28:21.804ZCrucible runtime is spawned
8862024-02-01T17:28:21.804Z{"msg":"Upstairs opts: Upstairs UUID: ddb82b6d-5a14-4e72-917e-5059a644723c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050233324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8872024-02-01T17:28:21.804Z{"msg":"Crucible stats registered with UUID: ddb82b6d-5a14-4e72-917e-5059a644723c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050296874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8882024-02-01T17:28:21.805Z{"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050338566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
8892024-02-01T17:28:21.805Z{"msg":"ddb82b6d-5a14-4e72-917e-5059a644723c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050396825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8902024-02-01T17:28:21.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050437353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8912024-02-01T17:28:21.805Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050537893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8922024-02-01T17:28:21.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050583329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8932024-02-01T17:28:21.805Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050623904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8942024-02-01T17:28:21.805Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050659399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8952024-02-01T17:28:21.805Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050705834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8962024-02-01T17:28:21.806Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050763473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8972024-02-01T17:28:21.806Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050810234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8982024-02-01T17:28:21.806Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050855412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
8992024-02-01T17:28:21.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050899085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9002024-02-01T17:28:21.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.050945319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9012024-02-01T17:28:21.806Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.051031674Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"io task","client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9022024-02-01T17:28:21.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.052233359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9032024-02-01T17:28:21.806Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.052304043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9042024-02-01T17:28:21.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.05234854Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9052024-02-01T17:28:21.806Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.052463064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9062024-02-01T17:28:21.807Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.053261113Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9072024-02-01T17:28:21.807Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.053352624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9082024-02-01T17:28:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.053405681Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9092024-02-01T17:28:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.053492184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9102024-02-01T17:28:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.05428764Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9112024-02-01T17:28:21.807Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054349777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9122024-02-01T17:28:21.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054376941Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9132024-02-01T17:28:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054437668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9142024-02-01T17:28:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054464536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9152024-02-01T17:28:21.808Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054507198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9162024-02-01T17:28:21.808Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.054531479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9172024-02-01T17:28:21.808Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055208267Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9182024-02-01T17:28:21.808Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055251624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9192024-02-01T17:28:21.808Z{"msg":"[0]R flush_numbers[0..12]: [5, 10, 7, 1, 7, 2, 9, 10, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055277283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9202024-02-01T17:28:21.808Z{"msg":"[0]R generation[0..12]: [3, 7, 4, 1, 4, 2, 6, 7, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.05532373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9212024-02-01T17:28:21.808Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055351121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9222024-02-01T17:28:21.808Z{"msg":"[1]R flush_numbers[0..12]: [5, 10, 7, 1, 7, 2, 9, 10, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055393089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9232024-02-01T17:28:21.809Z{"msg":"[1]R generation[0..12]: [3, 7, 4, 1, 4, 2, 6, 7, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055418122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9242024-02-01T17:28:21.809Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055460203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9252024-02-01T17:28:21.809Z{"msg":"[2]R flush_numbers[0..12]: [5, 10, 7, 1, 7, 2, 9, 10, 1, 6, 7, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055485132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9262024-02-01T17:28:21.809Z{"msg":"[2]R generation[0..12]: [3, 7, 4, 1, 4, 2, 6, 7, 1, 4, 4, 1]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055528027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9272024-02-01T17:28:21.809Z{"msg":"Wait for a query_work_queue command to finish before sending IO
9282024-02-01T17:28:21.809Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055555975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9292024-02-01T17:28:21.809Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055596567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9302024-02-01T17:28:21.809Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055620999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9312024-02-01T17:28:21.809Z{"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055661351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9322024-02-01T17:28:21.810Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055685171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"":"downstairs","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9332024-02-01T17:28:21.811Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055739898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9342024-02-01T17:28:21.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055774021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"0","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9352024-02-01T17:28:21.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055831023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"1","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9362024-02-01T17:28:21.811Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055890834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"client":"2","":"downstairs","session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9372024-02-01T17:28:21.811Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055922187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9382024-02-01T17:28:21.811Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.055986745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9392024-02-01T17:28:21.811Z{"msg":"ddb82b6d-5a14-4e72-917e-5059a644723c is now active with session: 2c2ad4ff-0b85-4536-b998-69912ef4d8e6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.056014196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9402024-02-01T17:28:21.811Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.056073102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161,"session_id":"2c2ad4ff-0b85-4536-b998-69912ef4d8e6"}
9412024-02-01T17:28:21.811ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
9422024-02-01T17:28:21.811Z{"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:21.056103516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
9432024-02-01T17:28:21.811ZLoading write count information from file "/tmp/test_repair_verify.data"
9442024-02-01T17:28:21.811ZRun Repair workload
9452024-02-01T17:28:21.811Z01/30 Write block 475 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9462024-02-01T17:28:21.811Z02/30 Flush
9472024-02-01T17:28:21.815Z03/30 Write block 358 len 4096 data: 3
9482024-02-01T17:28:21.816Z04/30 Read block 416 len 16384
9492024-02-01T17:28:21.816Z05/30 Write block 578 len 40960 data: 2 2 2 2 2 2 2 2 2 2
9502024-02-01T17:28:21.816Z06/30 Read block 271 len 20480
9512024-02-01T17:28:21.816Z07/30 Read block 201 len 24576
9522024-02-01T17:28:21.816Z08/30 Write block 275 len 8192 data: 2 2
9532024-02-01T17:28:21.816Z09/30 Read block 426 len 40960
9542024-02-01T17:28:21.816Z10/30 Write block 354 len 4096 data: 3
9552024-02-01T17:28:21.816Z11/30 Read block 186 len 12288
9562024-02-01T17:28:21.816Z12/30 Write block 111 len 20480 data: 2 2 2 2 2
9572024-02-01T17:28:21.820Z13/30 Read block 29 len 36864
9582024-02-01T17:28:21.820Z14/30 Write block 101 len 12288 data: 2 2 2
9592024-02-01T17:28:21.820Z15/30 Flush
9602024-02-01T17:28:21.824Z16/30 Write block 228 len 28672 data: 2 2 2 2 2 2 2
9612024-02-01T17:28:21.824Z17/30 Read block 142 len 20480
9622024-02-01T17:28:21.829Z18/30 Write block 70 len 24576 data: 2 2 2 2 2 2
9632024-02-01T17:28:21.829Z19/30 Read block 352 len 12288
9642024-02-01T17:28:21.833Z20/30 Write block 494 len 12288 data: 2 2 2
9652024-02-01T17:28:21.833Z21/30 Read block 497 len 20480
9662024-02-01T17:28:21.833Z22/30 Read block 519 len 28672
9672024-02-01T17:28:21.834Z23/30 Read block 430 len 12288
9682024-02-01T17:28:21.840Z24/30 Write block 541 len 28672 data: 4 4 4 4 4 4 2
9692024-02-01T17:28:21.840Z25/30 Write block 568 len 12288 data: 2 2 2
9702024-02-01T17:28:21.840Z26/30 Write block 113 len 40960 data: 3 3 3 2 2 2 2 2 2 3
9712024-02-01T17:28:21.840Z27/30 Read block 516 len 20480
9722024-02-01T17:28:21.844Z28/30 Write block 439 len 32768 data: 3 2 2 2 2 2 2 2
9732024-02-01T17:28:21.844Z29/30 Read block 39 len 28672
9742024-02-01T17:28:21.844Z30/30 Write block 54 len 40960 data: 2 2 2 3 2 2 2 2 2 2
9752024-02-01T17:28:21.849Z----------------------------------------------------------------
9762024-02-01T17:28:21.850Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:28
9772024-02-01T17:28:21.850ZStates: Active Active Active
9782024-02-01T17:28:21.850ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9792024-02-01T17:28:21.850Z 3 Acked 1002 Write 1 Sent Done Done false
9802024-02-01T17:28:21.850Z 4 Acked 1003 Read 4 Sent Done Done false
9812024-02-01T17:28:21.850Z 5 Acked 1004 Write 10 Sent Done Done false
9822024-02-01T17:28:21.850Z 6 Acked 1005 Read 5 Sent Done Done false
9832024-02-01T17:28:21.850Z 7 Acked 1006 Read 6 Sent Done Done false
9842024-02-01T17:28:21.851Z 8 Acked 1007 Write 2 Sent Done Done false
9852024-02-01T17:28:21.851Z 9 Acked 1008 Read 10 Sent Done Done false
9862024-02-01T17:28:21.852Z 10 Acked 1009 Write 1 Sent Done Done false
9872024-02-01T17:28:21.852Z 11 Acked 1010 Read 3 Sent Done Done false
9882024-02-01T17:28:21.852Z 12 Acked 1011 Write 5 Sent Done Done false
9892024-02-01T17:28:21.852Z 13 Acked 1012 Read 9 Sent Done Done false
9902024-02-01T17:28:21.852Z 14 Acked 1013 Write 3 Sent Done Done false
9912024-02-01T17:28:21.852Z 15 Acked 1014 Flush 0 Sent Done Done false
9922024-02-01T17:28:21.852Z 16 Acked 1015 Write 7 Sent Done Done false
9932024-02-01T17:28:21.852Z 17 Acked 1016 Read 5 Sent Done Done false
9942024-02-01T17:28:21.852Z 18 Acked 1017 Write 6 Sent Done Done false
9952024-02-01T17:28:21.852Z 19 Acked 1018 Read 3 Sent Done Done false
9962024-02-01T17:28:21.852Z 20 Acked 1019 Write 3 Sent Done Done false
9972024-02-01T17:28:21.852Z 21 Acked 1020 Read 5 Sent Done Done false
9982024-02-01T17:28:21.852Z 22 Acked 1021 Read 7 Sent Done Done false
9992024-02-01T17:28:21.852Z 23 Acked 1022 Read 3 Sent Done Done false
10002024-02-01T17:28:21.852Z 24 Acked 1023 Write 7 Sent Done Done false
10012024-02-01T17:28:21.852Z 25 Acked 1024 Write 3 Sent Done Done false
10022024-02-01T17:28:21.852Z 26 Acked 1025 Write 10 Sent Done Done false
10032024-02-01T17:28:21.852Z 27 Acked 1026 Read 5 Sent Done Done false
10042024-02-01T17:28:21.852Z 28 Acked 1027 Write 8 Sent Done Done false
10052024-02-01T17:28:21.853Z 29 Acked 1028 Read 7 Sent Done Done false
10062024-02-01T17:28:21.853Z 30 Acked 1029 Write 10 Sent Sent Sent false
10072024-02-01T17:28:21.853Z STATES DS:0 DS:1 DS:2 TOTAL
10082024-02-01T17:28:21.853Z New 0 0 0 0
10092024-02-01T17:28:21.853Z Sent 28 1 1 30
10102024-02-01T17:28:21.853Z Done 0 27 27 54
10112024-02-01T17:28:21.853Z Skipped 0 0 0 0
10122024-02-01T17:28:21.853Z Error 0 0 0 0
10132024-02-01T17:28:21.853ZLast Flush: 1001 1014 1014
10142024-02-01T17:28:21.853ZDownstairs last five completed: 1001 1000
10152024-02-01T17:28:21.853ZUpstairs last five completed: 30 29 28 27 26
10162024-02-01T17:28:21.853Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:21.106316779Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1161}
10172024-02-01T17:28:21.853ZWrote out file "/tmp/test_repair_verify.data"
10182024-02-01T17:28:21.858Z
10192024-02-01T17:28:21.862ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
10202024-02-01T17:28:21.886ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10212024-02-01T17:28:21.886Z 0 000-019 3 3 3 5 5 5 F F F
10222024-02-01T17:28:21.886Z 1 020-039 7 7 7 10 10 10 F F F
10232024-02-01T17:28:21.892Z 2 040-059 4 4 4 7 7 7 F T T <---
10242024-02-01T17:28:21.892Z 3 060-079 1 1 1 1 1 1 F T T <---
10252024-02-01T17:28:21.893Z 4 080-099 4 4 4 7 7 7 F F F
10262024-02-01T17:28:21.893Z 5 100-119 2 8 8 2 12 12 F T T <---
10272024-02-01T17:28:21.893Z 6 120-139 6 6 6 9 9 9 F T T <---
10282024-02-01T17:28:21.893Z 7 140-159 7 7 7 10 10 10 F F F
10292024-02-01T17:28:21.893Z 8 160-179 1 1 1 1 1 1 F F F
10302024-02-01T17:28:21.893Z 9 180-199 4 4 4 6 6 6 F F F
10312024-02-01T17:28:21.894Z 10 200-219 4 4 4 7 7 7 F F F
10322024-02-01T17:28:21.894Z 11 220-239 1 1 1 1 1 1 F T T <---
10332024-02-01T17:28:21.894Z 12 240-259 5 5 5 8 8 8 F F F
10342024-02-01T17:28:21.894Z 13 260-279 5 8 8 8 12 12 F F F <---
10352024-02-01T17:28:21.894Z 14 280-299 2 2 2 3 3 3 F F F
10362024-02-01T17:28:21.894Z 15 300-319 1 1 1 1 1 1 F F F
10372024-02-01T17:28:21.894Z 16 320-339 7 7 7 10 10 10 F F F
10382024-02-01T17:28:21.894Z 17 340-359 7 8 8 10 12 12 F F F <---
10392024-02-01T17:28:21.894Z 18 360-379 4 4 4 6 6 6 F F F
10402024-02-01T17:28:21.894Z 19 380-399 6 6 6 9 9 9 F F F
10412024-02-01T17:28:21.894Z 20 400-419 4 4 4 7 7 7 F F F
10422024-02-01T17:28:21.894Z 21 420-439 7 7 7 10 10 10 F T T <---
10432024-02-01T17:28:21.894Z 22 440-459 5 5 5 8 8 8 F T T <---
10442024-02-01T17:28:21.894Z 23 460-479 8 8 8 11 11 11 F F F
10452024-02-01T17:28:21.894Z 24 480-499 8 8 8 11 11 11 F T T <---
10462024-02-01T17:28:21.894Z 25 500-519 1 1 1 1 1 1 F F F
10472024-02-01T17:28:21.894Z 26 520-539 7 7 7 10 10 10 F F F
10482024-02-01T17:28:21.894Z 27 540-559 7 7 7 10 10 10 F T T <---
10492024-02-01T17:28:21.894Z 28 560-579 2 8 8 2 12 12 F T T <---
10502024-02-01T17:28:21.895Z 29 580-599 4 8 8 7 12 12 F F F <---
10512024-02-01T17:28:21.895ZMax gen: 8, Max flush: 12
10522024-02-01T17:28:21.895ZError: Difference in extent metadata found!
10532024-02-01T17:28:21.895ZA Difference in extent metadata is expected here
10542024-02-01T17:28:21.895ZOn loop 4
10552024-02-01T17:28:21.895Z
10562024-02-01T17:28:21.898ZVerifying data now
10572024-02-01T17:28:27.024ZNo change after verify
10582024-02-01T17:28:27.024ZLoop: 4 Downstairs dump after verify (and repair):
10592024-02-01T17:28:27.047ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
10602024-02-01T17:28:27.047Z 0 000-019 3 3 3 5 5 5 F F F
10612024-02-01T17:28:27.056Z 1 020-039 7 7 7 10 10 10 F F F
10622024-02-01T17:28:27.056Z 2 040-059 9 9 9 13 13 13 F F F
10632024-02-01T17:28:27.056Z 3 060-079 9 9 9 13 13 13 F F F
10642024-02-01T17:28:27.056Z 4 080-099 4 4 4 7 7 7 F F F
10652024-02-01T17:28:27.056Z 5 100-119 9 9 9 13 13 13 F F F
10662024-02-01T17:28:27.056Z 6 120-139 9 9 9 13 13 13 F F F
10672024-02-01T17:28:27.056Z 7 140-159 7 7 7 10 10 10 F F F
10682024-02-01T17:28:27.056Z 8 160-179 1 1 1 1 1 1 F F F
10692024-02-01T17:28:27.056Z 9 180-199 4 4 4 6 6 6 F F F
10702024-02-01T17:28:27.056Z 10 200-219 4 4 4 7 7 7 F F F
10712024-02-01T17:28:27.056Z 11 220-239 9 9 9 13 13 13 F F F
10722024-02-01T17:28:27.056Z 12 240-259 5 5 5 8 8 8 F F F
10732024-02-01T17:28:27.056Z 13 260-279 8 8 8 12 12 12 F F F
10742024-02-01T17:28:27.056Z 14 280-299 2 2 2 3 3 3 F F F
10752024-02-01T17:28:27.056Z 15 300-319 1 1 1 1 1 1 F F F
10762024-02-01T17:28:27.056Z 16 320-339 7 7 7 10 10 10 F F F
10772024-02-01T17:28:27.056Z 17 340-359 8 8 8 12 12 12 F F F
10782024-02-01T17:28:27.056Z 18 360-379 4 4 4 6 6 6 F F F
10792024-02-01T17:28:27.057Z 19 380-399 6 6 6 9 9 9 F F F
10802024-02-01T17:28:27.057Z 20 400-419 4 4 4 7 7 7 F F F
10812024-02-01T17:28:27.057Z 21 420-439 9 9 9 13 13 13 F F F
10822024-02-01T17:28:27.057Z 22 440-459 9 9 9 13 13 13 F F F
10832024-02-01T17:28:27.057Z 23 460-479 8 8 8 11 11 11 F F F
10842024-02-01T17:28:27.057Z 24 480-499 9 9 9 13 13 13 F F F
10852024-02-01T17:28:27.057Z 25 500-519 1 1 1 1 1 1 F F F
10862024-02-01T17:28:27.057Z 26 520-539 7 7 7 10 10 10 F F F
10872024-02-01T17:28:27.057Z 27 540-559 9 9 9 13 13 13 F F F
10882024-02-01T17:28:27.057Z 28 560-579 9 9 9 13 13 13 F F F
10892024-02-01T17:28:27.057Z 29 580-599 8 8 8 12 12 12 F F F
10902024-02-01T17:28:27.057ZMax gen: 9, Max flush: 13
10912024-02-01T17:28:27.057Z
10922024-02-01T17:28:27.057ZBegin loop 5
10932024-02-01T17:28:27.057ZDownstairs to restart: 0
10942024-02-01T17:28:27.083Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.343867543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
10952024-02-01T17:28:27.083Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.3443885Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
10962024-02-01T17:28:27.088Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.344470057Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
10972024-02-01T17:28:27.088Z{Crucible runtime is spawned
10982024-02-01T17:28:27.088Z"msg":"Crucible 3aeedc05-736a-49f2-b562-b2538f8f5efc has session id: b40315f1-b6f5-4e61-9501-42705ebe3064","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.344686673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
10992024-02-01T17:28:27.088Z{"msg":"Upstairs opts: Upstairs UUID: 3aeedc05-736a-49f2-b562-b2538f8f5efc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.344769419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11002024-02-01T17:28:27.088Z{"msg":"Crucible stats registered with UUID: 3aeedc05-736a-49f2-b562-b2538f8f5efc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.34483449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11012024-02-01T17:28:27.088Z{"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.344910114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
11022024-02-01T17:28:27.088Z{"msg":"3aeedc05-736a-49f2-b562-b2538f8f5efc active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.34494747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11032024-02-01T17:28:27.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345104445Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11042024-02-01T17:28:27.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345160244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11052024-02-01T17:28:27.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345253821Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11062024-02-01T17:28:27.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345299103Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11072024-02-01T17:28:27.088Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345344116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11082024-02-01T17:28:27.088Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345387959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11092024-02-01T17:28:27.089Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345431234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11102024-02-01T17:28:27.089Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345475146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11112024-02-01T17:28:27.089Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345555112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11122024-02-01T17:28:27.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345584174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11132024-02-01T17:28:27.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345607424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11142024-02-01T17:28:27.089Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.345649599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"io task","client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11152024-02-01T17:28:27.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.346515082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11162024-02-01T17:28:27.089Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.346610915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11172024-02-01T17:28:27.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.346654962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11182024-02-01T17:28:27.089Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.346733756Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11192024-02-01T17:28:27.089Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.347653135Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11202024-02-01T17:28:27.089Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.34775126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11212024-02-01T17:28:27.089Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.347822516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11222024-02-01T17:28:27.095Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.347970688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11232024-02-01T17:28:27.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.348792972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11242024-02-01T17:28:27.095Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.348927124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11252024-02-01T17:28:27.095Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.348977894Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11262024-02-01T17:28:27.095Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349041413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11272024-02-01T17:28:27.095Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349112016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11282024-02-01T17:28:27.095Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349156223Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11292024-02-01T17:28:27.097Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349209424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11302024-02-01T17:28:27.097Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349587441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11312024-02-01T17:28:27.097Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349650989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11322024-02-01T17:28:27.097Z{"msg":"[0]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 13, 10, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349679644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11332024-02-01T17:28:27.097Z{"msg":"[0]R generation[0..12]: [3, 7, 9, 9, 4, 9, 9, 7, 1, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349739331Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11342024-02-01T17:28:27.098Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.3497688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11352024-02-01T17:28:27.098Z{"msg":"[1]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 13, 10, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349827401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11362024-02-01T17:28:27.098Z{"msg":"[1]R generation[0..12]: [3, 7, 9, 9, 4, 9, 9, 7, 1, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349854961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11372024-02-01T17:28:27.098Z{"msg":Wait for a query_work_queue command to finish before sending IO
11382024-02-01T17:28:27.098Z"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.349941766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11392024-02-01T17:28:27.101Z{"msg":"[2]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 13, 10, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350002168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11402024-02-01T17:28:27.101Z{"msg":"[2]R generation[0..12]: [3, 7, 9, 9, 4, 9, 9, 7, 1, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350045364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11412024-02-01T17:28:27.101Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350100464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11422024-02-01T17:28:27.101Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350126582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11432024-02-01T17:28:27.101Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350185168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11442024-02-01T17:28:27.101Z{"msg":"Next flush: 14","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350247259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11452024-02-01T17:28:27.101Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350279573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"":"downstairs","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11462024-02-01T17:28:27.101Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350333656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11472024-02-01T17:28:27.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350362677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"0","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11482024-02-01T17:28:27.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350417417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"1","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11492024-02-01T17:28:27.101Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350443329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"client":"2","":"downstairs","session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11502024-02-01T17:28:27.101Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.35050847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11512024-02-01T17:28:27.101Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350538813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11522024-02-01T17:28:27.101Z{"msg":"3aeedc05-736a-49f2-b562-b2538f8f5efc is now active with session: b40315f1-b6f5-4e61-9501-42705ebe3064","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350605415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11532024-02-01T17:28:27.101Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350633084Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169,"session_id":"b40315f1-b6f5-4e61-9501-42705ebe3064"}
11542024-02-01T17:28:27.101Z{"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:26.350685905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
11552024-02-01T17:28:27.101ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11562024-02-01T17:28:27.101ZLoading write count information from file "/tmp/test_repair_verify.data"
11572024-02-01T17:28:27.101ZRun Repair workload
11582024-02-01T17:28:27.102Z01/30 Write block 502 len 8192 data: 2 2
11592024-02-01T17:28:27.102Z02/30 Read block 136 len 8192
11602024-02-01T17:28:27.105Z03/30 Read block 568 len 40960
11612024-02-01T17:28:27.105Z04/30 Write block 540 len 16384 data: 3 5 5 5
11622024-02-01T17:28:27.105Z05/30 Read block 114 len 24576
11632024-02-01T17:28:27.105Z06/30 Read block 452 len 36864
11642024-02-01T17:28:27.105Z07/30 Read block 206 len 32768
11652024-02-01T17:28:27.106Z08/30 Write block 397 len 8192 data: 2 2
11662024-02-01T17:28:27.106Z09/30 Write block 363 len 4096 data: 2
11672024-02-01T17:28:27.106Z10/30 Read block 81 len 36864
11682024-02-01T17:28:27.106Z11/30 Write block 465 len 4096 data: 2
11692024-02-01T17:28:27.109Z12/30 Read block 219 len 4096
11702024-02-01T17:28:27.109Z13/30 Read block 569 len 32768
11712024-02-01T17:28:27.109Z14/30 Read block 232 len 24576
11722024-02-01T17:28:27.112Z15/30 Write block 354 len 12288 data: 4 2 2
11732024-02-01T17:28:27.112Z16/30 Read block 271 len 20480
11742024-02-01T17:28:27.112Z17/30 Read block 430 len 16384
11752024-02-01T17:28:27.115Z18/30 Write block 414 len 12288 data: 2 2 3
11762024-02-01T17:28:27.115Z19/30 Read block 185 len 36864
11772024-02-01T17:28:27.119Z20/30 Read block 61 len 36864
11782024-02-01T17:28:27.119Z21/30 Flush
11792024-02-01T17:28:27.122Z22/30 Write block 137 len 8192 data: 2 2
11802024-02-01T17:28:27.122Z23/30 Flush
11812024-02-01T17:28:27.125Z24/30 Write block 139 len 28672 data: 2 2 2 2 2 2 2
11822024-02-01T17:28:27.128Z25/30 Write block 520 len 36864 data: 2 2 2 2 2 2 2 2 2
11832024-02-01T17:28:27.128Z26/30 Flush
11842024-02-01T17:28:27.131Z27/30 Write block 298 len 32768 data: 2 2 2 2 2 2 2 2
11852024-02-01T17:28:27.131Z28/30 Write block 286 len 4096 data: 2
11862024-02-01T17:28:27.134Z29/30 Read block 513 len 28672
11872024-02-01T17:28:27.134Z30/30 Write block 483 len 4096 data: 3
11882024-02-01T17:28:27.138Z----------------------------------------------------------------
11892024-02-01T17:28:27.138Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
11902024-02-01T17:28:27.138ZStates: Active Active Active
11912024-02-01T17:28:27.138ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11922024-02-01T17:28:27.138Z 1 Acked 1000 Write 2 Sent Done Done false
11932024-02-01T17:28:27.138Z 2 Acked 1001 Read 2 Sent Done Done false
11942024-02-01T17:28:27.138Z 3 Acked 1002 Read 10 Sent Done Done false
11952024-02-01T17:28:27.138Z 4 Acked 1003 Write 4 Sent Done Done false
11962024-02-01T17:28:27.138Z 5 Acked 1004 Read 6 Sent Done Done false
11972024-02-01T17:28:27.138Z 6 Acked 1005 Read 9 Sent Done Done false
11982024-02-01T17:28:27.138Z 7 Acked 1006 Read 8 Sent Done Done false
11992024-02-01T17:28:27.138Z 8 Acked 1007 Write 2 Sent Done Done false
12002024-02-01T17:28:27.138Z 9 Acked 1008 Write 1 Sent Done Done false
12012024-02-01T17:28:27.138Z 10 Acked 1009 Read 9 Sent Done Done false
12022024-02-01T17:28:27.139Z 11 Acked 1010 Write 1 Sent Done Done false
12032024-02-01T17:28:27.139Z 12 Acked 1011 Read 1 Sent Done Done false
12042024-02-01T17:28:27.139Z 13 Acked 1012 Read 8 Sent Done Done false
12052024-02-01T17:28:27.139Z 14 Acked 1013 Read 6 Sent Done Done false
12062024-02-01T17:28:27.139Z 15 Acked 1014 Write 3 Sent Done Done false
12072024-02-01T17:28:27.139Z 16 Acked 1015 Read 5 Sent Done Done false
12082024-02-01T17:28:27.139Z 17 Acked 1016 Read 4 Sent Done Done false
12092024-02-01T17:28:27.139Z 18 Acked 1017 Write 3 Sent Done Done false
12102024-02-01T17:28:27.139Z 19 Acked 1018 Read 9 Sent Done Done false
12112024-02-01T17:28:27.139Z 20 Acked 1019 Read 9 Sent Done Done false
12122024-02-01T17:28:27.139Z 21 Acked 1020 Flush 0 Sent Done Done false
12132024-02-01T17:28:27.139Z 22 Acked 1021 Write 2 Sent Done Done false
12142024-02-01T17:28:27.140Z 23 Acked 1022 Flush 0 Sent Done Done false
12152024-02-01T17:28:27.140Z 24 Acked 1023 Write 7 Sent Done Done false
12162024-02-01T17:28:27.140Z 25 Acked 1024 Write 9 Sent Done Done false
12172024-02-01T17:28:27.140Z 26 Acked 1025 Flush 0 Sent Done Done false
12182024-02-01T17:28:27.140Z 27 Acked 1026 Write 8 Sent Done Done false
12192024-02-01T17:28:27.140Z 28 Acked 1027 Write 1 Sent Done Done false
12202024-02-01T17:28:27.140Z 29 Acked 1028 Read 7 Sent Done Done false
12212024-02-01T17:28:27.141Z 30 Acked 1029 Write 1 Sent Sent Sent false
12222024-02-01T17:28:27.141Z STATES DS:0 DS:1 DS:2 TOTAL
12232024-02-01T17:28:27.141Z New 0 0 0 0
12242024-02-01T17:28:27.141Z Sent 30 1 1 32
12252024-02-01T17:28:27.141Z Done 0 29 29 58
12262024-02-01T17:28:27.141Z Skipped 0 0 0 0
12272024-02-01T17:28:27.141Z Error 0 0 0 0
12282024-02-01T17:28:27.141ZLast Flush: 0 1025 1025
12292024-02-01T17:28:27.141ZDownstairs last five completed:
12302024-02-01T17:28:27.141ZUpstairs last five completed: 30 29 28 27 26
12312024-02-01T17:28:27.141Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:26.396429153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1169}
12322024-02-01T17:28:27.141ZWrote out file "/tmp/test_repair_verify.data"
12332024-02-01T17:28:27.147Z
12342024-02-01T17:28:27.275ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12352024-02-01T17:28:27.285ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12362024-02-01T17:28:27.285Z 0 000-019 3 3 3 5 5 5 F F F
12372024-02-01T17:28:27.287Z 1 020-039 7 7 7 10 10 10 F F F
12382024-02-01T17:28:27.287Z 2 040-059 9 9 9 13 13 13 F F F
12392024-02-01T17:28:27.288Z 3 060-079 9 9 9 13 13 13 F F F
12402024-02-01T17:28:27.288Z 4 080-099 4 4 4 7 7 7 F F F
12412024-02-01T17:28:27.288Z 5 100-119 9 9 9 13 13 13 F F F
12422024-02-01T17:28:27.288Z 6 120-139 9 10 10 13 16 16 F F F <---
12432024-02-01T17:28:27.288Z 7 140-159 7 10 10 10 16 16 F F F <---
12442024-02-01T17:28:27.288Z 8 160-179 1 1 1 1 1 1 F F F
12452024-02-01T17:28:27.288Z 9 180-199 4 4 4 6 6 6 F F F
12462024-02-01T17:28:27.288Z 10 200-219 4 4 4 7 7 7 F F F
12472024-02-01T17:28:27.288Z 11 220-239 9 9 9 13 13 13 F F F
12482024-02-01T17:28:27.289Z 12 240-259 5 5 5 8 8 8 F F F
12492024-02-01T17:28:27.289Z 13 260-279 8 8 8 12 12 12 F F F
12502024-02-01T17:28:27.289Z 14 280-299 2 2 2 3 3 3 F T T <---
12512024-02-01T17:28:27.289Z 15 300-319 1 1 1 1 1 1 F T T <---
12522024-02-01T17:28:27.289Z 16 320-339 7 7 7 10 10 10 F F F
12532024-02-01T17:28:27.289Z 17 340-359 8 10 10 12 14 14 F F F <---
12542024-02-01T17:28:27.289Z 18 360-379 4 10 10 6 14 14 F F F <---
12552024-02-01T17:28:27.290Z 19 380-399 6 10 10 9 14 14 F F F <---
12562024-02-01T17:28:27.290Z 20 400-419 4 10 10 7 14 14 F F F <---
12572024-02-01T17:28:27.290Z 21 420-439 9 9 9 13 13 13 F F F
12582024-02-01T17:28:27.290Z 22 440-459 9 9 9 13 13 13 F F F
12592024-02-01T17:28:27.290Z 23 460-479 8 10 10 11 14 14 F F F <---
12602024-02-01T17:28:27.290Z 24 480-499 9 9 9 13 13 13 F T T <---
12612024-02-01T17:28:27.291Z 25 500-519 1 10 10 1 14 14 F F F <---
12622024-02-01T17:28:27.291Z 26 520-539 7 10 10 10 16 16 F F F <---
12632024-02-01T17:28:27.292Z 27 540-559 9 10 10 13 14 14 F F F <---
12642024-02-01T17:28:27.292Z 28 560-579 9 9 9 13 13 13 F F F
12652024-02-01T17:28:27.292Z 29 580-599 8 8 8 12 12 12 F F F
12662024-02-01T17:28:27.292ZMax gen: 10, Max flush: 16
12672024-02-01T17:28:27.292ZError: Difference in extent metadata found!
12682024-02-01T17:28:27.292ZA Difference in extent metadata is expected here
12692024-02-01T17:28:27.292ZOn loop 5
12702024-02-01T17:28:27.292Z
12712024-02-01T17:28:27.292ZVerifying data now
12722024-02-01T17:28:32.278ZNo change after verify
12732024-02-01T17:28:32.278ZLoop: 5 Downstairs dump after verify (and repair):
12742024-02-01T17:28:32.323ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12752024-02-01T17:28:32.323Z 0 000-019 3 3 3 5 5 5 F F F
12762024-02-01T17:28:32.330Z 1 020-039 7 7 7 10 10 10 F F F
12772024-02-01T17:28:32.330Z 2 040-059 9 9 9 13 13 13 F F F
12782024-02-01T17:28:32.330Z 3 060-079 9 9 9 13 13 13 F F F
12792024-02-01T17:28:32.330Z 4 080-099 4 4 4 7 7 7 F F F
12802024-02-01T17:28:32.330Z 5 100-119 9 9 9 13 13 13 F F F
12812024-02-01T17:28:32.330Z 6 120-139 10 10 10 16 16 16 F F F
12822024-02-01T17:28:32.330Z 7 140-159 10 10 10 16 16 16 F F F
12832024-02-01T17:28:32.330Z 8 160-179 1 1 1 1 1 1 F F F
12842024-02-01T17:28:32.330Z 9 180-199 4 4 4 6 6 6 F F F
12852024-02-01T17:28:32.330Z 10 200-219 4 4 4 7 7 7 F F F
12862024-02-01T17:28:32.330Z 11 220-239 9 9 9 13 13 13 F F F
12872024-02-01T17:28:32.331Z 12 240-259 5 5 5 8 8 8 F F F
12882024-02-01T17:28:32.331Z 13 260-279 8 8 8 12 12 12 F F F
12892024-02-01T17:28:32.331Z 14 280-299 11 11 11 17 17 17 F F F
12902024-02-01T17:28:32.331Z 15 300-319 11 11 11 17 17 17 F F F
12912024-02-01T17:28:32.331Z 16 320-339 7 7 7 10 10 10 F F F
12922024-02-01T17:28:32.331Z 17 340-359 10 10 10 14 14 14 F F F
12932024-02-01T17:28:32.331Z 18 360-379 10 10 10 14 14 14 F F F
12942024-02-01T17:28:32.331Z 19 380-399 10 10 10 14 14 14 F F F
12952024-02-01T17:28:32.331Z 20 400-419 10 10 10 14 14 14 F F F
12962024-02-01T17:28:32.331Z 21 420-439 9 9 9 13 13 13 F F F
12972024-02-01T17:28:32.331Z 22 440-459 9 9 9 13 13 13 F F F
12982024-02-01T17:28:32.331Z 23 460-479 10 10 10 14 14 14 F F F
12992024-02-01T17:28:32.331Z 24 480-499 11 11 11 17 17 17 F F F
13002024-02-01T17:28:32.331Z 25 500-519 10 10 10 14 14 14 F F F
13012024-02-01T17:28:32.331Z 26 520-539 10 10 10 16 16 16 F F F
13022024-02-01T17:28:32.331Z 27 540-559 10 10 10 14 14 14 F F F
13032024-02-01T17:28:32.331Z 28 560-579 9 9 9 13 13 13 F F F
13042024-02-01T17:28:32.331Z 29 580-599 8 8 8 12 12 12 F F F
13052024-02-01T17:28:32.332ZMax gen: 11, Max flush: 17
13062024-02-01T17:28:32.332Z
13072024-02-01T17:28:32.332ZBegin loop 6
13082024-02-01T17:28:32.332ZDownstairs to restart: 0
13092024-02-01T17:28:32.335Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.596549805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
13102024-02-01T17:28:32.335Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597127732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
13112024-02-01T17:28:32.341Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597201082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
13122024-02-01T17:28:32.341Z{"msg":"Crucible 470f47cf-1c20-4240-a1f7-d7e96a04608d has session id: 356be535-a8b7-4934-b7e5-7549016b719e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.59731798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13132024-02-01T17:28:32.341Z{"msg":"Upstairs opts: Upstairs UUID: 470f47cf-1c20-4240-a1f7-d7e96a04608d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
13142024-02-01T17:28:32.341Z,"time":"2024-02-01T17:28:31.597406091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13152024-02-01T17:28:32.341Z{"msg":"Crucible stats registered with UUID: 470f47cf-1c20-4240-a1f7-d7e96a04608d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.5975642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13162024-02-01T17:28:32.341Z{"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597608311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
13172024-02-01T17:28:32.341Z{"msg":"470f47cf-1c20-4240-a1f7-d7e96a04608d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.59766324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13182024-02-01T17:28:32.341Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597687664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13192024-02-01T17:28:32.341Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597717942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13202024-02-01T17:28:32.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597790859Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13212024-02-01T17:28:32.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597860633Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13222024-02-01T17:28:32.342Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597919273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13232024-02-01T17:28:32.342Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.597946068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13242024-02-01T17:28:32.342Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.59797348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13252024-02-01T17:28:32.342Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.598065524Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13262024-02-01T17:28:32.342Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.598133726Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13272024-02-01T17:28:32.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.598406139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13282024-02-01T17:28:32.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.598532847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13292024-02-01T17:28:32.342Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.598579825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"io task","client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13302024-02-01T17:28:32.343Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.599712803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13312024-02-01T17:28:32.343Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.599779982Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13322024-02-01T17:28:32.343Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.599861815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13332024-02-01T17:28:32.343Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.599944559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13342024-02-01T17:28:32.344Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.600570686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13352024-02-01T17:28:32.344Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.600674635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13362024-02-01T17:28:32.350Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.600843946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13372024-02-01T17:28:32.350Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.600907452Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13382024-02-01T17:28:32.350Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.60143645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13392024-02-01T17:28:32.350Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.601770516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13402024-02-01T17:28:32.350Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.601896413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13412024-02-01T17:28:32.350Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.601974823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13422024-02-01T17:28:32.350Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.60201431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13432024-02-01T17:28:32.351Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602039208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13442024-02-01T17:28:32.351Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602061517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13452024-02-01T17:28:32.351Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602312648Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13462024-02-01T17:28:32.351Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602382056Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13472024-02-01T17:28:32.351Z{"msg":"[0]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 16, 16, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602411376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13482024-02-01T17:28:32.351Z{"msg":"[0]R generation[0..12]: [3, 7, 9, 9, 4, 9, 10, 10, 1, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602454261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13492024-02-01T17:28:32.351Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602480757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13502024-02-01T17:28:32.351Z{"msg":"[1]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 16, 16, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602504514Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13512024-02-01T17:28:32.351Z{"msg":"[1]R generation[0..12]: [3, 7, 9, 9, 4, 9, 10, 10, 1, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602545738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13522024-02-01T17:28:32.353Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602571506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13532024-02-01T17:28:32.353Z{"msg":"[2]R flush_numbers[0..12]: [5, 10, 13, 13, 7, 13, 16, 16, 1, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602630644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13542024-02-01T17:28:32.353Z{"msg":"[2]R generation[0..12]: [3, 7, 9, 9, 4, 9, 10, 10, 1, 4, 4, 9]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
13552024-02-01T17:28:32.353Zcrucible","level":30,"time":"2024-02-01T17:28:31.60272486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13562024-02-01T17:28:32.353Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602796695Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13572024-02-01T17:28:32.353Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602858041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13582024-02-01T17:28:32.353Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602927003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13592024-02-01T17:28:32.353Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.602970178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13602024-02-01T17:28:32.353Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603009678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"":"downstairs","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13612024-02-01T17:28:32.354Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603050771Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13622024-02-01T17:28:32.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603112382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"0","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13632024-02-01T17:28:32.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603156795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"1","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13642024-02-01T17:28:32.354Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603235733Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"client":"2","":"downstairs","session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13652024-02-01T17:28:32.354Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603306955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13662024-02-01T17:28:32.354Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603358999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13672024-02-01T17:28:32.354Z{"msg":"470f47cf-1c20-4240-a1f7-d7e96a04608d is now active with session: 356be535-a8b7-4934-b7e5-7549016b719e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603383635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13682024-02-01T17:28:32.354Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.60344844Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177,"session_id":"356be535-a8b7-4934-b7e5-7549016b719e"}
13692024-02-01T17:28:32.354Z{"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:31.603508832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
13702024-02-01T17:28:32.355ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13712024-02-01T17:28:32.355ZLoading write count information from file "/tmp/test_repair_verify.data"
13722024-02-01T17:28:32.355ZRun Repair workload
13732024-02-01T17:28:32.355Z01/30 Write block 304 len 40960 data: 3 3 2 2 2 2 2 2 2 2
13742024-02-01T17:28:32.356Z02/30 Read block 228 len 32768
13752024-02-01T17:28:32.360Z03/30 Read block 593 len 4096
13762024-02-01T17:28:32.360Z04/30 Read block 12 len 20480
13772024-02-01T17:28:32.360Z05/30 Write block 105 len 20480 data: 2 2 3 2 2
13782024-02-01T17:28:32.361Z06/30 Read block 103 len 20480
13792024-02-01T17:28:32.361Z07/30 Read block 477 len 4096
13802024-02-01T17:28:32.361Z08/30 Write block 523 len 16384 data: 3 3 3 3
13812024-02-01T17:28:32.361Z09/30 Read block 439 len 8192
13822024-02-01T17:28:32.361Z10/30 Flush
13832024-02-01T17:28:32.361Z11/30 Write block 569 len 12288 data: 3 3 2
13842024-02-01T17:28:32.364Z12/30 Read block 372 len 20480
13852024-02-01T17:28:32.365Z13/30 Read block 438 len 36864
13862024-02-01T17:28:32.365Z14/30 Write block 444 len 24576 data: 3 3 3 2 2 3
13872024-02-01T17:28:32.368Z15/30 Write block 55 len 4096 data: 3
13882024-02-01T17:28:32.368Z16/30 Read block 270 len 28672
13892024-02-01T17:28:32.368Z17/30 Write block 169 len 32768 data: 2 2 2 2 2 2 2 2
13902024-02-01T17:28:32.374Z18/30 Read block 576 len 24576
13912024-02-01T17:28:32.374Z19/30 Read block 487 len 36864
13922024-02-01T17:28:32.374Z20/30 Read block 577 len 28672
13932024-02-01T17:28:32.374Z21/30 Read block 17 len 28672
13942024-02-01T17:28:32.378Z22/30 Read block 219 len 4096
13952024-02-01T17:28:32.378Z23/30 Write block 277 len 20480 data: 2 2 2 2 2
13962024-02-01T17:28:32.378Z24/30 Write block 429 len 28672 data: 3 4 4 5 5 5 5
13972024-02-01T17:28:32.381Z25/30 Read block 377 len 20480
13982024-02-01T17:28:32.381Z26/30 Read block 347 len 32768
13992024-02-01T17:28:32.384Z27/30 Read block 184 len 12288
14002024-02-01T17:28:32.384Z28/30 Read block 30 len 40960
14012024-02-01T17:28:32.387Z29/30 Write block 376 len 24576 data: 4 4 4 4 4 4
14022024-02-01T17:28:32.387Z30/30 Write block 291 len 24576 data: 3 2 2 2 2 2
14032024-02-01T17:28:32.391Z----------------------------------------------------------------
14042024-02-01T17:28:32.391Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
14052024-02-01T17:28:32.391ZStates: Active Active Active
14062024-02-01T17:28:32.391ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14072024-02-01T17:28:32.391Z 1 Acked 1000 Write 10 Done Done Done false
14082024-02-01T17:28:32.391Z 2 Acked 1001 Read 8 Done Done Done false
14092024-02-01T17:28:32.391Z 3 Acked 1002 Read 1 Done Done Done false
14102024-02-01T17:28:32.391Z 4 Acked 1003 Read 5 Done Done Done false
14112024-02-01T17:28:32.391Z 5 Acked 1004 Write 5 Done Done Done false
14122024-02-01T17:28:32.391Z 6 Acked 1005 Read 5 Sent Done Done false
14132024-02-01T17:28:32.391Z 7 Acked 1006 Read 1 Sent Done Done false
14142024-02-01T17:28:32.391Z 8 Acked 1007 Write 4 Sent Done Done false
14152024-02-01T17:28:32.391Z 9 Acked 1008 Read 2 Sent Done Done false
14162024-02-01T17:28:32.391Z 10 Acked 1009 Flush 0 Sent Done Done false
14172024-02-01T17:28:32.391Z 11 Acked 1010 Write 3 Sent Done Done false
14182024-02-01T17:28:32.391Z 12 Acked 1011 Read 5 Sent Done Done false
14192024-02-01T17:28:32.391Z 13 Acked 1012 Read 9 Sent Done Done false
14202024-02-01T17:28:32.391Z 14 Acked 1013 Write 6 Sent Done Done false
14212024-02-01T17:28:32.392Z 15 Acked 1014 Write 1 Sent Done Done false
14222024-02-01T17:28:32.392Z 16 Acked 1015 Read 7 Sent Done Done false
14232024-02-01T17:28:32.392Z 17 Acked 1016 Write 8 Sent Done Done false
14242024-02-01T17:28:32.392Z 18 Acked 1017 Read 6 Sent Done Done false
14252024-02-01T17:28:32.392Z 19 Acked 1018 Read 9 Sent Done Done false
14262024-02-01T17:28:32.392Z 20 Acked 1019 Read 7 Sent Done Done false
14272024-02-01T17:28:32.392Z 21 Acked 1020 Read 7 Sent Done Done false
14282024-02-01T17:28:32.392Z 22 Acked 1021 Read 1 Sent Done Done false
14292024-02-01T17:28:32.392Z 23 Acked 1022 Write 5 Sent Done Done false
14302024-02-01T17:28:32.392Z 24 Acked 1023 Write 7 Sent Done Done false
14312024-02-01T17:28:32.392Z 25 Acked 1024 Read 5 Sent Done Done false
14322024-02-01T17:28:32.392Z 26 Acked 1025 Read 8 Sent Done Done false
14332024-02-01T17:28:32.392Z 27 Acked 1026 Read 3 Sent Done Done false
14342024-02-01T17:28:32.392Z 28 Acked 1027 Read 10 Sent Done Done false
14352024-02-01T17:28:32.392Z 29 Acked 1028 Write 6 Sent Done Done false
14362024-02-01T17:28:32.392Z 30 Acked 1029 Write 6 Sent Sent Sent false
14372024-02-01T17:28:32.392Z STATES DS:0 DS:1 DS:2 TOTAL
14382024-02-01T17:28:32.392Z New 0 0 0 0
14392024-02-01T17:28:32.392Z Sent 25 1 1 27
14402024-02-01T17:28:32.392Z Done 5 29 29 63
14412024-02-01T17:28:32.393Z Skipped 0 0 0 0
14422024-02-01T17:28:32.393Z Error 0 0 0 0
14432024-02-01T17:28:32.393ZLast Flush: 0 1009 1009
14442024-02-01T17:28:32.393ZDownstairs last five completed:
14452024-02-01T17:28:32.393ZUpstairs last five completed: 30 29 28 27 26
14462024-02-01T17:28:32.393Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:31.649795981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1177}
14472024-02-01T17:28:32.393ZWrote out file "/tmp/test_repair_verify.data"
14482024-02-01T17:28:32.400Z
14492024-02-01T17:28:32.410ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14502024-02-01T17:28:32.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14512024-02-01T17:28:32.443Z 0 000-019 3 3 3 5 5 5 F F F
14522024-02-01T17:28:32.443Z 1 020-039 7 7 7 10 10 10 F F F
14532024-02-01T17:28:32.443Z 2 040-059 9 9 9 13 13 13 F T T <---
14542024-02-01T17:28:32.444Z 3 060-079 9 9 9 13 13 13 F F F
14552024-02-01T17:28:32.444Z 4 080-099 4 4 4 7 7 7 F F F
14562024-02-01T17:28:32.457Z 5 100-119 9 12 12 13 18 18 T F F <---
14572024-02-01T17:28:32.458Z 6 120-139 10 10 10 16 16 16 F F F
14582024-02-01T17:28:32.458Z 7 140-159 10 10 10 16 16 16 F F F
14592024-02-01T17:28:32.458Z 8 160-179 1 1 1 1 1 1 F T T <---
14602024-02-01T17:28:32.458Z 9 180-199 4 4 4 6 6 6 F F F
14612024-02-01T17:28:32.458Z 10 200-219 4 4 4 7 7 7 F F F
14622024-02-01T17:28:32.458Z 11 220-239 9 9 9 13 13 13 F F F
14632024-02-01T17:28:32.458Z 12 240-259 5 5 5 8 8 8 F F F
14642024-02-01T17:28:32.459Z 13 260-279 8 8 8 12 12 12 F T T <---
14652024-02-01T17:28:32.459Z 14 280-299 11 11 11 17 17 17 F T T <---
14662024-02-01T17:28:32.459Z 15 300-319 11 12 12 17 18 18 T F F <---
14672024-02-01T17:28:32.459Z 16 320-339 7 7 7 10 10 10 F F F
14682024-02-01T17:28:32.459Z 17 340-359 10 10 10 14 14 14 F F F
14692024-02-01T17:28:32.459Z 18 360-379 10 10 10 14 14 14 F T T <---
14702024-02-01T17:28:32.459Z 19 380-399 10 10 10 14 14 14 F T T <---
14712024-02-01T17:28:32.460Z 20 400-419 10 10 10 14 14 14 F F F
14722024-02-01T17:28:32.460Z 21 420-439 9 9 9 13 13 13 F T T <---
14732024-02-01T17:28:32.460Z 22 440-459 9 9 9 13 13 13 F T T <---
14742024-02-01T17:28:32.460Z 23 460-479 10 10 10 14 14 14 F F F
14752024-02-01T17:28:32.460Z 24 480-499 11 11 11 17 17 17 F F F
14762024-02-01T17:28:32.460Z 25 500-519 10 10 10 14 14 14 F F F
14772024-02-01T17:28:32.462Z 26 520-539 10 12 12 16 18 18 F F F <---
14782024-02-01T17:28:32.462Z 27 540-559 10 10 10 14 14 14 F F F
14792024-02-01T17:28:32.462Z 28 560-579 9 9 9 13 13 13 F T T <---
14802024-02-01T17:28:32.462Z 29 580-599 8 8 8 12 12 12 F F F
14812024-02-01T17:28:32.462ZMax gen: 12, Max flush: 18
14822024-02-01T17:28:32.462ZError: Difference in extent metadata found!
14832024-02-01T17:28:32.463ZA Difference in extent metadata is expected here
14842024-02-01T17:28:32.463ZOn loop 6
14852024-02-01T17:28:32.463Z
14862024-02-01T17:28:32.463ZVerifying data now
14872024-02-01T17:28:34.481ZNo change after verify
14882024-02-01T17:28:34.482ZLoop: 6 Downstairs dump after verify (and repair):
14892024-02-01T17:28:34.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14902024-02-01T17:28:34.504Z 0 000-019 3 3 3 5 5 5 F F F
14912024-02-01T17:28:34.504Z 1 020-039 7 7 7 10 10 10 F F F
14922024-02-01T17:28:34.504Z 2 040-059 13 13 13 19 19 19 F F F
14932024-02-01T17:28:34.504Z 3 060-079 9 9 9 13 13 13 F F F
14942024-02-01T17:28:34.513Z 4 080-099 4 4 4 7 7 7 F F F
14952024-02-01T17:28:34.513Z 5 100-119 12 12 12 18 18 18 F F F
14962024-02-01T17:28:34.513Z 6 120-139 10 10 10 16 16 16 F F F
14972024-02-01T17:28:34.513Z 7 140-159 10 10 10 16 16 16 F F F
14982024-02-01T17:28:34.514Z 8 160-179 13 13 13 19 19 19 F F F
14992024-02-01T17:28:34.517Z 9 180-199 4 4 4 6 6 6 F F F
15002024-02-01T17:28:34.517Z 10 200-219 4 4 4 7 7 7 F F F
15012024-02-01T17:28:34.517Z 11 220-239 9 9 9 13 13 13 F F F
15022024-02-01T17:28:34.517Z 12 240-259 5 5 5 8 8 8 F F F
15032024-02-01T17:28:34.517Z 13 260-279 13 13 13 19 19 19 F F F
15042024-02-01T17:28:34.517Z 14 280-299 13 13 13 19 19 19 F F F
15052024-02-01T17:28:34.517Z 15 300-319 12 12 12 18 18 18 F F F
15062024-02-01T17:28:34.517Z 16 320-339 7 7 7 10 10 10 F F F
15072024-02-01T17:28:34.517Z 17 340-359 10 10 10 14 14 14 F F F
15082024-02-01T17:28:34.517Z 18 360-379 13 13 13 19 19 19 F F F
15092024-02-01T17:28:34.518Z 19 380-399 13 13 13 19 19 19 F F F
15102024-02-01T17:28:34.518Z 20 400-419 10 10 10 14 14 14 F F F
15112024-02-01T17:28:34.518Z 21 420-439 13 13 13 19 19 19 F F F
15122024-02-01T17:28:34.518Z 22 440-459 13 13 13 19 19 19 F F F
15132024-02-01T17:28:34.518Z 23 460-479 10 10 10 14 14 14 F F F
15142024-02-01T17:28:34.518Z 24 480-499 11 11 11 17 17 17 F F F
15152024-02-01T17:28:34.518Z 25 500-519 10 10 10 14 14 14 F F F
15162024-02-01T17:28:34.518Z 26 520-539 12 12 12 18 18 18 F F F
15172024-02-01T17:28:34.518Z 27 540-559 10 10 10 14 14 14 F F F
15182024-02-01T17:28:34.518Z 28 560-579 13 13 13 19 19 19 F F F
15192024-02-01T17:28:34.518Z 29 580-599 8 8 8 12 12 12 F F F
15202024-02-01T17:28:34.518ZMax gen: 13, Max flush: 19
15212024-02-01T17:28:34.518Z
15222024-02-01T17:28:34.518ZBegin loop 7
15232024-02-01T17:28:34.518ZDownstairs to restart: 1
15242024-02-01T17:28:34.540Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800094109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
15252024-02-01T17:28:34.540Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800620179Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
15262024-02-01T17:28:34.550Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800689601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
15272024-02-01T17:28:34.550Z{"msg":"Crucible 931f6fa4-2366-4b12-91a1-01bfa7675049 has session id: a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800782842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15282024-02-01T17:28:34.550Z{"msg":"Crucible runtime is spawned
15292024-02-01T17:28:34.550ZUpstairs opts: Upstairs UUID: 931f6fa4-2366-4b12-91a1-01bfa7675049, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800875793Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15302024-02-01T17:28:34.550Z{"msg":"Crucible stats registered with UUID: 931f6fa4-2366-4b12-91a1-01bfa7675049","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800913193Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15312024-02-01T17:28:34.550Z{"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.800966893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
15322024-02-01T17:28:34.550Z{"msg":"931f6fa4-2366-4b12-91a1-01bfa7675049 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801203913Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15332024-02-01T17:28:34.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801305846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15342024-02-01T17:28:34.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801460658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15352024-02-01T17:28:34.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801648226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15362024-02-01T17:28:34.551Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801867993Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15372024-02-01T17:28:34.551Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.801976629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15382024-02-01T17:28:34.552Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802027146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15392024-02-01T17:28:34.552Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802059089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15402024-02-01T17:28:34.552Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802125535Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15412024-02-01T17:28:34.552Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802169615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15422024-02-01T17:28:34.552Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.80220665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15432024-02-01T17:28:34.553Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802303938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15442024-02-01T17:28:34.553Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.802369369Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"io task","client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15452024-02-01T17:28:34.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803001819Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15462024-02-01T17:28:34.553Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803089184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15472024-02-01T17:28:34.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803231733Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15482024-02-01T17:28:34.553Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803339555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15492024-02-01T17:28:34.553Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803540268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15502024-02-01T17:28:34.553Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.803600211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15512024-02-01T17:28:34.553Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.804086288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15522024-02-01T17:28:34.553Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.804465928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15532024-02-01T17:28:34.553Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.804679961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15542024-02-01T17:28:34.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.80486784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15552024-02-01T17:28:34.553Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.804929753Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15562024-02-01T17:28:34.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.804956352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15572024-02-01T17:28:34.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805228241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15582024-02-01T17:28:34.554Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805289669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15592024-02-01T17:28:34.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805316328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15602024-02-01T17:28:34.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805619575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15612024-02-01T17:28:34.554Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805702833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15622024-02-01T17:28:34.554Z{"msg":"[0]R flush_numbers[0..12]: [5, 10, 19, 13, 7, 18, 16, 16, 19, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805747429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15632024-02-01T17:28:34.555Z{"msg":"[0]R generation[0..12]: [3, 7, 13, 9, 4, 12, 10, 10, 13, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805813513Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15642024-02-01T17:28:34.555Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805860042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15652024-02-01T17:28:34.555Z{"msg":"[1]R flush_numbers[0..12]: [5, 10, 19, 13, 7, 18, 16, 16, 19, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.805929722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15662024-02-01T17:28:34.555Z{"msg":"[1]R generation[0..12]: [3, 7, 13, 9, 4, 12, 10, 10, 13, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806000315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15672024-02-01T17:28:34.555Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806075805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15682024-02-01T17:28:34.555Z{"msg":"[2]R flush_numbers[0..12]: [5, 10, 19, 13, 7, 18, 16, 16, 19, 6, 7, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806145872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15692024-02-01T17:28:34.555Z{"msg":"[2]R generation[0..12]: [3, 7, 13, 9, 4, 12, 10, 10, 13, 4, 4, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806194167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15702024-02-01T17:28:34.555Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15712024-02-01T17:28:34.555Z,"time":"2024-02-01T17:28:33.806269954Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15722024-02-01T17:28:34.555Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806296229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15732024-02-01T17:28:34.557Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.80631958Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15742024-02-01T17:28:34.557Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806387292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15752024-02-01T17:28:34.557Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806432943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"":"downstairs","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15762024-02-01T17:28:34.557Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806521122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15772024-02-01T17:28:34.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.80656397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"0","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15782024-02-01T17:28:34.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806637238Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"1","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15792024-02-01T17:28:34.557Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806680823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"client":"2","":"downstairs","session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15802024-02-01T17:28:34.557Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806770832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15812024-02-01T17:28:34.558Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806816726Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15822024-02-01T17:28:34.558Z{"msg":"931f6fa4-2366-4b12-91a1-01bfa7675049 is now active with session: a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806856834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15832024-02-01T17:28:34.558Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.806923891Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185,"session_id":"a9e8c5d5-ac96-41fa-b7c8-14b0d98ea2de"}
15842024-02-01T17:28:34.558Z{"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:33.807011956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
15852024-02-01T17:28:34.558ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15862024-02-01T17:28:34.558ZLoading write count information from file "/tmp/test_repair_verify.data"
15872024-02-01T17:28:34.558ZRun Repair workload
15882024-02-01T17:28:34.558Z01/30 Write block 470 len 12288 data: 2 2 2
15892024-02-01T17:28:34.561Z02/30 Read block 429 len 40960
15902024-02-01T17:28:34.562Z03/30 Write block 597 len 12288 data: 2 2 2
15912024-02-01T17:28:34.562Z04/30 Write block 53 len 8192 data: 2 3
15922024-02-01T17:28:34.562Z05/30 Write block 450 len 20480 data: 3 3 3 3 3
15932024-02-01T17:28:34.562Z06/30 Write block 456 len 28672 data: 3 3 3 3 3 2 2
15942024-02-01T17:28:34.562Z07/30 Write block 426 len 4096 data: 2
15952024-02-01T17:28:34.562Z08/30 Write block 41 len 24576 data: 2 2 2 2 2 2
15962024-02-01T17:28:34.562Z09/30 Write block 437 len 40960 data: 5 3 4 3 3 3 3 4 4 4
15972024-02-01T17:28:34.562Z10/30 Read block 306 len 40960
15982024-02-01T17:28:34.562Z11/30 Flush
15992024-02-01T17:28:34.571Z12/30 Write block 37 len 36864 data: 3 3 3 2 3 3 3 3 3
16002024-02-01T17:28:34.574Z13/30 Read block 583 len 8192
16012024-02-01T17:28:34.574Z14/30 Read block 56 len 4096
16022024-02-01T17:28:34.574Z15/30 Write block 50 len 28672 data: 2 2 2 3 4 4 3
16032024-02-01T17:28:34.574Z16/30 Read block 458 len 4096
16042024-02-01T17:28:34.575Z17/30 Read block 404 len 16384
16052024-02-01T17:28:34.575Z18/30 Read block 484 len 24576
16062024-02-01T17:28:34.575Z19/30 Read block 241 len 24576
16072024-02-01T17:28:34.578Z20/30 Write block 204 len 16384 data: 4 4 4 4
16082024-02-01T17:28:34.578Z21/30 Write block 414 len 24576 data: 3 3 4 2 2 2
16092024-02-01T17:28:34.578Z22/30 Write block 174 len 24576 data: 3 3 3 2 2 2
16102024-02-01T17:28:34.578Z23/30 Read block 561 len 12288
16112024-02-01T17:28:34.581Z24/30 Write block 376 len 40960 data: 5 5 5 5 5 5 3 2 2 2
16122024-02-01T17:28:34.581Z25/30 Read block 84 len 20480
16132024-02-01T17:28:34.584Z26/30 Flush
16142024-02-01T17:28:34.590Z27/30 Write block 202 len 32768 data: 3 3 5 5 5 5 4 2
16152024-02-01T17:28:34.593Z28/30 Read block 253 len 8192
16162024-02-01T17:28:34.593Z29/30 Read block 503 len 40960
16172024-02-01T17:28:34.593Z30/30 Write block 492 len 20480 data: 3 3 3 3 3
16182024-02-01T17:28:34.594Z----------------------------------------------------------------
16192024-02-01T17:28:34.594Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
16202024-02-01T17:28:34.594ZStates: Active Active Active
16212024-02-01T17:28:34.594ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16222024-02-01T17:28:34.594Z 1 Acked 1000 Write 3 Done Done Done false
16232024-02-01T17:28:34.594Z 2 Acked 1001 Read 10 Done Sent Done false
16242024-02-01T17:28:34.594Z 3 Acked 1002 Write 3 Done Sent Done false
16252024-02-01T17:28:34.594Z 4 Acked 1003 Write 2 Done Sent Done false
16262024-02-01T17:28:34.594Z 5 Acked 1004 Write 5 Done Sent Done false
16272024-02-01T17:28:34.594Z 6 Acked 1005 Write 7 Done Sent Done false
16282024-02-01T17:28:34.594Z 7 Acked 1006 Write 1 Done Sent Done false
16292024-02-01T17:28:34.594Z 8 Acked 1007 Write 6 Done Sent Done false
16302024-02-01T17:28:34.594Z 9 Acked 1008 Write 10 Done Sent Done false
16312024-02-01T17:28:34.594Z 10 Acked 1009 Read 10 Done Sent Done false
16322024-02-01T17:28:34.594Z 11 Acked 1010 Flush 0 Done Sent Done false
16332024-02-01T17:28:34.594Z 12 Acked 1011 Write 9 Done Sent Done false
16342024-02-01T17:28:34.594Z 13 Acked 1012 Read 2 Done Sent Done false
16352024-02-01T17:28:34.594Z 14 Acked 1013 Read 1 Done Sent Done false
16362024-02-01T17:28:34.594Z 15 Acked 1014 Write 7 Done Sent Done false
16372024-02-01T17:28:34.594Z 16 Acked 1015 Read 1 Done Sent Done false
16382024-02-01T17:28:34.594Z 17 Acked 1016 Read 4 Done Sent Done false
16392024-02-01T17:28:34.594Z 18 Acked 1017 Read 6 Done Sent Done false
16402024-02-01T17:28:34.595Z 19 Acked 1018 Read 6 Done Sent Done false
16412024-02-01T17:28:34.595Z 20 Acked 1019 Write 4 Done Sent Done false
16422024-02-01T17:28:34.595Z 21 Acked 1020 Write 6 Done Sent Done false
16432024-02-01T17:28:34.595Z 22 Acked 1021 Write 6 Done Sent Done false
16442024-02-01T17:28:34.595Z 23 Acked 1022 Read 3 Done Sent Done false
16452024-02-01T17:28:34.595Z 24 Acked 1023 Write 10 Done Sent Done false
16462024-02-01T17:28:34.595Z 25 Acked 1024 Read 5 Done Sent Done false
16472024-02-01T17:28:34.595Z 26 Acked 1025 Flush 0 Done Sent Done false
16482024-02-01T17:28:34.595Z 27 Acked 1026 Write 8 Done Sent Done false
16492024-02-01T17:28:34.595Z 28 Acked 1027 Read 2 Done Sent Done false
16502024-02-01T17:28:34.595Z 29 Acked 1028 Read 10 Done Sent Done false
16512024-02-01T17:28:34.595Z 30 Acked 1029 Write 5 Sent Sent Sent false
16522024-02-01T17:28:34.595Z STATES DS:0 DS:1 DS:2 TOTAL
16532024-02-01T17:28:34.595Z New 0 0 0 0
16542024-02-01T17:28:34.595Z Sent 1 29 1 31
16552024-02-01T17:28:34.595Z Done 29 1 29 59
16562024-02-01T17:28:34.595Z Skipped 0 0 0 0
16572024-02-01T17:28:34.595Z Error 0 0 0 0
16582024-02-01T17:28:34.595ZLast Flush: 1025 0 1025
16592024-02-01T17:28:34.595ZDownstairs last five completed:
16602024-02-01T17:28:34.595ZUpstairs last five completed: 30 29 28 27 26
16612024-02-01T17:28:34.595Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:33.854235107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1185}
16622024-02-01T17:28:34.595ZWrote out file "/tmp/test_repair_verify.data"
16632024-02-01T17:28:34.599Z
16642024-02-01T17:28:34.604ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16652024-02-01T17:28:34.628ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16662024-02-01T17:28:34.628Z 0 000-019 3 3 3 5 5 5 F F F
16672024-02-01T17:28:34.628Z 1 020-039 14 7 14 21 10 21 F F F <---
16682024-02-01T17:28:34.628Z 2 040-059 14 13 14 21 19 21 F F F <---
16692024-02-01T17:28:34.628Z 3 060-079 9 9 9 13 13 13 F F F
16702024-02-01T17:28:34.628Z 4 080-099 4 4 4 7 7 7 F F F
16712024-02-01T17:28:34.631Z 5 100-119 12 12 12 18 18 18 F F F
16722024-02-01T17:28:34.631Z 6 120-139 10 10 10 16 16 16 F F F
16732024-02-01T17:28:34.631Z 7 140-159 10 10 10 16 16 16 F F F
16742024-02-01T17:28:34.631Z 8 160-179 14 13 14 21 19 21 F F F <---
16752024-02-01T17:28:34.631Z 9 180-199 4 4 4 6 6 6 F F F
16762024-02-01T17:28:34.632Z 10 200-219 14 4 14 21 7 21 T F T <---
16772024-02-01T17:28:34.632Z 11 220-239 9 9 9 13 13 13 F F F
16782024-02-01T17:28:34.632Z 12 240-259 5 5 5 8 8 8 F F F
16792024-02-01T17:28:34.632Z 13 260-279 13 13 13 19 19 19 F F F
16802024-02-01T17:28:34.632Z 14 280-299 13 13 13 19 19 19 F F F
16812024-02-01T17:28:34.632Z 15 300-319 12 12 12 18 18 18 F F F
16822024-02-01T17:28:34.632Z 16 320-339 7 7 7 10 10 10 F F F
16832024-02-01T17:28:34.632Z 17 340-359 10 10 10 14 14 14 F F F
16842024-02-01T17:28:34.632Z 18 360-379 14 13 14 21 19 21 F F F <---
16852024-02-01T17:28:34.632Z 19 380-399 14 13 14 21 19 21 F F F <---
16862024-02-01T17:28:34.632Z 20 400-419 14 10 14 21 14 21 F F F <---
16872024-02-01T17:28:34.632Z 21 420-439 14 13 14 20 19 20 F F F <---
16882024-02-01T17:28:34.632Z 22 440-459 14 13 14 20 19 20 F F F <---
16892024-02-01T17:28:34.632Z 23 460-479 14 10 14 20 14 20 F T F <---
16902024-02-01T17:28:34.632Z 24 480-499 11 11 11 17 17 17 T F T <---
16912024-02-01T17:28:34.632Z 25 500-519 10 10 10 14 14 14 F F F
16922024-02-01T17:28:34.632Z 26 520-539 12 12 12 18 18 18 F F F
16932024-02-01T17:28:34.633Z 27 540-559 10 10 10 14 14 14 F F F
16942024-02-01T17:28:34.633Z 28 560-579 13 13 13 19 19 19 F F F
16952024-02-01T17:28:34.633Z 29 580-599 14 8 14 20 12 20 F F F <---
16962024-02-01T17:28:34.633ZMax gen: 14, Max flush: 21
16972024-02-01T17:28:34.633ZError: Difference in extent metadata found!
16982024-02-01T17:28:34.636ZA Difference in extent metadata is expected here
16992024-02-01T17:28:34.636ZOn loop 7
17002024-02-01T17:28:34.636Z
17012024-02-01T17:28:34.636ZVerifying data now
17022024-02-01T17:28:35.669ZNo change after verify
17032024-02-01T17:28:35.669ZLoop: 7 Downstairs dump after verify (and repair):
17042024-02-01T17:28:35.692ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17052024-02-01T17:28:35.692Z 0 000-019 3 3 3 5 5 5 F F F
17062024-02-01T17:28:35.696Z 1 020-039 14 14 14 21 21 21 F F F
17072024-02-01T17:28:35.696Z 2 040-059 14 14 14 21 21 21 F F F
17082024-02-01T17:28:35.696Z 3 060-079 9 9 9 13 13 13 F F F
17092024-02-01T17:28:35.696Z 4 080-099 4 4 4 7 7 7 F F F
17102024-02-01T17:28:35.696Z 5 100-119 12 12 12 18 18 18 F F F
17112024-02-01T17:28:35.696Z 6 120-139 10 10 10 16 16 16 F F F
17122024-02-01T17:28:35.696Z 7 140-159 10 10 10 16 16 16 F F F
17132024-02-01T17:28:35.696Z 8 160-179 14 14 14 21 21 21 F F F
17142024-02-01T17:28:35.696Z 9 180-199 4 4 4 6 6 6 F F F
17152024-02-01T17:28:35.697Z 10 200-219 15 15 15 22 22 22 F F F
17162024-02-01T17:28:35.697Z 11 220-239 9 9 9 13 13 13 F F F
17172024-02-01T17:28:35.697Z 12 240-259 5 5 5 8 8 8 F F F
17182024-02-01T17:28:35.697Z 13 260-279 13 13 13 19 19 19 F F F
17192024-02-01T17:28:35.697Z 14 280-299 13 13 13 19 19 19 F F F
17202024-02-01T17:28:35.697Z 15 300-319 12 12 12 18 18 18 F F F
17212024-02-01T17:28:35.697Z 16 320-339 7 7 7 10 10 10 F F F
17222024-02-01T17:28:35.697Z 17 340-359 10 10 10 14 14 14 F F F
17232024-02-01T17:28:35.697Z 18 360-379 14 14 14 21 21 21 F F F
17242024-02-01T17:28:35.697Z 19 380-399 14 14 14 21 21 21 F F F
17252024-02-01T17:28:35.697Z 20 400-419 14 14 14 21 21 21 F F F
17262024-02-01T17:28:35.697Z 21 420-439 14 14 14 20 20 20 F F F
17272024-02-01T17:28:35.697Z 22 440-459 14 14 14 20 20 20 F F F
17282024-02-01T17:28:35.697Z 23 460-479 14 14 14 20 20 20 F F F
17292024-02-01T17:28:35.697Z 24 480-499 15 15 15 22 22 22 F F F
17302024-02-01T17:28:35.697Z 25 500-519 10 10 10 14 14 14 F F F
17312024-02-01T17:28:35.697Z 26 520-539 12 12 12 18 18 18 F F F
17322024-02-01T17:28:35.697Z 27 540-559 10 10 10 14 14 14 F F F
17332024-02-01T17:28:35.697Z 28 560-579 13 13 13 19 19 19 F F F
17342024-02-01T17:28:35.697Z 29 580-599 14 14 14 20 20 20 F F F
17352024-02-01T17:28:35.697ZMax gen: 15, Max flush: 22
17362024-02-01T17:28:35.700Z
17372024-02-01T17:28:35.700ZBegin loop 8
17382024-02-01T17:28:35.701ZDownstairs to restart: 1
17392024-02-01T17:28:35.726Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.987752665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
17402024-02-01T17:28:35.726Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.98834316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
17412024-02-01T17:28:35.731Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988393273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
17422024-02-01T17:28:35.731Z{"msg":"Crucible 9921d2d6-1515-420e-894a-311721631f9f has session id: 77adb42c-5cee-488f-82af-415eb77f5e2e","v":0,"name":Crucible runtime is spawned
17432024-02-01T17:28:35.731Z"crucible","level":30,"time":"2024-02-01T17:28:34.988570677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17442024-02-01T17:28:35.731Z{"msg":"Upstairs opts: Upstairs UUID: 9921d2d6-1515-420e-894a-311721631f9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988672197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17452024-02-01T17:28:35.731Z{"msg":"Crucible stats registered with UUID: 9921d2d6-1515-420e-894a-311721631f9f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988742809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17462024-02-01T17:28:35.731Z{"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988770898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
17472024-02-01T17:28:35.731Z{"msg":"9921d2d6-1515-420e-894a-311721631f9f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988803459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17482024-02-01T17:28:35.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988840013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17492024-02-01T17:28:35.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.988957222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17502024-02-01T17:28:35.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989036129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17512024-02-01T17:28:35.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989127421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17522024-02-01T17:28:35.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989272809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17532024-02-01T17:28:35.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989321802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17542024-02-01T17:28:35.731Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989470006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17552024-02-01T17:28:35.731Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989640175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17562024-02-01T17:28:35.732Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989700655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17572024-02-01T17:28:35.732Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989745235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17582024-02-01T17:28:35.732Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.989825165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17592024-02-01T17:28:35.732Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.98990983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"io task","client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17602024-02-01T17:28:35.732Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.990644602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17612024-02-01T17:28:35.735Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.990707511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17622024-02-01T17:28:35.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.990733396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17632024-02-01T17:28:35.735Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.990775736Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17642024-02-01T17:28:35.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.99079999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17652024-02-01T17:28:35.735Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.990943292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17662024-02-01T17:28:35.740Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.991876266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17672024-02-01T17:28:35.740Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.9920427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17682024-02-01T17:28:35.740Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.992159968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17692024-02-01T17:28:35.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993008081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17702024-02-01T17:28:35.740Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993123251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17712024-02-01T17:28:35.740Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993156296Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17722024-02-01T17:28:35.740Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993189719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17732024-02-01T17:28:35.741Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993231019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17742024-02-01T17:28:35.741Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993278548Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17752024-02-01T17:28:35.741Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993301772Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17762024-02-01T17:28:35.741Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993324702Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17772024-02-01T17:28:35.741Z{"msg":"[0]R flush_numbers[0..12]: [5, 21, 21, 13, 7, 18, 16, 16, 21, 6, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993366081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17782024-02-01T17:28:35.741Z{"msg":"[0]R generation[0..12]: [3, 14, 14, 9, 4, 12, 10, 10, 14, 4, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993391059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17792024-02-01T17:28:35.741Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993433061Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17802024-02-01T17:28:35.741Z{"msg":"[1]R flush_numbers[0..12]: [5, 21, 21, 13, 7, 18, 16, 16, 21, 6, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993457791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17812024-02-01T17:28:35.741Z{"msg":"[1]R generation[0..12]: [3, 14, 14, 9, 4, 12, 10, 10, 14, 4, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993505591Z","hostname":"Wait for a query_work_queue command to finish before sending IO
17822024-02-01T17:28:35.741Zip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17832024-02-01T17:28:35.742Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993545668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17842024-02-01T17:28:35.742Z{"msg":"[2]R flush_numbers[0..12]: [5, 21, 21, 13, 7, 18, 16, 16, 21, 6, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.99361227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17852024-02-01T17:28:35.742Z{"msg":"[2]R generation[0..12]: [3, 14, 14, 9, 4, 12, 10, 10, 14, 4, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993643977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17862024-02-01T17:28:35.742Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993707792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17872024-02-01T17:28:35.742Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993751144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17882024-02-01T17:28:35.742Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993800741Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17892024-02-01T17:28:35.742Z{"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993825067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17902024-02-01T17:28:35.742Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993848163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"":"downstairs","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17912024-02-01T17:28:35.743Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993870258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17922024-02-01T17:28:35.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993890034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"0","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17932024-02-01T17:28:35.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993933925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"1","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17942024-02-01T17:28:35.743Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993957457Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"client":"2","":"downstairs","session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17952024-02-01T17:28:35.743Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.993979645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17962024-02-01T17:28:35.743Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.994017782Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17972024-02-01T17:28:35.743Z{"msg":"9921d2d6-1515-420e-894a-311721631f9f is now active with session: 77adb42c-5cee-488f-82af-415eb77f5e2e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.994065928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17982024-02-01T17:28:35.743Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.994088423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193,"session_id":"77adb42c-5cee-488f-82af-415eb77f5e2e"}
17992024-02-01T17:28:35.743Z{"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:34.994109107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
18002024-02-01T17:28:35.743ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18012024-02-01T17:28:35.743ZLoading write count information from file "/tmp/test_repair_verify.data"
18022024-02-01T17:28:35.743ZRun Repair workload
18032024-02-01T17:28:35.745Z01/30 Flush
18042024-02-01T17:28:35.745Z02/30 Write block 60 len 20480 data: 3 3 3 3 2
18052024-02-01T17:28:35.745Z03/30 Write block 174 len 32768 data: 4 4 4 3 3 3 2 3
18062024-02-01T17:28:35.745Z04/30 Write block 375 len 12288 data: 4 6 6
18072024-02-01T17:28:35.748Z05/30 Read block 320 len 28672
18082024-02-01T17:28:35.748Z06/30 Read block 7 len 16384
18092024-02-01T17:28:35.749Z07/30 Write block 185 len 20480 data: 3 3 2 2 2
18102024-02-01T17:28:35.749Z08/30 Read block 227 len 36864
18112024-02-01T17:28:35.749Z09/30 Read block 35 len 24576
18122024-02-01T17:28:35.749Z10/30 Write block 41 len 12288 data: 4 4 4
18132024-02-01T17:28:35.749Z11/30 Write block 92 len 36864 data: 2 2 2 2 2 2 2 2 2
18142024-02-01T17:28:35.752Z12/30 Read block 264 len 32768
18152024-02-01T17:28:35.752Z13/30 Read block 186 len 20480
18162024-02-01T17:28:35.755Z14/30 Read block 474 len 24576
18172024-02-01T17:28:35.758Z15/30 Write block 105 len 24576 data: 3 3 4 3 3 2
18182024-02-01T17:28:35.758Z16/30 Flush
18192024-02-01T17:28:35.760Z17/30 Write block 459 len 40960 data: 4 4 3 3 2 2 3 2 2 2
18202024-02-01T17:28:35.764Z18/30 Write block 586 len 20480 data: 3 3 3 3 3
18212024-02-01T17:28:35.764Z19/30 Flush
18222024-02-01T17:28:35.771Z20/30 Write block 312 len 36864 data: 3 3 2 2 2 2 2 2 2
18232024-02-01T17:28:35.771Z21/30 Read block 295 len 32768
18242024-02-01T17:28:35.778Z22/30 Read block 95 len 40960
18252024-02-01T17:28:35.778Z23/30 Write block 294 len 40960 data: 3 3 3 2 3 3 3 3 3 3
18262024-02-01T17:28:35.779Z24/30 Flush
18272024-02-01T17:28:35.794Z25/30 Write block 348 len 40960 data: 2 2 2 2 2 2 5 3 3 2
18282024-02-01T17:28:35.795Z26/30 Read block 404 len 40960
18292024-02-01T17:28:35.795Z27/30 Read block 193 len 20480
18302024-02-01T17:28:35.795Z28/30 Read block 212 len 32768
18312024-02-01T17:28:35.806Z29/30 Read block 515 len 12288
18322024-02-01T17:28:35.807Z30/30 Read block 255 len 36864
18332024-02-01T17:28:35.807Z----------------------------------------------------------------
18342024-02-01T17:28:35.807Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:29
18352024-02-01T17:28:35.807ZStates: Active Active Active
18362024-02-01T17:28:35.807ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18372024-02-01T17:28:35.807Z 2 Acked 1001 Write 5 Done Sent Done false
18382024-02-01T17:28:35.807Z 3 Acked 1002 Write 8 Done Sent Done false
18392024-02-01T17:28:35.807Z 4 Acked 1003 Write 3 Done Sent Done false
18402024-02-01T17:28:35.807Z 5 Acked 1004 Read 7 Done Sent Done false
18412024-02-01T17:28:35.807Z 6 Acked 1005 Read 4 Done Sent Done false
18422024-02-01T17:28:35.807Z 7 Acked 1006 Write 5 Done Sent Done false
18432024-02-01T17:28:35.808Z 8 Acked 1007 Read 9 Done Sent Done false
18442024-02-01T17:28:35.808Z 9 Acked 1008 Read 6 Done Sent Done false
18452024-02-01T17:28:35.808Z 10 Acked 1009 Write 3 Done Sent Done false
18462024-02-01T17:28:35.808Z 11 Acked 1010 Write 9 Done Sent Done false
18472024-02-01T17:28:35.808Z 12 Acked 1011 Read 8 Done Sent Done false
18482024-02-01T17:28:35.808Z 13 Acked 1012 Read 5 Done Sent Done false
18492024-02-01T17:28:35.808Z 14 Acked 1013 Read 6 Done Sent Done false
18502024-02-01T17:28:35.808Z 15 Acked 1014 Write 6 Done Sent Done false
18512024-02-01T17:28:35.808Z 16 Acked 1015 Flush 0 Done Sent Done false
18522024-02-01T17:28:35.809Z 17 Acked 1016 Write 10 Done Sent Done false
18532024-02-01T17:28:35.809Z 18 Acked 1017 Write 5 Done Sent Done false
18542024-02-01T17:28:35.809Z 19 Acked 1018 Flush 0 Done Sent Done false
18552024-02-01T17:28:35.809Z 20 Acked 1019 Write 9 Done Sent Done false
18562024-02-01T17:28:35.809Z 21 Acked 1020 Read 8 Done Sent Done false
18572024-02-01T17:28:35.809Z 22 Acked 1021 Read 10 Done Sent Done false
18582024-02-01T17:28:35.809Z 23 Acked 1022 Write 10 Done Sent Done false
18592024-02-01T17:28:35.809Z 24 Acked 1023 Flush 0 Done Sent Done false
18602024-02-01T17:28:35.809Z 25 Acked 1024 Write 10 Done Sent Done false
18612024-02-01T17:28:35.810Z 26 Acked 1025 Read 10 Done Sent Done false
18622024-02-01T17:28:35.810Z 27 Acked 1026 Read 5 Done Sent Done false
18632024-02-01T17:28:35.810Z 28 Acked 1027 Read 8 Done Sent Done false
18642024-02-01T17:28:35.810Z 29 Acked 1028 Read 3 Done Sent Done false
18652024-02-01T17:28:35.810Z 30 Acked 1029 Read 9 Done Sent Done false
18662024-02-01T17:28:35.810Z STATES DS:0 DS:1 DS:2 TOTAL
18672024-02-01T17:28:35.810Z New 0 0 0 0
18682024-02-01T17:28:35.810Z Sent 0 29 0 29
18692024-02-01T17:28:35.810Z Done 29 0 29 58
18702024-02-01T17:28:35.811Z Skipped 0 0 0 0
18712024-02-01T17:28:35.811Z Error 0 0 0 0
18722024-02-01T17:28:35.811ZLast Flush: 1023 1000 1023
18732024-02-01T17:28:35.811ZDownstairs last five completed: 1000
18742024-02-01T17:28:35.811ZUpstairs last five completed: 30 29 28 27 26
18752024-02-01T17:28:35.811Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:35.048787182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1193}
18762024-02-01T17:28:35.812ZWrote out file "/tmp/test_repair_verify.data"
18772024-02-01T17:28:35.812Z
18782024-02-01T17:28:35.812ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
18792024-02-01T17:28:35.831ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
18802024-02-01T17:28:35.831Z 0 000-019 3 3 3 5 5 5 F F F
18812024-02-01T17:28:35.831Z 1 020-039 14 14 14 21 21 21 F F F
18822024-02-01T17:28:35.831Z 2 040-059 16 14 16 24 21 24 F F F <---
18832024-02-01T17:28:35.831Z 3 060-079 16 9 16 24 13 24 F F F <---
18842024-02-01T17:28:35.831Z 4 080-099 16 4 16 24 7 24 F F F <---
18852024-02-01T17:28:35.831Z 5 100-119 16 12 16 24 18 24 F F F <---
18862024-02-01T17:28:35.845Z 6 120-139 10 10 10 16 16 16 F F F
18872024-02-01T17:28:35.845Z 7 140-159 10 10 10 16 16 16 F F F
18882024-02-01T17:28:35.846Z 8 160-179 16 14 16 24 21 24 F F F <---
18892024-02-01T17:28:35.846Z 9 180-199 16 4 16 24 6 24 F F F <---
18902024-02-01T17:28:35.846Z 10 200-219 15 15 15 22 22 22 F F F
18912024-02-01T17:28:35.846Z 11 220-239 9 9 9 13 13 13 F F F
18922024-02-01T17:28:35.846Z 12 240-259 5 5 5 8 8 8 F F F
18932024-02-01T17:28:35.846Z 13 260-279 13 13 13 19 19 19 F F F
18942024-02-01T17:28:35.846Z 14 280-299 16 13 16 26 19 26 F F F <---
18952024-02-01T17:28:35.846Z 15 300-319 16 12 16 26 18 26 F F F <---
18962024-02-01T17:28:35.847Z 16 320-339 16 7 16 26 10 26 F F F <---
18972024-02-01T17:28:35.847Z 17 340-359 10 10 10 14 14 14 T F T <---
18982024-02-01T17:28:35.847Z 18 360-379 16 14 16 24 21 24 F F F <---
18992024-02-01T17:28:35.847Z 19 380-399 14 14 14 21 21 21 F F F
19002024-02-01T17:28:35.847Z 20 400-419 14 14 14 21 21 21 F F F
19012024-02-01T17:28:35.847Z 21 420-439 14 14 14 20 20 20 F F F
19022024-02-01T17:28:35.847Z 22 440-459 16 14 16 25 20 25 F F F <---
19032024-02-01T17:28:35.847Z 23 460-479 16 14 16 25 20 25 F F F <---
19042024-02-01T17:28:35.847Z 24 480-499 15 15 15 22 22 22 F F F
19052024-02-01T17:28:35.848Z 25 500-519 10 10 10 14 14 14 F F F
19062024-02-01T17:28:35.848Z 26 520-539 12 12 12 18 18 18 F F F
19072024-02-01T17:28:35.848Z 27 540-559 10 10 10 14 14 14 F F F
19082024-02-01T17:28:35.848Z 28 560-579 13 13 13 19 19 19 F F F
19092024-02-01T17:28:35.848Z 29 580-599 16 14 16 25 20 25 F F F <---
19102024-02-01T17:28:35.848ZMax gen: 16, Max flush: 26
19112024-02-01T17:28:35.848ZError: Difference in extent metadata found!
19122024-02-01T17:28:35.848ZA Difference in extent metadata is expected here
19132024-02-01T17:28:35.849ZOn loop 8
19142024-02-01T17:28:35.849Z
19152024-02-01T17:28:35.849ZVerifying data now
19162024-02-01T17:28:40.986ZNo change after verify
19172024-02-01T17:28:40.987ZLoop: 8 Downstairs dump after verify (and repair):
19182024-02-01T17:28:41.011ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19192024-02-01T17:28:41.011Z 0 000-019 3 3 3 5 5 5 F F F
19202024-02-01T17:28:41.023Z 1 020-039 14 14 14 21 21 21 F F F
19212024-02-01T17:28:41.023Z 2 040-059 16 16 16 24 24 24 F F F
19222024-02-01T17:28:41.023Z 3 060-079 16 16 16 24 24 24 F F F
19232024-02-01T17:28:41.023Z 4 080-099 16 16 16 24 24 24 F F F
19242024-02-01T17:28:41.023Z 5 100-119 16 16 16 24 24 24 F F F
19252024-02-01T17:28:41.023Z 6 120-139 10 10 10 16 16 16 F F F
19262024-02-01T17:28:41.023Z 7 140-159 10 10 10 16 16 16 F F F
19272024-02-01T17:28:41.023Z 8 160-179 16 16 16 24 24 24 F F F
19282024-02-01T17:28:41.023Z 9 180-199 16 16 16 24 24 24 F F F
19292024-02-01T17:28:41.023Z 10 200-219 15 15 15 22 22 22 F F F
19302024-02-01T17:28:41.023Z 11 220-239 9 9 9 13 13 13 F F F
19312024-02-01T17:28:41.023Z 12 240-259 5 5 5 8 8 8 F F F
19322024-02-01T17:28:41.023Z 13 260-279 13 13 13 19 19 19 F F F
19332024-02-01T17:28:41.023Z 14 280-299 16 16 16 26 26 26 F F F
19342024-02-01T17:28:41.023Z 15 300-319 16 16 16 26 26 26 F F F
19352024-02-01T17:28:41.023Z 16 320-339 16 16 16 26 26 26 F F F
19362024-02-01T17:28:41.023Z 17 340-359 17 17 17 27 27 27 F F F
19372024-02-01T17:28:41.023Z 18 360-379 16 16 16 24 24 24 F F F
19382024-02-01T17:28:41.023Z 19 380-399 14 14 14 21 21 21 F F F
19392024-02-01T17:28:41.024Z 20 400-419 14 14 14 21 21 21 F F F
19402024-02-01T17:28:41.024Z 21 420-439 14 14 14 20 20 20 F F F
19412024-02-01T17:28:41.024Z 22 440-459 16 16 16 25 25 25 F F F
19422024-02-01T17:28:41.024Z 23 460-479 16 16 16 25 25 25 F F F
19432024-02-01T17:28:41.024Z 24 480-499 15 15 15 22 22 22 F F F
19442024-02-01T17:28:41.024Z 25 500-519 10 10 10 14 14 14 F F F
19452024-02-01T17:28:41.024Z 26 520-539 12 12 12 18 18 18 F F F
19462024-02-01T17:28:41.024Z 27 540-559 10 10 10 14 14 14 F F F
19472024-02-01T17:28:41.024Z 28 560-579 13 13 13 19 19 19 F F F
19482024-02-01T17:28:41.024Z 29 580-599 16 16 16 25 25 25 F F F
19492024-02-01T17:28:41.024ZMax gen: 17, Max flush: 27
19502024-02-01T17:28:41.024Z
19512024-02-01T17:28:41.024ZBegin loop 9
19522024-02-01T17:28:41.024ZDownstairs to restart: 1
19532024-02-01T17:28:41.067Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.304861337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
19542024-02-01T17:28:41.106Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
19552024-02-01T17:28:41.106Z,"time":"2024-02-01T17:28:40.30556357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
19562024-02-01T17:28:41.106Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.305717027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
19572024-02-01T17:28:41.106Z{"msg":"Crucible f7bd2324-f383-4a90-be1b-f3e441d31f27 has session id: 7073f89b-ceac-4d46-9a28-ee63666d1156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.305800148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19582024-02-01T17:28:41.106Z{"msg":"Upstairs opts: Upstairs UUID: f7bd2324-f383-4a90-be1b-f3e441d31f27, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.30589926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19592024-02-01T17:28:41.107Z{"msg":"Crucible stats registered with UUID: f7bd2324-f383-4a90-be1b-f3e441d31f27","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.305972005Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19602024-02-01T17:28:41.107Z{"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306080334Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
19612024-02-01T17:28:41.108Z{"msg":"f7bd2324-f383-4a90-be1b-f3e441d31f27 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306112942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19622024-02-01T17:28:41.108Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306180462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19632024-02-01T17:28:41.108Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306290089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19642024-02-01T17:28:41.108Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306338022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19652024-02-01T17:28:41.108Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306384742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19662024-02-01T17:28:41.108Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306459815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19672024-02-01T17:28:41.108Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.30657821Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19682024-02-01T17:28:41.108Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306628058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19692024-02-01T17:28:41.108Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306672174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19702024-02-01T17:28:41.108Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306714196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19712024-02-01T17:28:41.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306759923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19722024-02-01T17:28:41.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306802854Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19732024-02-01T17:28:41.108Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.306974942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"io task","client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19742024-02-01T17:28:41.108Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.307779392Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19752024-02-01T17:28:41.108Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.30784897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19762024-02-01T17:28:41.108Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.3078985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19772024-02-01T17:28:41.108Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.307943532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19782024-02-01T17:28:41.108Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.308803559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19792024-02-01T17:28:41.109Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309053641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19802024-02-01T17:28:41.109Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309168226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19812024-02-01T17:28:41.109Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309216403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19822024-02-01T17:28:41.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309614786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19832024-02-01T17:28:41.109Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309653266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19842024-02-01T17:28:41.109Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.309757618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19852024-02-01T17:28:41.109Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.310145101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19862024-02-01T17:28:41.109Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.310220576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19872024-02-01T17:28:41.109Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.310346613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19882024-02-01T17:28:41.109Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.310447206Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19892024-02-01T17:28:41.110Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311036414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19902024-02-01T17:28:41.110Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311117515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19912024-02-01T17:28:41.110Z{"msg":"[0]R flush_numbers[0..12]: [5, 21, 24, 24, 24, 24, 16, 16, 24, 24, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311148136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19922024-02-01T17:28:41.110Z{"msg":"[0]R generation[0..12]: [3, 14, 16, 16, 16, 16, 10, 10, 16, 16, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311219638Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19932024-02-01T17:28:41.110Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311265877Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19942024-02-01T17:28:41.110Z{"msg":"[1]R flush_numbers[0..12]: [5, 21, 24, 24, 24, 24, 16, 16, 24, 24, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311291807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19952024-02-01T17:28:41.111Z{"msg":"[1]R generation[0..12]: [3, 14, 16, 16, 16, 16, 10, 10, 16, 16, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311315072Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19962024-02-01T17:28:41.111Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
19972024-02-01T17:28:41.111Z,"time":"2024-02-01T17:28:40.311357435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19982024-02-01T17:28:41.111Z{"msg":"[2]R flush_numbers[0..12]: [5, 21, 24, 24, 24, 24, 16, 16, 24, 24, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311384182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
19992024-02-01T17:28:41.111Z{"msg":"[2]R generation[0..12]: [3, 14, 16, 16, 16, 16, 10, 10, 16, 16, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.31142939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20002024-02-01T17:28:41.111Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311455721Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20012024-02-01T17:28:41.111Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311517315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20022024-02-01T17:28:41.111Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311549532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20032024-02-01T17:28:41.111Z{"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.31160927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20042024-02-01T17:28:41.111Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.31168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"":"downstairs","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20052024-02-01T17:28:41.111Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311748292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20062024-02-01T17:28:41.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.31177942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"0","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20072024-02-01T17:28:41.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311846888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"1","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20082024-02-01T17:28:41.111Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20092024-02-01T17:28:41.112Z,"time":"2024-02-01T17:28:40.311921483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"client":"2","":"downstairs","session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20102024-02-01T17:28:41.112Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.311957965Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20112024-02-01T17:28:41.112Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.312017091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20122024-02-01T17:28:41.112Z{"msg":"f7bd2324-f383-4a90-be1b-f3e441d31f27 is now active with session: 7073f89b-ceac-4d46-9a28-ee63666d1156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.312044387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20132024-02-01T17:28:41.112Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.312083983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201,"session_id":"7073f89b-ceac-4d46-9a28-ee63666d1156"}
20142024-02-01T17:28:41.112Z{"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:40.312106557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
20152024-02-01T17:28:41.112ZLoading write count information from file "/tmp/test_repair_verify.data"
20162024-02-01T17:28:41.112ZRun Repair workload
20172024-02-01T17:28:41.112Z01/30 Write block 258 len 36864 data: 3 3 3 2 2 2 2 2 2
20182024-02-01T17:28:41.112Z02/30 Read block 130 len 8192
20192024-02-01T17:28:41.112Z03/30 Read block 331 len 12288
20202024-02-01T17:28:41.112Z04/30 Write block 315 len 12288 data: 3 3 3
20212024-02-01T17:28:41.112Z05/30 Read block 162 len 24576
20222024-02-01T17:28:41.112Z06/30 Write block 245 len 40960 data: 2 2 2 2 2 2 2 2 2 3
20232024-02-01T17:28:41.112Z07/30 Write block 155 len 40960 data: 3 2 2 2 2 2 2 2 2 2
20242024-02-01T17:28:41.112Z08/30 Flush
20252024-02-01T17:28:41.112Z09/30 Write block 171 len 28672 data: 3 3 3 5 5 5 4
20262024-02-01T17:28:41.113Z10/30 Write block 529 len 24576 data: 2 3 3 4 3 4
20272024-02-01T17:28:41.114Z11/30 Read block 360 len 36864
20282024-02-01T17:28:41.114Z12/30 Flush
20292024-02-01T17:28:41.114Z13/30 Write block 383 len 4096 data: 3
20302024-02-01T17:28:41.114Z14/30 Read block 306 len 32768
20312024-02-01T17:28:41.114Z15/30 Read block 64 len 28672
20322024-02-01T17:28:41.114Z16/30 Read block 94 len 8192
20332024-02-01T17:28:41.114Z17/30 Write block 561 len 24576 data: 3 3 3 3 3 2
20342024-02-01T17:28:41.114Z18/30 Read block 388 len 36864
20352024-02-01T17:28:41.114Z19/30 Write block 550 len 16384 data: 3 3 3 3
20362024-02-01T17:28:41.114Z20/30 Write block 183 len 12288 data: 3 3 4
20372024-02-01T17:28:41.114Z21/30 Write block 563 len 12288 data: 4 4 4
20382024-02-01T17:28:41.114Z22/30 Read block 555 len 8192
20392024-02-01T17:28:41.124Z23/30 Read block 92 len 36864
20402024-02-01T17:28:41.124Z24/30 Write block 285 len 20480 data: 2 3 3 3 3
20412024-02-01T17:28:41.124Z25/30 Read block 276 len 12288
20422024-02-01T17:28:41.124Z26/30 Read block 148 len 4096
20432024-02-01T17:28:41.124Z27/30 Write block 55 len 40960 data: 5 4 4 3 3 4 4 4 4 3
20442024-02-01T17:28:41.124Z28/30 Write block 314 len 36864 data: 3 4 4 4 3 3 3 2 2
20452024-02-01T17:28:41.124Z29/30 Write block 557 len 16384 data: 3 3 3 3
20462024-02-01T17:28:41.124Z30/30 Read block 37 len 32768
20472024-02-01T17:28:41.124Z----------------------------------------------------------------
20482024-02-01T17:28:41.124Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
20492024-02-01T17:28:41.124ZStates: Active Active Active
20502024-02-01T17:28:41.125ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
20512024-02-01T17:28:41.125Z 1 Acked 1000 Write 9 Done Done Done false
20522024-02-01T17:28:41.125Z 2 Acked 1001 Read 2 Done Done Done false
20532024-02-01T17:28:41.125Z 3 Acked 1002 Read 3 Done Done Done false
20542024-02-01T17:28:41.125Z 4 Acked 1003 Write 3 Done Done Done false
20552024-02-01T17:28:41.125Z 5 Acked 1004 Read 6 Done Sent Done false
20562024-02-01T17:28:41.125Z 6 Acked 1005 Write 10 Done Sent Done false
20572024-02-01T17:28:41.125Z 7 Acked 1006 Write 10 Done Sent Done false
20582024-02-01T17:28:41.125Z 8 Acked 1007 Flush 0 Done Sent Done false
20592024-02-01T17:28:41.125Z 9 Acked 1008 Write 7 Done Sent Done false
20602024-02-01T17:28:41.126Z 10 Acked 1009 Write 6 Done Sent Done false
20612024-02-01T17:28:41.126Z 11 Acked 1010 Read 9 Done Sent Done false
20622024-02-01T17:28:41.126Z 12 Acked 1011 Flush 0 Done Sent Done false
20632024-02-01T17:28:41.126Z 13 Acked 1012 Write 1 Done Sent Done false
20642024-02-01T17:28:41.126Z 14 Acked 1013 Read 8 Done Sent Done false
20652024-02-01T17:28:41.126Z 15 Acked 1014 Read 7 Done Sent Done false
20662024-02-01T17:28:41.126Z 16 Acked 1015 Read 2 Done Sent Done false
20672024-02-01T17:28:41.126Z 17 Acked 1016 Write 6 Done Sent Done false
20682024-02-01T17:28:41.126Z 18 Acked 1017 Read 9 Done Sent Done false
20692024-02-01T17:28:41.126Z 19 Acked 1018 Write 4 Done Sent Done false
20702024-02-01T17:28:41.126Z 20 Acked 1019 Write 3 Done Sent Done false
20712024-02-01T17:28:41.126Z 21 Acked 1020 Write 3 Done Sent Done false
20722024-02-01T17:28:41.126Z 22 Acked 1021 Read 2 Done Sent Done false
20732024-02-01T17:28:41.127Z 23 Acked 1022 Read 9 Done Sent Done false
20742024-02-01T17:28:41.127Z 24 Acked 1023 Write 5 Done Sent Done false
20752024-02-01T17:28:41.127Z 25 Acked 1024 Read 3 Done Sent Done false
20762024-02-01T17:28:41.127Z 26 Acked 1025 Read 1 Done Sent Done false
20772024-02-01T17:28:41.127Z 27 Acked 1026 Write 10 Done Sent Done false
20782024-02-01T17:28:41.127Z 28 Acked 1027 Write 9 Done Sent Done false
20792024-02-01T17:28:41.127Z 29 Acked 1028 Write 4 Done Sent Done false
20802024-02-01T17:28:41.127Z 30 Acked 1029 Read 8 Sent Sent Done false
20812024-02-01T17:28:41.127Z STATES DS:0 DS:1 DS:2 TOTAL
20822024-02-01T17:28:41.127Z New 0 0 0 0
20832024-02-01T17:28:41.127Z Sent 1 26 0 27
20842024-02-01T17:28:41.127Z Done 29 4 30 63
20852024-02-01T17:28:41.127Z Skipped 0 0 0 0
20862024-02-01T17:28:41.127Z Error 0 0 0 0
20872024-02-01T17:28:41.127ZLast Flush: 1011 0 1011
20882024-02-01T17:28:41.128ZDownstairs last five completed:
20892024-02-01T17:28:41.128ZUpstairs last five completed: 30 29 28 27 26
20902024-02-01T17:28:41.128Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:40.358844345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1201}
20912024-02-01T17:28:41.128ZWrote out file "/tmp/test_repair_verify.data"
20922024-02-01T17:28:41.128Z
20932024-02-01T17:28:41.128ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
20942024-02-01T17:28:41.137ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20952024-02-01T17:28:41.137Z 0 000-019 3 3 3 5 5 5 F F F
20962024-02-01T17:28:41.138Z 1 020-039 14 14 14 21 21 21 F F F
20972024-02-01T17:28:41.138Z 2 040-059 16 16 16 24 24 24 T F T <---
20982024-02-01T17:28:41.138Z 3 060-079 16 16 16 24 24 24 T F T <---
20992024-02-01T17:28:41.138Z 4 080-099 16 16 16 24 24 24 F F F
21002024-02-01T17:28:41.142Z 5 100-119 16 16 16 24 24 24 F F F
21012024-02-01T17:28:41.142Z 6 120-139 10 10 10 16 16 16 F F F
21022024-02-01T17:28:41.143Z 7 140-159 18 10 18 28 16 28 F F F <---
21032024-02-01T17:28:41.143Z 8 160-179 18 16 18 29 24 29 F F F <---
21042024-02-01T17:28:41.143Z 9 180-199 16 16 16 24 24 24 T F T <---
21052024-02-01T17:28:41.143Z 10 200-219 15 15 15 22 22 22 F F F
21062024-02-01T17:28:41.143Z 11 220-239 9 9 9 13 13 13 F F F
21072024-02-01T17:28:41.143Z 12 240-259 18 5 18 28 8 28 F T F <---
21082024-02-01T17:28:41.143Z 13 260-279 18 13 18 28 19 28 F T F <---
21092024-02-01T17:28:41.143Z 14 280-299 16 16 16 26 26 26 T F T <---
21102024-02-01T17:28:41.143Z 15 300-319 18 16 18 28 26 28 T T T <---
21112024-02-01T17:28:41.143Z 16 320-339 16 16 16 26 26 26 T F T <---
21122024-02-01T17:28:41.143Z 17 340-359 17 17 17 27 27 27 F F F
21132024-02-01T17:28:41.143Z 18 360-379 16 16 16 24 24 24 F F F
21142024-02-01T17:28:41.143Z 19 380-399 14 14 14 21 21 21 T F T <---
21152024-02-01T17:28:41.143Z 20 400-419 14 14 14 21 21 21 F F F
21162024-02-01T17:28:41.143Z 21 420-439 14 14 14 20 20 20 F F F
21172024-02-01T17:28:41.143Z 22 440-459 16 16 16 25 25 25 F F F
21182024-02-01T17:28:41.143Z 23 460-479 16 16 16 25 25 25 F F F
21192024-02-01T17:28:41.143Z 24 480-499 15 15 15 22 22 22 F F F
21202024-02-01T17:28:41.143Z 25 500-519 10 10 10 14 14 14 F F F
21212024-02-01T17:28:41.143Z 26 520-539 18 12 18 29 18 29 F F F <---
21222024-02-01T17:28:41.144Z 27 540-559 10 10 10 14 14 14 T F T <---
21232024-02-01T17:28:41.144Z 28 560-579 13 13 13 19 19 19 T F T <---
21242024-02-01T17:28:41.144Z 29 580-599 16 16 16 25 25 25 F F F
21252024-02-01T17:28:41.144ZMax gen: 18, Max flush: 29
21262024-02-01T17:28:41.144ZError: Difference in extent metadata found!
21272024-02-01T17:28:41.149ZA Difference in extent metadata is expected here
21282024-02-01T17:28:41.149ZOn loop 9
21292024-02-01T17:28:41.149Z
21302024-02-01T17:28:41.149ZVerifying data now
21312024-02-01T17:28:47.337ZNo change after verify
21322024-02-01T17:28:47.337ZLoop: 9 Downstairs dump after verify (and repair):
21332024-02-01T17:28:47.359ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21342024-02-01T17:28:47.359Z 0 000-019 3 3 3 5 5 5 F F F
21352024-02-01T17:28:47.359Z 1 020-039 14 14 14 21 21 21 F F F
21362024-02-01T17:28:47.359Z 2 040-059 19 19 19 30 30 30 F F F
21372024-02-01T17:28:47.359Z 3 060-079 19 19 19 30 30 30 F F F
21382024-02-01T17:28:47.360Z 4 080-099 16 16 16 24 24 24 F F F
21392024-02-01T17:28:47.360Z 5 100-119 16 16 16 24 24 24 F F F
21402024-02-01T17:28:47.364Z 6 120-139 10 10 10 16 16 16 F F F
21412024-02-01T17:28:47.364Z 7 140-159 18 18 18 28 28 28 F F F
21422024-02-01T17:28:47.364Z 8 160-179 18 18 18 29 29 29 F F F
21432024-02-01T17:28:47.364Z 9 180-199 19 19 19 30 30 30 F F F
21442024-02-01T17:28:47.364Z 10 200-219 15 15 15 22 22 22 F F F
21452024-02-01T17:28:47.364Z 11 220-239 9 9 9 13 13 13 F F F
21462024-02-01T17:28:47.364Z 12 240-259 18 18 18 28 28 28 F F F
21472024-02-01T17:28:47.364Z 13 260-279 18 18 18 28 28 28 F F F
21482024-02-01T17:28:47.364Z 14 280-299 19 19 19 30 30 30 F F F
21492024-02-01T17:28:47.364Z 15 300-319 19 19 19 30 30 30 F F F
21502024-02-01T17:28:47.364Z 16 320-339 19 19 19 30 30 30 F F F
21512024-02-01T17:28:47.364Z 17 340-359 17 17 17 27 27 27 F F F
21522024-02-01T17:28:47.365Z 18 360-379 16 16 16 24 24 24 F F F
21532024-02-01T17:28:47.365Z 19 380-399 19 19 19 30 30 30 F F F
21542024-02-01T17:28:47.365Z 20 400-419 14 14 14 21 21 21 F F F
21552024-02-01T17:28:47.365Z 21 420-439 14 14 14 20 20 20 F F F
21562024-02-01T17:28:47.365Z 22 440-459 16 16 16 25 25 25 F F F
21572024-02-01T17:28:47.365Z 23 460-479 16 16 16 25 25 25 F F F
21582024-02-01T17:28:47.365Z 24 480-499 15 15 15 22 22 22 F F F
21592024-02-01T17:28:47.365Z 25 500-519 10 10 10 14 14 14 F F F
21602024-02-01T17:28:47.365Z 26 520-539 18 18 18 29 29 29 F F F
21612024-02-01T17:28:47.365Z 27 540-559 19 19 19 30 30 30 F F F
21622024-02-01T17:28:47.365Z 28 560-579 19 19 19 30 30 30 F F F
21632024-02-01T17:28:47.365Z 29 580-599 16 16 16 25 25 25 F F F
21642024-02-01T17:28:47.365ZMax gen: 19, Max flush: 30
21652024-02-01T17:28:47.365Z
21662024-02-01T17:28:47.365ZBegin loop 10
21672024-02-01T17:28:47.365ZDownstairs to restart: 0
21682024-02-01T17:28:47.394Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.655670651Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
21692024-02-01T17:28:47.401Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656172776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
21702024-02-01T17:28:47.401Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656277699Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
21712024-02-01T17:28:47.401Z{"msg":"Crucible ea45f787-43d0-4612-aa14-656ab0131713 has session id: a0d1dbdc-c0bc-48bc-9480-3401ffd31a32","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656340993Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21722024-02-01T17:28:47.401Z{"msg":"Upstairs opts: Upstairs UUID: ea45f787-43d0-4612-aa14-656ab0131713, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
21732024-02-01T17:28:47.401Z,"time":"2024-02-01T17:28:46.656386078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21742024-02-01T17:28:47.401Z{"msg":"Crucible stats registered with UUID: ea45f787-43d0-4612-aa14-656ab0131713","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656437746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21752024-02-01T17:28:47.401Z{"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656560636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
21762024-02-01T17:28:47.401Z{"msg":"ea45f787-43d0-4612-aa14-656ab0131713 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656718716Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21772024-02-01T17:28:47.401Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656878163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21782024-02-01T17:28:47.401Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656916195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21792024-02-01T17:28:47.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656940165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21802024-02-01T17:28:47.402Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.656964238Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21812024-02-01T17:28:47.402Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657011212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21822024-02-01T17:28:47.403Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657039724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21832024-02-01T17:28:47.403Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657149367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21842024-02-01T17:28:47.403Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657347646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21852024-02-01T17:28:47.403Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.6573983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21862024-02-01T17:28:47.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657483911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21872024-02-01T17:28:47.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657560177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21882024-02-01T17:28:47.403Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.657606632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"io task","client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21892024-02-01T17:28:47.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.658707098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21902024-02-01T17:28:47.403Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.658774509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21912024-02-01T17:28:47.403Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.658801607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21922024-02-01T17:28:47.404Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.658865869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21932024-02-01T17:28:47.404Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.658911745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21942024-02-01T17:28:47.404Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.65904259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21952024-02-01T17:28:47.404Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.659833874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21962024-02-01T17:28:47.404Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.659914279Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21972024-02-01T17:28:47.404Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.660255686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21982024-02-01T17:28:47.404Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.660762403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
21992024-02-01T17:28:47.404Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.660825499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22002024-02-01T17:28:47.404Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.660852338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22012024-02-01T17:28:47.404Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661050506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22022024-02-01T17:28:47.404Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661111423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22032024-02-01T17:28:47.409Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661189126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22042024-02-01T17:28:47.409Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661244285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22052024-02-01T17:28:47.409Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661273566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22062024-02-01T17:28:47.409Z{"msg":"[0]R flush_numbers[0..12]: [5, 21, 30, 30, 24, 24, 16, 28, 29, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661332602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22072024-02-01T17:28:47.409Z{"msg":"[0]R generation[0..12]: [3, 14, 19, 19, 16, 16, 10, 18, 18, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661394164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22082024-02-01T17:28:47.409Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661459358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22092024-02-01T17:28:47.409Z{"msg":Wait for a query_work_queue command to finish before sending IO
22102024-02-01T17:28:47.409Z"[1]R flush_numbers[0..12]: [5, 21, 30, 30, 24, 24, 16, 28, 29, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661527423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22112024-02-01T17:28:47.409Z{"msg":"[1]R generation[0..12]: [3, 14, 19, 19, 16, 16, 10, 18, 18, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661591095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22122024-02-01T17:28:47.409Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661624481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22132024-02-01T17:28:47.409Z{"msg":"[2]R flush_numbers[0..12]: [5, 21, 30, 30, 24, 24, 16, 28, 29, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.66168401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22142024-02-01T17:28:47.409Z{"msg":"[2]R generation[0..12]: [3, 14, 19, 19, 16, 16, 10, 18, 18, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661717872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22152024-02-01T17:28:47.409Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.66177478Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22162024-02-01T17:28:47.409Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661804715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22172024-02-01T17:28:47.409Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661866848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22182024-02-01T17:28:47.409Z{"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661892634Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22192024-02-01T17:28:47.409Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661951273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"":"downstairs","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22202024-02-01T17:28:47.410Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.661978495Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22212024-02-01T17:28:47.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.6620124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"0","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22222024-02-01T17:28:47.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.662040769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"1","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22232024-02-01T17:28:47.410Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.662099556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"client":"2","":"downstairs","session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22242024-02-01T17:28:47.410Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.66212706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22252024-02-01T17:28:47.410Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.66222803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22262024-02-01T17:28:47.410Z{"msg":"ea45f787-43d0-4612-aa14-656ab0131713 is now active with session: a0d1dbdc-c0bc-48bc-9480-3401ffd31a32","v":0,"name":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
22272024-02-01T17:28:47.410Zcrucible","level":30,"time":"2024-02-01T17:28:46.662300482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22282024-02-01T17:28:47.410Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.66235735Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209,"session_id":"a0d1dbdc-c0bc-48bc-9480-3401ffd31a32"}
22292024-02-01T17:28:47.410Z{"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:46.662388263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
22302024-02-01T17:28:47.419ZLoading write count information from file "/tmp/test_repair_verify.data"
22312024-02-01T17:28:47.419ZRun Repair workload
22322024-02-01T17:28:47.420Z01/30 Write block 484 len 40960 data: 3 2 2 3 3 3 3 3 4 4
22332024-02-01T17:28:47.420Z02/30 Write block 36 len 16384 data: 3 4 4 4
22342024-02-01T17:28:47.420Z03/30 Read block 337 len 8192
22352024-02-01T17:28:47.420Z04/30 Read block 368 len 24576
22362024-02-01T17:28:47.424Z05/30 Read block 316 len 40960
22372024-02-01T17:28:47.424Z06/30 Read block 281 len 12288
22382024-02-01T17:28:47.424Z07/30 Read block 291 len 20480
22392024-02-01T17:28:47.424Z08/30 Read block 482 len 4096
22402024-02-01T17:28:47.424Z09/30 Read block 335 len 4096
22412024-02-01T17:28:47.424Z10/30 Flush
22422024-02-01T17:28:47.424Z11/30 Write block 161 len 8192 data: 3 3
22432024-02-01T17:28:47.424Z12/30 Read block 151 len 36864
22442024-02-01T17:28:47.424Z13/30 Read block 422 len 8192
22452024-02-01T17:28:47.425Z14/30 Read block 437 len 16384
22462024-02-01T17:28:47.425Z15/30 Write block 313 len 4096 data: 4
22472024-02-01T17:28:47.425Z16/30 Read block 123 len 40960
22482024-02-01T17:28:47.425Z17/30 Read block 429 len 8192
22492024-02-01T17:28:47.428Z18/30 Flush
22502024-02-01T17:28:47.428Z19/30 Flush
22512024-02-01T17:28:47.428Z20/30 Write block 170 len 20480 data: 3 4 4 4 6
22522024-02-01T17:28:47.431Z21/30 Read block 491 len 12288
22532024-02-01T17:28:47.432Z22/30 Write block 242 len 32768 data: 3 3 2 3 3 3 3 3
22542024-02-01T17:28:47.432Z23/30 Read block 571 len 12288
22552024-02-01T17:28:47.435Z24/30 Write block 447 len 12288 data: 3 3 4
22562024-02-01T17:28:47.436Z25/30 Read block 96 len 8192
22572024-02-01T17:28:47.440Z26/30 Write block 246 len 24576 data: 4 4 4 4 3 3
22582024-02-01T17:28:47.440Z27/30 Read block 492 len 16384
22592024-02-01T17:28:47.440Z28/30 Write block 133 len 36864 data: 2 2 2 2 3 3 3 3 3
22602024-02-01T17:28:47.444Z29/30 Read block 348 len 12288
22612024-02-01T17:28:47.444Z30/30 Write block 296 len 40960 data: 4 3 4 4 4 4 4 4 4 4
22622024-02-01T17:28:47.444Z----------------------------------------------------------------
22632024-02-01T17:28:47.444Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
22642024-02-01T17:28:47.445ZStates: Active Active Active
22652024-02-01T17:28:47.445ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22662024-02-01T17:28:47.445Z 1 Acked 1000 Write 10 Done Done Done false
22672024-02-01T17:28:47.445Z 2 Acked 1001 Write 4 Done Done Done false
22682024-02-01T17:28:47.445Z 3 Acked 1002 Read 2 Done Done Done false
22692024-02-01T17:28:47.445Z 4 Acked 1003 Read 6 Sent Done Done false
22702024-02-01T17:28:47.445Z 5 Acked 1004 Read 10 Sent Done Done false
22712024-02-01T17:28:47.445Z 6 Acked 1005 Read 3 Sent Done Done false
22722024-02-01T17:28:47.445Z 7 Acked 1006 Read 5 Sent Done Done false
22732024-02-01T17:28:47.445Z 8 Acked 1007 Read 1 Sent Done Done false
22742024-02-01T17:28:47.445Z 9 Acked 1008 Read 1 Sent Done Done false
22752024-02-01T17:28:47.445Z 10 Acked 1009 Flush 0 Sent Done Done false
22762024-02-01T17:28:47.445Z 11 Acked 1010 Write 2 Sent Done Done false
22772024-02-01T17:28:47.446Z 12 Acked 1011 Read 9 Sent Done Done false
22782024-02-01T17:28:47.446Z 13 Acked 1012 Read 2 Sent Done Done false
22792024-02-01T17:28:47.446Z 14 Acked 1013 Read 4 Sent Done Done false
22802024-02-01T17:28:47.446Z 15 Acked 1014 Write 1 Sent Done Done false
22812024-02-01T17:28:47.446Z 16 Acked 1015 Read 10 Sent Done Done false
22822024-02-01T17:28:47.446Z 17 Acked 1016 Read 2 Sent Done Done false
22832024-02-01T17:28:47.446Z 18 Acked 1017 Flush 0 Sent Done Done false
22842024-02-01T17:28:47.446Z 19 Acked 1018 Flush 0 Sent Done Done false
22852024-02-01T17:28:47.446Z 20 Acked 1019 Write 5 Sent Done Done false
22862024-02-01T17:28:47.446Z 21 Acked 1020 Read 3 Sent Done Done false
22872024-02-01T17:28:47.446Z 22 Acked 1021 Write 8 Sent Done Done false
22882024-02-01T17:28:47.447Z 23 Acked 1022 Read 3 Sent Done Done false
22892024-02-01T17:28:47.447Z 24 Acked 1023 Write 3 Sent Done Done false
22902024-02-01T17:28:47.447Z 25 Acked 1024 Read 2 Sent Done Done false
22912024-02-01T17:28:47.447Z 26 Acked 1025 Write 6 Sent Done Done false
22922024-02-01T17:28:47.447Z 27 Acked 1026 Read 4 Sent Done Done false
22932024-02-01T17:28:47.447Z 28 Acked 1027 Write 9 Sent Done Done false
22942024-02-01T17:28:47.447Z 29 Acked 1028 Read 3 Sent Done Done false
22952024-02-01T17:28:47.447Z 30 Acked 1029 Write 10 Sent Sent Sent false
22962024-02-01T17:28:47.447Z STATES DS:0 DS:1 DS:2 TOTAL
22972024-02-01T17:28:47.447Z New 0 0 0 0
22982024-02-01T17:28:47.447Z Sent 27 1 1 29
22992024-02-01T17:28:47.447Z Done 3 29 29 61
23002024-02-01T17:28:47.447Z Skipped 0 0 0 0
23012024-02-01T17:28:47.448Z Error 0 0 0 0
23022024-02-01T17:28:47.448ZLast Flush: 0 1018 1018
23032024-02-01T17:28:47.448ZDownstairs last five completed:
23042024-02-01T17:28:47.448ZUpstairs last five completed: 30 29 28 27 26
23052024-02-01T17:28:47.449Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:46.705522874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1209}
23062024-02-01T17:28:47.449ZWrote out file "/tmp/test_repair_verify.data"
23072024-02-01T17:28:47.452Z
23082024-02-01T17:28:47.456ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23092024-02-01T17:28:47.481ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23102024-02-01T17:28:47.481Z 0 000-019 3 3 3 5 5 5 F F F
23112024-02-01T17:28:47.481Z 1 020-039 14 20 20 21 31 31 T F F <---
23122024-02-01T17:28:47.485Z 2 040-059 19 19 19 30 30 30 F F F
23132024-02-01T17:28:47.485Z 3 060-079 19 19 19 30 30 30 F F F
23142024-02-01T17:28:47.485Z 4 080-099 16 16 16 24 24 24 F F F
23152024-02-01T17:28:47.485Z 5 100-119 16 16 16 24 24 24 F F F
23162024-02-01T17:28:47.485Z 6 120-139 10 10 10 16 16 16 F T T <---
23172024-02-01T17:28:47.485Z 7 140-159 18 18 18 28 28 28 F T T <---
23182024-02-01T17:28:47.485Z 8 160-179 18 20 20 29 32 32 F T T <---
23192024-02-01T17:28:47.486Z 9 180-199 19 19 19 30 30 30 F F F
23202024-02-01T17:28:47.486Z 10 200-219 15 15 15 22 22 22 F F F
23212024-02-01T17:28:47.486Z 11 220-239 9 9 9 13 13 13 F F F
23222024-02-01T17:28:47.486Z 12 240-259 18 18 18 28 28 28 F T T <---
23232024-02-01T17:28:47.486Z 13 260-279 18 18 18 28 28 28 F F F
23242024-02-01T17:28:47.486Z 14 280-299 19 19 19 30 30 30 F T T <---
23252024-02-01T17:28:47.486Z 15 300-319 19 20 20 30 32 32 F T T <---
23262024-02-01T17:28:47.486Z 16 320-339 19 19 19 30 30 30 F F F
23272024-02-01T17:28:47.486Z 17 340-359 17 17 17 27 27 27 F F F
23282024-02-01T17:28:47.486Z 18 360-379 16 16 16 24 24 24 F F F
23292024-02-01T17:28:47.486Z 19 380-399 19 19 19 30 30 30 F F F
23302024-02-01T17:28:47.487Z 20 400-419 14 14 14 21 21 21 F F F
23312024-02-01T17:28:47.487Z 21 420-439 14 14 14 20 20 20 F F F
23322024-02-01T17:28:47.488Z 22 440-459 16 16 16 25 25 25 F T T <---
23332024-02-01T17:28:47.488Z 23 460-479 16 16 16 25 25 25 F F F
23342024-02-01T17:28:47.488Z 24 480-499 15 20 20 22 31 31 T F F <---
23352024-02-01T17:28:47.488Z 25 500-519 10 10 10 14 14 14 F F F
23362024-02-01T17:28:47.488Z 26 520-539 18 18 18 29 29 29 F F F
23372024-02-01T17:28:47.488Z 27 540-559 19 19 19 30 30 30 F F F
23382024-02-01T17:28:47.488Z 28 560-579 19 19 19 30 30 30 F F F
23392024-02-01T17:28:47.488Z 29 580-599 16 16 16 25 25 25 F F F
23402024-02-01T17:28:47.488ZMax gen: 20, Max flush: 32
23412024-02-01T17:28:47.488ZError: Difference in extent metadata found!
23422024-02-01T17:28:47.504ZA Difference in extent metadata is expected here
23432024-02-01T17:28:47.504ZOn loop 10
23442024-02-01T17:28:47.504Z
23452024-02-01T17:28:47.504ZVerifying data now
23462024-02-01T17:28:52.327ZNo change after verify
23472024-02-01T17:28:52.327ZLoop: 10 Downstairs dump after verify (and repair):
23482024-02-01T17:28:52.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23492024-02-01T17:28:52.350Z 0 000-019 3 3 3 5 5 5 F F F
23502024-02-01T17:28:52.351Z 1 020-039 20 20 20 31 31 31 F F F
23512024-02-01T17:28:52.351Z 2 040-059 19 19 19 30 30 30 F F F
23522024-02-01T17:28:52.351Z 3 060-079 19 19 19 30 30 30 F F F
23532024-02-01T17:28:52.361Z 4 080-099 16 16 16 24 24 24 F F F
23542024-02-01T17:28:52.361Z 5 100-119 16 16 16 24 24 24 F F F
23552024-02-01T17:28:52.361Z 6 120-139 21 21 21 33 33 33 F F F
23562024-02-01T17:28:52.361Z 7 140-159 21 21 21 33 33 33 F F F
23572024-02-01T17:28:52.361Z 8 160-179 21 21 21 33 33 33 F F F
23582024-02-01T17:28:52.361Z 9 180-199 19 19 19 30 30 30 F F F
23592024-02-01T17:28:52.361Z 10 200-219 15 15 15 22 22 22 F F F
23602024-02-01T17:28:52.361Z 11 220-239 9 9 9 13 13 13 F F F
23612024-02-01T17:28:52.361Z 12 240-259 21 21 21 33 33 33 F F F
23622024-02-01T17:28:52.361Z 13 260-279 18 18 18 28 28 28 F F F
23632024-02-01T17:28:52.361Z 14 280-299 21 21 21 33 33 33 F F F
23642024-02-01T17:28:52.361Z 15 300-319 21 21 21 33 33 33 F F F
23652024-02-01T17:28:52.361Z 16 320-339 19 19 19 30 30 30 F F F
23662024-02-01T17:28:52.361Z 17 340-359 17 17 17 27 27 27 F F F
23672024-02-01T17:28:52.361Z 18 360-379 16 16 16 24 24 24 F F F
23682024-02-01T17:28:52.362Z 19 380-399 19 19 19 30 30 30 F F F
23692024-02-01T17:28:52.362Z 20 400-419 14 14 14 21 21 21 F F F
23702024-02-01T17:28:52.362Z 21 420-439 14 14 14 20 20 20 F F F
23712024-02-01T17:28:52.362Z 22 440-459 21 21 21 33 33 33 F F F
23722024-02-01T17:28:52.362Z 23 460-479 16 16 16 25 25 25 F F F
23732024-02-01T17:28:52.362Z 24 480-499 20 20 20 31 31 31 F F F
23742024-02-01T17:28:52.362Z 25 500-519 10 10 10 14 14 14 F F F
23752024-02-01T17:28:52.362Z 26 520-539 18 18 18 29 29 29 F F F
23762024-02-01T17:28:52.362Z 27 540-559 19 19 19 30 30 30 F F F
23772024-02-01T17:28:52.362Z 28 560-579 19 19 19 30 30 30 F F F
23782024-02-01T17:28:52.362Z 29 580-599 16 16 16 25 25 25 F F F
23792024-02-01T17:28:52.363ZMax gen: 21, Max flush: 33
23802024-02-01T17:28:52.363Z
23812024-02-01T17:28:52.363ZBegin loop 11
23822024-02-01T17:28:52.363ZDownstairs to restart: 0
23832024-02-01T17:28:52.381Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.642995856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
23842024-02-01T17:28:52.381Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643468489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
23852024-02-01T17:28:52.381Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643503357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
23862024-02-01T17:28:52.385Z{"msg":"Crucible b75e065f-450b-404b-9ce4-bda666941fe2 has session id: c2253f3a-a1fe-446f-9db5-72c2ffa263ea","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643572271Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23872024-02-01T17:28:52.385Z{"msg":"Upstairs opts: Upstairs UUID: b75e065f-450b-404b-9ce4-bda666941fe2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.6436559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23882024-02-01T17:28:52.385Z{"msg":"Crucible stats registered with UUID: b75e065f-450b-404b-9ce4-bda666941fe2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643702138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"Crucible runtime is spawned
23892024-02-01T17:28:52.385Zc2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23902024-02-01T17:28:52.385Z{"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643931707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
23912024-02-01T17:28:52.385Z{"msg":"b75e065f-450b-404b-9ce4-bda666941fe2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.643993216Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23922024-02-01T17:28:52.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64408355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23932024-02-01T17:28:52.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644123749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23942024-02-01T17:28:52.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644148105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23952024-02-01T17:28:52.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64420655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23962024-02-01T17:28:52.386Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644232348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23972024-02-01T17:28:52.386Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644293426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23982024-02-01T17:28:52.386Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64432191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
23992024-02-01T17:28:52.386Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644360571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24002024-02-01T17:28:52.386Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64450734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24012024-02-01T17:28:52.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644582976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24022024-02-01T17:28:52.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644631358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24032024-02-01T17:28:52.387Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.644679358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"io task","client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24042024-02-01T17:28:52.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.646053107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24052024-02-01T17:28:52.387Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.646197755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24062024-02-01T17:28:52.387Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.646244546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24072024-02-01T17:28:52.387Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64628837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24082024-02-01T17:28:52.394Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.647001596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24092024-02-01T17:28:52.394Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64707178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24102024-02-01T17:28:52.394Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.647170171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24112024-02-01T17:28:52.394Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.647219361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24122024-02-01T17:28:52.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648020572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24132024-02-01T17:28:52.395Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648091094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24142024-02-01T17:28:52.395Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648117608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24152024-02-01T17:28:52.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648146738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24162024-02-01T17:28:52.395Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648192623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24172024-02-01T17:28:52.395Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648217889Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24182024-02-01T17:28:52.395Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.648241529Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24192024-02-01T17:28:52.395Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.64899324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24202024-02-01T17:28:52.395Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649072491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24212024-02-01T17:28:52.395Z{"msg":"[0]R flush_numbers[0..12]: [5, 31, 30, 30, 24, 24, 33, 33, 33, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649102199Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24222024-02-01T17:28:52.395Z{"msg":"[0]R generation[0..12]: [3, 20, 19, 19, 16, 16, 21, 21, 21, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649144883Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24232024-02-01T17:28:52.395Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649171379Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24242024-02-01T17:28:52.395Z{"msg":"[1]R flush_numbers[0..12]: [5, 31, 30, 30, 24, 24, 33, 33, 33, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649212918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24252024-02-01T17:28:52.395Z{"msg":"[1]R generation[0..12]: [3, 20, 19, 19, 16, 16, 21, 21, 21, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649238017Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24262024-02-01T17:28:52.396Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649279755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24272024-02-01T17:28:52.396Z{"msg":"[2]R flush_numbers[0..12]: [5, 31, 30, 30, 24, 24, 33, 33, 33, 30, 22, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649304699Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24282024-02-01T17:28:52.396ZWait for a query_work_queue command to finish before sending IO
24292024-02-01T17:28:52.396Z{"msg":"[2]R generation[0..12]: [3, 20, 19, 19, 16, 16, 21, 21, 21, 19, 15, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649353146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24302024-02-01T17:28:52.396Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649378904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24312024-02-01T17:28:52.396Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649401222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24322024-02-01T17:28:52.396Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649445367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24332024-02-01T17:28:52.396Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649468826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24342024-02-01T17:28:52.397Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649490344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"":"downstairs","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24352024-02-01T17:28:52.397Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649555662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24362024-02-01T17:28:52.397Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649582383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"0","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24372024-02-01T17:28:52.397Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649616149Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"1","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24382024-02-01T17:28:52.397Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649680328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"client":"2","":"downstairs","session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24392024-02-01T17:28:52.397Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649713303Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24402024-02-01T17:28:52.397Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649775338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24412024-02-01T17:28:52.397Z{"msg":"b75e065f-450b-404b-9ce4-bda666941fe2 is now active with session: c2253f3a-a1fe-446f-9db5-72c2ffa263ea","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649807329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24422024-02-01T17:28:52.397Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649844335Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217,"session_id":"c2253f3a-a1fe-446f-9db5-72c2ffa263ea"}
24432024-02-01T17:28:52.397Z{"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:51.649889612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
24442024-02-01T17:28:52.397ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24452024-02-01T17:28:52.397ZLoading write count information from file "/tmp/test_repair_verify.data"
24462024-02-01T17:28:52.397ZRun Repair workload
24472024-02-01T17:28:52.397Z01/30 Write block 333 len 24576 data: 3 4 4 4 2 2
24482024-02-01T17:28:52.397Z02/30 Read block 571 len 16384
24492024-02-01T17:28:52.401Z03/30 Write block 351 len 20480 data: 3 3 3 6 4
24502024-02-01T17:28:52.401Z04/30 Write block 547 len 36864 data: 3 3 3 4 4 4 4 3 3
24512024-02-01T17:28:52.401Z05/30 Write block 540 len 12288 data: 4 6 6
24522024-02-01T17:28:52.401Z06/30 Read block 65 len 8192
24532024-02-01T17:28:52.401Z07/30 Write block 283 len 16384 data: 2 2 3 4
24542024-02-01T17:28:52.401Z08/30 Write block 48 len 36864 data: 2 2 3 3 3 4 5 6 5
24552024-02-01T17:28:52.401Z09/30 Read block 421 len 16384
24562024-02-01T17:28:52.404Z10/30 Read block 198 len 8192
24572024-02-01T17:28:52.404Z11/30 Write block 47 len 16384 data: 2 3 3 4
24582024-02-01T17:28:52.404Z12/30 Read block 184 len 8192
24592024-02-01T17:28:52.407Z13/30 Read block 446 len 32768
24602024-02-01T17:28:52.407Z14/30 Write block 67 len 12288 data: 2 2 2
24612024-02-01T17:28:52.410Z15/30 Flush
24622024-02-01T17:28:52.413Z16/30 Flush
24632024-02-01T17:28:52.413Z17/30 Write block 99 len 32768 data: 3 3 3 3 3 2 4 4
24642024-02-01T17:28:52.413Z18/30 Read block 363 len 20480
24652024-02-01T17:28:52.416Z19/30 Read block 338 len 8192
24662024-02-01T17:28:52.416Z20/30 Write block 334 len 24576 data: 5 5 5 3 3 2
24672024-02-01T17:28:52.419Z21/30 Write block 119 len 32768 data: 3 3 3 4 3 3 3 3
24682024-02-01T17:28:52.419Z22/30 Read block 583 len 36864
24692024-02-01T17:28:52.422Z23/30 Write block 197 len 16384 data: 2 2 2 3
24702024-02-01T17:28:52.422Z24/30 Read block 166 len 40960
24712024-02-01T17:28:52.425Z25/30 Read block 104 len 36864
24722024-02-01T17:28:52.425Z26/30 Read block 192 len 4096
24732024-02-01T17:28:52.428Z27/30 Read block 343 len 16384
24742024-02-01T17:28:52.428Z28/30 Read block 220 len 28672
24752024-02-01T17:28:52.431Z29/30 Write block 285 len 40960 data: 4 5 4 4 4 3 4 3 3 4
24762024-02-01T17:28:52.431Z30/30 Write block 434 len 36864 data: 6 6 5 6 4 5 4 4 4
24772024-02-01T17:28:52.434Z----------------------------------------------------------------
24782024-02-01T17:28:52.434Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
24792024-02-01T17:28:52.434ZStates: Active Active Active
24802024-02-01T17:28:52.434ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24812024-02-01T17:28:52.435Z 1 Acked 1000 Write 6 Sent Done Done false
24822024-02-01T17:28:52.435Z 2 Acked 1001 Read 4 Sent Done Done false
24832024-02-01T17:28:52.435Z 3 Acked 1002 Write 5 Sent Done Done false
24842024-02-01T17:28:52.435Z 4 Acked 1003 Write 9 Sent Done Done false
24852024-02-01T17:28:52.435Z 5 Acked 1004 Write 3 Sent Done Done false
24862024-02-01T17:28:52.435Z 6 Acked 1005 Read 2 Sent Done Done false
24872024-02-01T17:28:52.435Z 7 Acked 1006 Write 4 Sent Done Done false
24882024-02-01T17:28:52.435Z 8 Acked 1007 Write 9 Sent Done Done false
24892024-02-01T17:28:52.435Z 9 Acked 1008 Read 4 Sent Done Done false
24902024-02-01T17:28:52.435Z 10 Acked 1009 Read 2 Sent Done Done false
24912024-02-01T17:28:52.435Z 11 Acked 1010 Write 4 Sent Done Done false
24922024-02-01T17:28:52.435Z 12 Acked 1011 Read 2 Sent Done Done false
24932024-02-01T17:28:52.435Z 13 Acked 1012 Read 8 Sent Done Done false
24942024-02-01T17:28:52.435Z 14 Acked 1013 Write 3 Sent Done Done false
24952024-02-01T17:28:52.435Z 15 Acked 1014 Flush 0 Sent Done Done false
24962024-02-01T17:28:52.435Z 16 Acked 1015 Flush 0 Sent Done Done false
24972024-02-01T17:28:52.435Z 17 Acked 1016 Write 8 Sent Done Done false
24982024-02-01T17:28:52.435Z 18 Acked 1017 Read 5 Sent Done Done false
24992024-02-01T17:28:52.435Z 19 Acked 1018 Read 2 Sent Done Done false
25002024-02-01T17:28:52.435Z 20 Acked 1019 Write 6 Sent Done Done false
25012024-02-01T17:28:52.435Z 21 Acked 1020 Write 8 Sent Done Done false
25022024-02-01T17:28:52.435Z 22 Acked 1021 Read 9 Sent Done Done false
25032024-02-01T17:28:52.435Z 23 Acked 1022 Write 4 Sent Done Done false
25042024-02-01T17:28:52.436Z 24 Acked 1023 Read 10 Sent Done Done false
25052024-02-01T17:28:52.436Z 25 Acked 1024 Read 9 Sent Done Done false
25062024-02-01T17:28:52.436Z 26 Acked 1025 Read 1 Sent Done Done false
25072024-02-01T17:28:52.436Z 27 Acked 1026 Read 4 Sent Done Done false
25082024-02-01T17:28:52.436Z 28 Acked 1027 Read 7 Sent Done Done false
25092024-02-01T17:28:52.436Z 29 Acked 1028 Write 10 Sent Done Done false
25102024-02-01T17:28:52.436Z 30 Acked 1029 Write 9 Sent Sent Sent false
25112024-02-01T17:28:52.437Z STATES DS:0 DS:1 DS:2 TOTAL
25122024-02-01T17:28:52.437Z New 0 0 0 0
25132024-02-01T17:28:52.437Z Sent 30 1 1 32
25142024-02-01T17:28:52.437Z Done 0 29 29 58
25152024-02-01T17:28:52.437Z Skipped 0 0 0 0
25162024-02-01T17:28:52.437Z Error 0 0 0 0
25172024-02-01T17:28:52.437ZLast Flush: 0 1015 1015
25182024-02-01T17:28:52.437ZDownstairs last five completed:
25192024-02-01T17:28:52.437ZUpstairs last five completed: 30 29 28 27 26
25202024-02-01T17:28:52.437Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:51.694697787Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1217}
25212024-02-01T17:28:52.437ZWrote out file "/tmp/test_repair_verify.data"
25222024-02-01T17:28:52.447Z
25232024-02-01T17:28:52.447ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25242024-02-01T17:28:52.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25252024-02-01T17:28:52.482Z 0 000-019 3 3 3 5 5 5 F F F
25262024-02-01T17:28:52.482Z 1 020-039 20 20 20 31 31 31 F F F
25272024-02-01T17:28:52.482Z 2 040-059 19 22 22 30 34 34 F F F <---
25282024-02-01T17:28:52.486Z 3 060-079 19 22 22 30 34 34 F F F <---
25292024-02-01T17:28:52.486Z 4 080-099 16 16 16 24 24 24 F T T <---
25302024-02-01T17:28:52.487Z 5 100-119 16 16 16 24 24 24 F T T <---
25312024-02-01T17:28:52.487Z 6 120-139 21 21 21 33 33 33 F T T <---
25322024-02-01T17:28:52.487Z 7 140-159 21 21 21 33 33 33 F F F
25332024-02-01T17:28:52.487Z 8 160-179 21 21 21 33 33 33 F F F
25342024-02-01T17:28:52.487Z 9 180-199 19 19 19 30 30 30 F T T <---
25352024-02-01T17:28:52.487Z 10 200-219 15 15 15 22 22 22 F T T <---
25362024-02-01T17:28:52.487Z 11 220-239 9 9 9 13 13 13 F F F
25372024-02-01T17:28:52.487Z 12 240-259 21 21 21 33 33 33 F F F
25382024-02-01T17:28:52.487Z 13 260-279 18 18 18 28 28 28 F F F
25392024-02-01T17:28:52.487Z 14 280-299 21 22 22 33 34 34 F T T <---
25402024-02-01T17:28:52.487Z 15 300-319 21 21 21 33 33 33 F F F
25412024-02-01T17:28:52.487Z 16 320-339 19 22 22 30 34 34 F T T <---
25422024-02-01T17:28:52.487Z 17 340-359 17 22 22 27 34 34 F F F <---
25432024-02-01T17:28:52.487Z 18 360-379 16 16 16 24 24 24 F F F
25442024-02-01T17:28:52.487Z 19 380-399 19 19 19 30 30 30 F F F
25452024-02-01T17:28:52.487Z 20 400-419 14 14 14 21 21 21 F F F
25462024-02-01T17:28:52.488Z 21 420-439 14 14 14 20 20 20 F T T <---
25472024-02-01T17:28:52.488Z 22 440-459 21 21 21 33 33 33 F T T <---
25482024-02-01T17:28:52.488Z 23 460-479 16 16 16 25 25 25 F F F
25492024-02-01T17:28:52.488Z 24 480-499 20 20 20 31 31 31 F F F
25502024-02-01T17:28:52.488Z 25 500-519 10 10 10 14 14 14 F F F
25512024-02-01T17:28:52.488Z 26 520-539 18 18 18 29 29 29 F F F
25522024-02-01T17:28:52.488Z 27 540-559 19 22 22 30 34 34 F F F <---
25532024-02-01T17:28:52.488Z 28 560-579 19 19 19 30 30 30 F F F
25542024-02-01T17:28:52.488Z 29 580-599 16 16 16 25 25 25 F F F
25552024-02-01T17:28:52.488ZMax gen: 22, Max flush: 34
25562024-02-01T17:28:52.488ZError: Difference in extent metadata found!
25572024-02-01T17:28:52.488ZA Difference in extent metadata is expected here
25582024-02-01T17:28:52.488ZOn loop 11
25592024-02-01T17:28:52.488Z
25602024-02-01T17:28:52.488ZVerifying data now
25612024-02-01T17:28:57.607ZNo change after verify
25622024-02-01T17:28:57.607ZLoop: 11 Downstairs dump after verify (and repair):
25632024-02-01T17:28:57.630ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25642024-02-01T17:28:57.630Z 0 000-019 3 3 3 5 5 5 F F F
25652024-02-01T17:28:57.633Z 1 020-039 20 20 20 31 31 31 F F F
25662024-02-01T17:28:57.633Z 2 040-059 22 22 22 34 34 34 F F F
25672024-02-01T17:28:57.633Z 3 060-079 22 22 22 34 34 34 F F F
25682024-02-01T17:28:57.633Z 4 080-099 23 23 23 35 35 35 F F F
25692024-02-01T17:28:57.633Z 5 100-119 23 23 23 35 35 35 F F F
25702024-02-01T17:28:57.633Z 6 120-139 23 23 23 35 35 35 F F F
25712024-02-01T17:28:57.633Z 7 140-159 21 21 21 33 33 33 F F F
25722024-02-01T17:28:57.633Z 8 160-179 21 21 21 33 33 33 F F F
25732024-02-01T17:28:57.633Z 9 180-199 23 23 23 35 35 35 F F F
25742024-02-01T17:28:57.633Z 10 200-219 23 23 23 35 35 35 F F F
25752024-02-01T17:28:57.633Z 11 220-239 9 9 9 13 13 13 F F F
25762024-02-01T17:28:57.633Z 12 240-259 21 21 21 33 33 33 F F F
25772024-02-01T17:28:57.634Z 13 260-279 18 18 18 28 28 28 F F F
25782024-02-01T17:28:57.634Z 14 280-299 23 23 23 35 35 35 F F F
25792024-02-01T17:28:57.634Z 15 300-319 21 21 21 33 33 33 F F F
25802024-02-01T17:28:57.634Z 16 320-339 23 23 23 35 35 35 F F F
25812024-02-01T17:28:57.634Z 17 340-359 22 22 22 34 34 34 F F F
25822024-02-01T17:28:57.634Z 18 360-379 16 16 16 24 24 24 F F F
25832024-02-01T17:28:57.634Z 19 380-399 19 19 19 30 30 30 F F F
25842024-02-01T17:28:57.634Z 20 400-419 14 14 14 21 21 21 F F F
25852024-02-01T17:28:57.634Z 21 420-439 23 23 23 35 35 35 F F F
25862024-02-01T17:28:57.634Z 22 440-459 23 23 23 35 35 35 F F F
25872024-02-01T17:28:57.634Z 23 460-479 16 16 16 25 25 25 F F F
25882024-02-01T17:28:57.634Z 24 480-499 20 20 20 31 31 31 F F F
25892024-02-01T17:28:57.634Z 25 500-519 10 10 10 14 14 14 F F F
25902024-02-01T17:28:57.634Z 26 520-539 18 18 18 29 29 29 F F F
25912024-02-01T17:28:57.634Z 27 540-559 22 22 22 34 34 34 F F F
25922024-02-01T17:28:57.634Z 28 560-579 19 19 19 30 30 30 F F F
25932024-02-01T17:28:57.634Z 29 580-599 16 16 16 25 25 25 F F F
25942024-02-01T17:28:57.634ZMax gen: 23, Max flush: 35
25952024-02-01T17:28:57.637Z
25962024-02-01T17:28:57.637ZBegin loop 12
25972024-02-01T17:28:57.637ZDownstairs to restart: 2
25982024-02-01T17:28:57.666Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929043185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
25992024-02-01T17:28:57.666Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929502789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
26002024-02-01T17:28:57.666Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929531687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
26012024-02-01T17:28:57.671Z{"msg":"Crucible c98da6b8-e8bb-468c-8e05-1274d9f6a893 has session id: eb3ebf76-f0f3-4baa-81b6-cc3b855035fb","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929575091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26022024-02-01T17:28:57.671Z{"msg":"Upstairs opts: Upstairs UUID: c98da6b8-e8bb-468c-8e05-1274d9f6a893, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929664848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26032024-02-01T17:28:57.671Z{"msg":"Crucible stats registered with UUID: c98da6b8-e8bb-468c-8e05-1274d9f6a893","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.929758472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26042024-02-01T17:28:57.671ZCrucible runtime is spawned
26052024-02-01T17:28:57.671Z{"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930060856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
26062024-02-01T17:28:57.671Z{"msg":"c98da6b8-e8bb-468c-8e05-1274d9f6a893 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930113327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26072024-02-01T17:28:57.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930137849Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26082024-02-01T17:28:57.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930168051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26092024-02-01T17:28:57.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930191576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26102024-02-01T17:28:57.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930214303Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26112024-02-01T17:28:57.671Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930270201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26122024-02-01T17:28:57.671Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930297145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26132024-02-01T17:28:57.672Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930346063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26142024-02-01T17:28:57.672Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930390619Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26152024-02-01T17:28:57.672Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930506261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26162024-02-01T17:28:57.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930635039Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26172024-02-01T17:28:57.672Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930712276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26182024-02-01T17:28:57.673Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.930979309Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"io task","client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26192024-02-01T17:28:57.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.931920975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26202024-02-01T17:28:57.673Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.932024424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26212024-02-01T17:28:57.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.932058897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26222024-02-01T17:28:57.673Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.932110253Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26232024-02-01T17:28:57.673Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.932684003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26242024-02-01T17:28:57.673Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.932723457Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26252024-02-01T17:28:57.673Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933058109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26262024-02-01T17:28:57.680Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933248613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26272024-02-01T17:28:57.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933695224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26282024-02-01T17:28:57.680Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933785758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26292024-02-01T17:28:57.680Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933835343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26302024-02-01T17:28:57.680Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933876569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26312024-02-01T17:28:57.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.933972446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26322024-02-01T17:28:57.680Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934034907Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26332024-02-01T17:28:57.680Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934061484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26342024-02-01T17:28:57.680Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934541769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26352024-02-01T17:28:57.680Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934603224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26362024-02-01T17:28:57.680Z{"msg":"[0]R flush_numbers[0..12]: [5, 31, 34, 34, 35, 35, 35, 33, 33, 35, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934635444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26372024-02-01T17:28:57.680Z{"msg":"[0]R generation[0..12]: [3, 20, 22, 22, 23, 23, 23, 21, 21, 23, 23, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934690277Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26382024-02-01T17:28:57.680Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934718102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26392024-02-01T17:28:57.681Z{"msg":"[1]R flush_numbers[0..12]: [5, 31, 34, 34, 35, 35, 35, 33, 33, 35, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934743184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26402024-02-01T17:28:57.681Z{"msg":"[1]R generation[0..12]: [3, 20, 22, 22, 23, 23, 23, 21, 21, 23, 23, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934766573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26412024-02-01T17:28:57.681Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.93481723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26422024-02-01T17:28:57.681ZWait for a query_work_queue command to finish before sending IO
26432024-02-01T17:28:57.681Z{"msg":"[2]R flush_numbers[0..12]: [5, 31, 34, 34, 35, 35, 35, 33, 33, 35, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934852797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26442024-02-01T17:28:57.681Z{"msg":"[2]R generation[0..12]: [3, 20, 22, 22, 23, 23, 23, 21, 21, 23, 23, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934876698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26452024-02-01T17:28:57.681Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934919614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26462024-02-01T17:28:57.681Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934943825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26472024-02-01T17:28:57.681Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.934966159Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26482024-02-01T17:28:57.681Z{"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.93502034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26492024-02-01T17:28:57.681Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935048499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"":"downstairs","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26502024-02-01T17:28:57.681Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935090233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26512024-02-01T17:28:57.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935112504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"0","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26522024-02-01T17:28:57.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935134377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"1","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26532024-02-01T17:28:57.681Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.9351559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"client":"2","":"downstairs","session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26542024-02-01T17:28:57.681Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935178252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26552024-02-01T17:28:57.681Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935198823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26562024-02-01T17:28:57.681Z{"msg":"c98da6b8-e8bb-468c-8e05-1274d9f6a893 is now active with session: eb3ebf76-f0f3-4baa-81b6-cc3b855035fb","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935238545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26572024-02-01T17:28:57.681Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935260101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225,"session_id":"eb3ebf76-f0f3-4baa-81b6-cc3b855035fb"}
26582024-02-01T17:28:57.681Z{"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:28:56.935280669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
26592024-02-01T17:28:57.681ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26602024-02-01T17:28:57.681ZLoading write count information from file "/tmp/test_repair_verify.data"
26612024-02-01T17:28:57.681ZRun Repair workload
26622024-02-01T17:28:57.681Z01/30 Flush
26632024-02-01T17:28:57.682Z02/30 Write block 8 len 4096 data: 2
26642024-02-01T17:28:57.682Z03/30 Read block 280 len 24576
26652024-02-01T17:28:57.682Z04/30 Read block 158 len 24576
26662024-02-01T17:28:57.685Z05/30 Read block 173 len 28672
26672024-02-01T17:28:57.685Z06/30 Write block 422 len 8192 data: 2 2
26682024-02-01T17:28:57.685Z07/30 Read block 30 len 32768
26692024-02-01T17:28:57.685Z08/30 Write block 255 len 28672 data: 3 3 3 4 4 4 3
26702024-02-01T17:28:57.685Z09/30 Read block 313 len 4096
26712024-02-01T17:28:57.688Z10/30 Read block 204 len 8192
26722024-02-01T17:28:57.688Z11/30 Write block 17 len 12288 data: 3 3 2
26732024-02-01T17:28:57.688Z12/30 Read block 519 len 4096
26742024-02-01T17:28:57.691Z13/30 Read block 189 len 20480
26752024-02-01T17:28:57.691Z14/30 Write block 35 len 20480 data: 3 4 5 5 5
26762024-02-01T17:28:57.691Z15/30 Read block 477 len 8192
26772024-02-01T17:28:57.694Z16/30 Flush
26782024-02-01T17:28:57.697Z17/30 Write block 426 len 12288 data: 3 2 3
26792024-02-01T17:28:57.697Z18/30 Read block 313 len 32768
26802024-02-01T17:28:57.699Z19/30 Read block 125 len 20480
26812024-02-01T17:28:57.699Z20/30 Flush
26822024-02-01T17:28:57.702Z21/30 Write block 544 len 4096 data: 5
26832024-02-01T17:28:57.705Z22/30 Write block 177 len 36864 data: 5 4 4 3 4 3 4 4 5
26842024-02-01T17:28:57.705Z23/30 Write block 466 len 28672 data: 3 3 3 2 3 3 3
26852024-02-01T17:28:57.708Z24/30 Write block 168 len 40960 data: 2 3 4 5 5 5 7 6 6 6
26862024-02-01T17:28:57.708Z25/30 Write block 352 len 4096 data: 4
26872024-02-01T17:28:57.708Z26/30 Read block 130 len 24576
26882024-02-01T17:28:57.711Z27/30 Write block 181 len 40960 data: 5 4 5 5 6 4 3 3 3 2
26892024-02-01T17:28:57.714Z28/30 Write block 3 len 24576 data: 2 2 2 2 2 3
26902024-02-01T17:28:57.714Z29/30 Read block 377 len 36864
26912024-02-01T17:28:57.718Z30/30 Write block 359 len 36864 data: 3 2 2 2 3 2 2 2 2
26922024-02-01T17:28:57.718Z----------------------------------------------------------------
26932024-02-01T17:28:57.718Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:29
26942024-02-01T17:28:57.718ZStates: Active Active Active
26952024-02-01T17:28:57.718ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
26962024-02-01T17:28:57.718Z 2 Acked 1001 Write 1 Done Done Done false
26972024-02-01T17:28:57.718Z 3 Acked 1002 Read 6 Done Done Sent false
26982024-02-01T17:28:57.718Z 4 Acked 1003 Read 6 Done Done Sent false
26992024-02-01T17:28:57.718Z 5 Acked 1004 Read 7 Done Done Sent false
27002024-02-01T17:28:57.718Z 6 Acked 1005 Write 2 Done Done Sent false
27012024-02-01T17:28:57.718Z 7 Acked 1006 Read 8 Done Done Sent false
27022024-02-01T17:28:57.718Z 8 Acked 1007 Write 7 Done Done Sent false
27032024-02-01T17:28:57.718Z 9 Acked 1008 Read 1 Done Done Sent false
27042024-02-01T17:28:57.718Z 10 Acked 1009 Read 2 Done Done Sent false
27052024-02-01T17:28:57.718Z 11 Acked 1010 Write 3 Done Done Sent false
27062024-02-01T17:28:57.718Z 12 Acked 1011 Read 1 Done Done Sent false
27072024-02-01T17:28:57.718Z 13 Acked 1012 Read 5 Done Done Sent false
27082024-02-01T17:28:57.718Z 14 Acked 1013 Write 5 Done Done Sent false
27092024-02-01T17:28:57.719Z 15 Acked 1014 Read 2 Done Done Sent false
27102024-02-01T17:28:57.719Z 16 Acked 1015 Flush 0 Done Done Sent false
27112024-02-01T17:28:57.719Z 17 Acked 1016 Write 3 Done Done Sent false
27122024-02-01T17:28:57.719Z 18 Acked 1017 Read 8 Done Done Sent false
27132024-02-01T17:28:57.719Z 19 Acked 1018 Read 5 Done Done Sent false
27142024-02-01T17:28:57.721Z 20 Acked 1019 Flush 0 Done Done Sent false
27152024-02-01T17:28:57.721Z 21 Acked 1020 Write 1 Done Done Sent false
27162024-02-01T17:28:57.721Z 22 Acked 1021 Write 9 Done Done Sent false
27172024-02-01T17:28:57.721Z 23 Acked 1022 Write 7 Done Done Sent false
27182024-02-01T17:28:57.721Z 24 Acked 1023 Write 10 Done Done Sent false
27192024-02-01T17:28:57.721Z 25 Acked 1024 Write 1 Done Done Sent false
27202024-02-01T17:28:57.721Z 26 Acked 1025 Read 6 Done Done Sent false
27212024-02-01T17:28:57.721Z 27 Acked 1026 Write 10 Done Done Sent false
27222024-02-01T17:28:57.721Z 28 Acked 1027 Write 6 Done Done Sent false
27232024-02-01T17:28:57.721Z 29 Acked 1028 Read 9 Done Done Sent false
27242024-02-01T17:28:57.721Z 30 Acked 1029 Write 9 Sent Sent Sent false
27252024-02-01T17:28:57.721Z STATES DS:0 DS:1 DS:2 TOTAL
27262024-02-01T17:28:57.721Z New 0 0 0 0
27272024-02-01T17:28:57.721Z Sent 1 1 28 30
27282024-02-01T17:28:57.721Z Done 28 28 1 57
27292024-02-01T17:28:57.721Z Skipped 0 0 0 0
27302024-02-01T17:28:57.721Z Error 0 0 0 0
27312024-02-01T17:28:57.721ZLast Flush: 1019 1019 1000
27322024-02-01T17:28:57.721ZDownstairs last five completed: 1000
27332024-02-01T17:28:57.721ZUpstairs last five completed: 30 29 28 27 26
27342024-02-01T17:28:57.721Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:28:56.979545933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1225}
27352024-02-01T17:28:57.722ZWrote out file "/tmp/test_repair_verify.data"
27362024-02-01T17:28:57.725Z
27372024-02-01T17:28:57.729ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27382024-02-01T17:28:57.756ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27392024-02-01T17:28:57.756Z 0 000-019 24 24 3 37 37 5 T T T <---
27402024-02-01T17:28:57.759Z 1 020-039 24 24 20 37 37 31 F F F <---
27412024-02-01T17:28:57.759Z 2 040-059 22 22 22 34 34 34 F F F
27422024-02-01T17:28:57.759Z 3 060-079 22 22 22 34 34 34 F F F
27432024-02-01T17:28:57.759Z 4 080-099 23 23 23 35 35 35 F F F
27442024-02-01T17:28:57.759Z 5 100-119 23 23 23 35 35 35 F F F
27452024-02-01T17:28:57.759Z 6 120-139 23 23 23 35 35 35 F F F
27462024-02-01T17:28:57.759Z 7 140-159 21 21 21 33 33 33 F F F
27472024-02-01T17:28:57.759Z 8 160-179 21 21 21 33 33 33 T T F <---
27482024-02-01T17:28:57.759Z 9 180-199 23 23 23 35 35 35 T T F <---
27492024-02-01T17:28:57.759Z 10 200-219 23 23 23 35 35 35 F F F
27502024-02-01T17:28:57.760Z 11 220-239 9 9 9 13 13 13 F F F
27512024-02-01T17:28:57.760Z 12 240-259 24 24 21 37 37 33 F F F <---
27522024-02-01T17:28:57.760Z 13 260-279 24 24 18 37 37 28 F F F <---
27532024-02-01T17:28:57.760Z 14 280-299 23 23 23 35 35 35 F F F
27542024-02-01T17:28:57.760Z 15 300-319 21 21 21 33 33 33 F F F
27552024-02-01T17:28:57.760Z 16 320-339 23 23 23 35 35 35 F F F
27562024-02-01T17:28:57.760Z 17 340-359 22 22 22 34 34 34 T T F <---
27572024-02-01T17:28:57.760Z 18 360-379 16 16 16 24 24 24 T T F <---
27582024-02-01T17:28:57.760Z 19 380-399 19 19 19 30 30 30 F F F
27592024-02-01T17:28:57.761Z 20 400-419 14 14 14 21 21 21 F F F
27602024-02-01T17:28:57.761Z 21 420-439 24 24 23 38 38 35 F F F <---
27612024-02-01T17:28:57.762Z 22 440-459 23 23 23 35 35 35 F F F
27622024-02-01T17:28:57.762Z 23 460-479 16 16 16 25 25 25 T T F <---
27632024-02-01T17:28:57.762Z 24 480-499 20 20 20 31 31 31 F F F
27642024-02-01T17:28:57.762Z 25 500-519 10 10 10 14 14 14 F F F
27652024-02-01T17:28:57.762Z 26 520-539 18 18 18 29 29 29 F F F
27662024-02-01T17:28:57.762Z 27 540-559 22 22 22 34 34 34 T T F <---
27672024-02-01T17:28:57.762Z 28 560-579 19 19 19 30 30 30 F F F
27682024-02-01T17:28:57.762Z 29 580-599 16 16 16 25 25 25 F F F
27692024-02-01T17:28:57.762ZMax gen: 24, Max flush: 38
27702024-02-01T17:28:57.762ZError: Difference in extent metadata found!
27712024-02-01T17:28:57.765ZA Difference in extent metadata is expected here
27722024-02-01T17:28:57.765ZOn loop 12
27732024-02-01T17:28:57.765Z
27742024-02-01T17:28:57.765ZVerifying data now
27752024-02-01T17:29:02.864ZNo change after verify
27762024-02-01T17:29:02.864ZLoop: 12 Downstairs dump after verify (and repair):
27772024-02-01T17:29:02.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27782024-02-01T17:29:02.873Z 0 000-019 25 25 25 39 39 39 F F F
27792024-02-01T17:29:02.873Z 1 020-039 24 24 24 37 37 37 F F F
27802024-02-01T17:29:02.873Z 2 040-059 22 22 22 34 34 34 F F F
27812024-02-01T17:29:02.873Z 3 060-079 22 22 22 34 34 34 F F F
27822024-02-01T17:29:02.873Z 4 080-099 23 23 23 35 35 35 F F F
27832024-02-01T17:29:02.873Z 5 100-119 23 23 23 35 35 35 F F F
27842024-02-01T17:29:02.874Z 6 120-139 23 23 23 35 35 35 F F F
27852024-02-01T17:29:02.874Z 7 140-159 21 21 21 33 33 33 F F F
27862024-02-01T17:29:02.874Z 8 160-179 25 25 25 39 39 39 F F F
27872024-02-01T17:29:02.874Z 9 180-199 25 25 25 39 39 39 F F F
27882024-02-01T17:29:02.874Z 10 200-219 23 23 23 35 35 35 F F F
27892024-02-01T17:29:02.874Z 11 220-239 9 9 9 13 13 13 F F F
27902024-02-01T17:29:02.874Z 12 240-259 24 24 24 37 37 37 F F F
27912024-02-01T17:29:02.874Z 13 260-279 24 24 24 37 37 37 F F F
27922024-02-01T17:29:02.874Z 14 280-299 23 23 23 35 35 35 F F F
27932024-02-01T17:29:02.874Z 15 300-319 21 21 21 33 33 33 F F F
27942024-02-01T17:29:02.874Z 16 320-339 23 23 23 35 35 35 F F F
27952024-02-01T17:29:02.874Z 17 340-359 25 25 25 39 39 39 F F F
27962024-02-01T17:29:02.874Z 18 360-379 25 25 25 39 39 39 F F F
27972024-02-01T17:29:02.875Z 19 380-399 19 19 19 30 30 30 F F F
27982024-02-01T17:29:02.875Z 20 400-419 14 14 14 21 21 21 F F F
27992024-02-01T17:29:02.875Z 21 420-439 24 24 24 38 38 38 F F F
28002024-02-01T17:29:02.875Z 22 440-459 23 23 23 35 35 35 F F F
28012024-02-01T17:29:02.875Z 23 460-479 25 25 25 39 39 39 F F F
28022024-02-01T17:29:02.875Z 24 480-499 20 20 20 31 31 31 F F F
28032024-02-01T17:29:02.875Z 25 500-519 10 10 10 14 14 14 F F F
28042024-02-01T17:29:02.875Z 26 520-539 18 18 18 29 29 29 F F F
28052024-02-01T17:29:02.875Z 27 540-559 25 25 25 39 39 39 F F F
28062024-02-01T17:29:02.875Z 28 560-579 19 19 19 30 30 30 F F F
28072024-02-01T17:29:02.875Z 29 580-599 16 16 16 25 25 25 F F F
28082024-02-01T17:29:02.876ZMax gen: 25, Max flush: 39
28092024-02-01T17:29:02.876Z
28102024-02-01T17:29:02.876ZBegin loop 13
28112024-02-01T17:29:02.876ZDownstairs to restart: 2
28122024-02-01T17:29:02.876Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.063816277Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
28132024-02-01T17:29:02.876Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064280576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
28142024-02-01T17:29:02.876Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064324626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
28152024-02-01T17:29:02.876Z{"msg":"Crucible 03d4311a-5a46-436d-8883-f99aaf18da94 has session id: 88823094-d4ac-4903-984f-3886a0333602","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064364977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28162024-02-01T17:29:02.876Z{"msg":"Upstairs opts: Upstairs UUID: 03d4311a-5a46-436d-8883-f99aaf18da94, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
28172024-02-01T17:29:02.876Z2024-02-01T17:29:02.064420275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28182024-02-01T17:29:02.876Z{"msg":"Crucible stats registered with UUID: 03d4311a-5a46-436d-8883-f99aaf18da94","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064524341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28192024-02-01T17:29:02.877Z{"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064574807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
28202024-02-01T17:29:02.877Z{"msg":"03d4311a-5a46-436d-8883-f99aaf18da94 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064784293Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28212024-02-01T17:29:02.877Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064846668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28222024-02-01T17:29:02.877Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064881496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28232024-02-01T17:29:02.878Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.06490578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28242024-02-01T17:29:02.878Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.064993591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28252024-02-01T17:29:02.879Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065026482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28262024-02-01T17:29:02.879Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065088235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28272024-02-01T17:29:02.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065130507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28282024-02-01T17:29:02.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065173107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28292024-02-01T17:29:02.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065286728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28302024-02-01T17:29:02.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065332505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28312024-02-01T17:29:02.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065377212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28322024-02-01T17:29:02.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.065454058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"io task","client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28332024-02-01T17:29:02.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.066580961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28342024-02-01T17:29:02.881Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.06664539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28352024-02-01T17:29:02.881Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.066694463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28362024-02-01T17:29:02.881Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.066808355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28372024-02-01T17:29:02.881Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.067593047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28382024-02-01T17:29:02.882Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.067823496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28392024-02-01T17:29:02.882Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.067893866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28402024-02-01T17:29:02.882Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.067938484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28412024-02-01T17:29:02.882Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068434912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28422024-02-01T17:29:02.882Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068496851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28432024-02-01T17:29:02.882Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068521298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28442024-02-01T17:29:02.882Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068757663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28452024-02-01T17:29:02.883Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068815472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28462024-02-01T17:29:02.883Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068841769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28472024-02-01T17:29:02.883Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.068882999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28482024-02-01T17:29:02.883Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069545384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28492024-02-01T17:29:02.883Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069635603Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28502024-02-01T17:29:02.884Z{"msg":"[0]R flush_numbers[0..12]: [39, 37, 34, 34, 35, 35, 35, 33, 39, 39, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069694691Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28512024-02-01T17:29:02.884Z{"msg":"[0]R generation[0..12]: [25, 24, 22, 22, 23, 23, 23, 21, 25, 25, 23, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069722389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28522024-02-01T17:29:02.884Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069765398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28532024-02-01T17:29:02.886Z{"msg":"[1]R flush_numbers[0..12]: [39, 37, 34, 34, 35, 35, 35, 33, 39, 39, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069790989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28542024-02-01T17:29:02.886Z{"msg":"[1]R generation[0..12]: [25, 24, 22, 22, 23, 23, 23, 21, 25, 25, 23, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069814041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28552024-02-01T17:29:02.886Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069861598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28562024-02-01T17:29:02.886Z{"msg":"[2]R flush_numbers[0..12]: [39, 37, 34, 34, 35, 35, 35, 33, 39, 39, 35, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069886949Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28572024-02-01T17:29:02.886Z{"msg":"[2]R generation[0..12]: [25, 24, 22, 22, 23, 23, 23, 21, 25, 25, 23, 9]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
28582024-02-01T17:29:02.886Z,"time":"2024-02-01T17:29:02.069945664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28592024-02-01T17:29:02.887Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.069991349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28602024-02-01T17:29:02.887Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070015398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28612024-02-01T17:29:02.887Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070063403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28622024-02-01T17:29:02.887Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070086833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28632024-02-01T17:29:02.887Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070127823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"":"downstairs","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28642024-02-01T17:29:02.887Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070152047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28652024-02-01T17:29:02.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070196755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"0","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28662024-02-01T17:29:02.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070252846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"1","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28672024-02-01T17:29:02.888Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070293707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"client":"2","":"downstairs","session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28682024-02-01T17:29:02.888Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070351794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28692024-02-01T17:29:02.888Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070380998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28702024-02-01T17:29:02.888Z{"msg":"03d4311a-5a46-436d-8883-f99aaf18da94 is now active with session: 88823094-d4ac-4903-984f-3886a0333602","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070444235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28712024-02-01T17:29:02.888Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:02.070514801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233,"session_id":"88823094-d4ac-4903-984f-3886a0333602"}
28722024-02-01T17:29:02.889Z{"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28732024-02-01T17:29:02.889Z,"time":"2024-02-01T17:29:02.070555928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
28742024-02-01T17:29:02.889ZLoading write count information from file "/tmp/test_repair_verify.data"
28752024-02-01T17:29:02.889ZRun Repair workload
28762024-02-01T17:29:02.889Z01/30 Write block 293 len 32768 data: 4 5 4 5 4 5 5 5
28772024-02-01T17:29:02.912Z02/30 Write block 244 len 28672 data: 3 4 5 5 5 5 4
28782024-02-01T17:29:02.912Z03/30 Read block 148 len 24576
28792024-02-01T17:29:02.912Z04/30 Write block 425 len 4096 data: 2
28802024-02-01T17:29:02.912Z05/30 Read block 586 len 40960
28812024-02-01T17:29:02.913Z06/30 Write block 83 len 4096 data: 3
28822024-02-01T17:29:02.913Z07/30 Read block 277 len 12288
28832024-02-01T17:29:02.913Z08/30 Write block 201 len 36864 data: 3 4 4 6 6 6 6 5 3
28842024-02-01T17:29:02.913Z09/30 Write block 361 len 40960 data: 3 3 4 3 3 3 3 2 2 2
28852024-02-01T17:29:02.913Z10/30 Read block 329 len 8192
28862024-02-01T17:29:02.913Z11/30 Write block 589 len 28672 data: 4 4 3 2 2 2 2
28872024-02-01T17:29:02.913Z12/30 Read block 325 len 8192
28882024-02-01T17:29:02.913Z13/30 Write block 381 len 16384 data: 6 4 4 3
28892024-02-01T17:29:02.913Z14/30 Write block 51 len 4096 data: 4
28902024-02-01T17:29:02.913Z15/30 Read block 6 len 32768
28912024-02-01T17:29:02.913Z16/30 Read block 55 len 12288
28922024-02-01T17:29:02.913Z17/30 Read block 252 len 24576
28932024-02-01T17:29:02.913Z18/30 Read block 81 len 4096
28942024-02-01T17:29:02.913Z19/30 Write block 165 len 4096 data: 2
28952024-02-01T17:29:02.913Z20/30 Read block 391 len 24576
28962024-02-01T17:29:02.913Z21/30 Write block 522 len 12288 data: 3 4 4
28972024-02-01T17:29:02.913Z22/30 Write block 50 len 8192 data: 5 5
28982024-02-01T17:29:02.914Z23/30 Read block 331 len 8192
28992024-02-01T17:29:02.914Z24/30 Read block 3 len 28672
29002024-02-01T17:29:02.914Z25/30 Write block 174 len 16384 data: 8 7 7 7
29012024-02-01T17:29:02.914Z26/30 Write block 449 len 8192 data: 5 4
29022024-02-01T17:29:02.914Z27/30 Write block 332 len 32768 data: 3 4 6 6 6 4 4 3
29032024-02-01T17:29:02.914Z28/30 Read block 324 len 24576
29042024-02-01T17:29:02.914Z29/30 Write block 558 len 40960 data: 4 4 4 4 4 5 5 5 3 2
29052024-02-01T17:29:02.914Z30/30 Read block 125 len 4096
29062024-02-01T17:29:02.916Z----------------------------------------------------------------
29072024-02-01T17:29:02.916Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
29082024-02-01T17:29:02.916ZStates: Active Active Active
29092024-02-01T17:29:02.916ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29102024-02-01T17:29:02.916Z 1 Acked 1000 Write 8 Done Done Done false
29112024-02-01T17:29:02.916Z 2 Acked 1001 Write 7 Done Done Done false
29122024-02-01T17:29:02.916Z 3 Acked 1002 Read 6 Done Done Done false
29132024-02-01T17:29:02.916Z 4 Acked 1003 Write 1 Done Done Done false
29142024-02-01T17:29:02.916Z 5 Acked 1004 Read 10 Done Done Done false
29152024-02-01T17:29:02.916Z 6 Acked 1005 Write 1 Done Done Sent false
29162024-02-01T17:29:02.916Z 7 Acked 1006 Read 3 Done Done Sent false
29172024-02-01T17:29:02.916Z 8 Acked 1007 Write 9 Done Done Sent false
29182024-02-01T17:29:02.916Z 9 Acked 1008 Write 10 Done Done Sent false
29192024-02-01T17:29:02.916Z 10 Acked 1009 Read 2 Done Done Sent false
29202024-02-01T17:29:02.916Z 11 Acked 1010 Write 7 Done Done Sent false
29212024-02-01T17:29:02.917Z 12 Acked 1011 Read 2 Done Done Sent false
29222024-02-01T17:29:02.917Z 13 Acked 1012 Write 4 Done Done Sent false
29232024-02-01T17:29:02.917Z 14 Acked 1013 Write 1 Done Done Sent false
29242024-02-01T17:29:02.917Z 15 Acked 1014 Read 8 Done Done Sent false
29252024-02-01T17:29:02.917Z 16 Acked 1015 Read 3 Done Done Sent false
29262024-02-01T17:29:02.917Z 17 Acked 1016 Read 6 Done Done Sent false
29272024-02-01T17:29:02.917Z 18 Acked 1017 Read 1 Done Done Sent false
29282024-02-01T17:29:02.917Z 19 Acked 1018 Write 1 Done Done Sent false
29292024-02-01T17:29:02.917Z 20 Acked 1019 Read 6 Done Done Sent false
29302024-02-01T17:29:02.917Z 21 Acked 1020 Write 3 Done Done Sent false
29312024-02-01T17:29:02.917Z 22 Acked 1021 Write 2 Done Done Sent false
29322024-02-01T17:29:02.917Z 23 Acked 1022 Read 2 Done Done Sent false
29332024-02-01T17:29:02.917Z 24 Acked 1023 Read 7 Done Done Sent false
29342024-02-01T17:29:02.917Z 25 Acked 1024 Write 4 Done Done Sent false
29352024-02-01T17:29:02.917Z 26 Acked 1025 Write 2 Done Done Sent false
29362024-02-01T17:29:02.917Z 27 Acked 1026 Write 8 Done Done Sent false
29372024-02-01T17:29:02.917Z 28 Acked 1027 Read 6 Done Done Sent false
29382024-02-01T17:29:02.917Z 29 Acked 1028 Write 10 Done Done Sent false
29392024-02-01T17:29:02.917Z 30 Acked 1029 Read 1 Done Sent Sent false
29402024-02-01T17:29:02.917Z STATES DS:0 DS:1 DS:2 TOTAL
29412024-02-01T17:29:02.917Z New 0 0 0 0
29422024-02-01T17:29:02.917Z Sent 0 1 25 26
29432024-02-01T17:29:02.918Z Done 30 29 5 64
29442024-02-01T17:29:02.918Z Skipped 0 0 0 0
29452024-02-01T17:29:02.918Z Error 0 0 0 0
29462024-02-01T17:29:02.919ZLast Flush: 0 0 0
29472024-02-01T17:29:02.919ZDownstairs last five completed:
29482024-02-01T17:29:02.919ZUpstairs last five completed: 30 29 28 27 26
29492024-02-01T17:29:02.919Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:02.110477053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1233}
29502024-02-01T17:29:02.919ZWrote out file "/tmp/test_repair_verify.data"
29512024-02-01T17:29:02.919Z
29522024-02-01T17:29:02.919ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
29532024-02-01T17:29:02.919ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29542024-02-01T17:29:02.919Z 0 000-019 25 25 25 39 39 39 F F F
29552024-02-01T17:29:02.919Z 1 020-039 24 24 24 37 37 37 F F F
29562024-02-01T17:29:02.919Z 2 040-059 22 22 22 34 34 34 T T F <---
29572024-02-01T17:29:02.919Z 3 060-079 22 22 22 34 34 34 F F F
29582024-02-01T17:29:02.919Z 4 080-099 23 23 23 35 35 35 T T F <---
29592024-02-01T17:29:02.920Z 5 100-119 23 23 23 35 35 35 F F F
29602024-02-01T17:29:02.920Z 6 120-139 23 23 23 35 35 35 F F F
29612024-02-01T17:29:02.920Z 7 140-159 21 21 21 33 33 33 F F F
29622024-02-01T17:29:02.920Z 8 160-179 25 25 25 39 39 39 T T F <---
29632024-02-01T17:29:02.920Z 9 180-199 25 25 25 39 39 39 F F F
29642024-02-01T17:29:02.920Z 10 200-219 23 23 23 35 35 35 T T F <---
29652024-02-01T17:29:02.920Z 11 220-239 9 9 9 13 13 13 F F F
29662024-02-01T17:29:02.920Z 12 240-259 24 24 24 37 37 37 T T T
29672024-02-01T17:29:02.920Z 13 260-279 24 24 24 37 37 37 F F F
29682024-02-01T17:29:02.920Z 14 280-299 23 23 23 35 35 35 T T T
29692024-02-01T17:29:02.920Z 15 300-319 21 21 21 33 33 33 T T T
29702024-02-01T17:29:02.921Z 16 320-339 23 23 23 35 35 35 T T F <---
29712024-02-01T17:29:02.921Z 17 340-359 25 25 25 39 39 39 F F F
29722024-02-01T17:29:02.921Z 18 360-379 25 25 25 39 39 39 T T F <---
29732024-02-01T17:29:02.921Z 19 380-399 19 19 19 30 30 30 T T F <---
29742024-02-01T17:29:02.921Z 20 400-419 14 14 14 21 21 21 F F F
29752024-02-01T17:29:02.921Z 21 420-439 24 24 24 38 38 38 T T T
29762024-02-01T17:29:02.921Z 22 440-459 23 23 23 35 35 35 T T F <---
29772024-02-01T17:29:02.928Z 23 460-479 25 25 25 39 39 39 F F F
29782024-02-01T17:29:02.928Z 24 480-499 20 20 20 31 31 31 F F F
29792024-02-01T17:29:02.928Z 25 500-519 10 10 10 14 14 14 F F F
29802024-02-01T17:29:02.928Z 26 520-539 18 18 18 29 29 29 T T F <---
29812024-02-01T17:29:02.928Z 27 540-559 25 25 25 39 39 39 T T F <---
29822024-02-01T17:29:02.928Z 28 560-579 19 19 19 30 30 30 T T F <---
29832024-02-01T17:29:02.928Z 29 580-599 16 16 16 25 25 25 T T F <---
29842024-02-01T17:29:02.928ZMax gen: 25, Max flush: 39
29852024-02-01T17:29:02.928ZError: Difference in extent metadata found!
29862024-02-01T17:29:02.928ZA Difference in extent metadata is expected here
29872024-02-01T17:29:02.929ZOn loop 13
29882024-02-01T17:29:02.929Z
29892024-02-01T17:29:02.929ZVerifying data now
29902024-02-01T17:29:09.261ZNo change after verify
29912024-02-01T17:29:09.261ZLoop: 13 Downstairs dump after verify (and repair):
29922024-02-01T17:29:09.398ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29932024-02-01T17:29:09.400Z 0 000-019 25 25 25 39 39 39 F F F
29942024-02-01T17:29:09.407Z 1 020-039 24 24 24 37 37 37 F F F
29952024-02-01T17:29:09.407Z 2 040-059 26 26 26 40 40 40 F F F
29962024-02-01T17:29:09.407Z 3 060-079 22 22 22 34 34 34 F F F
29972024-02-01T17:29:09.407Z 4 080-099 26 26 26 40 40 40 F F F
29982024-02-01T17:29:09.407Z 5 100-119 23 23 23 35 35 35 F F F
29992024-02-01T17:29:09.407Z 6 120-139 23 23 23 35 35 35 F F F
30002024-02-01T17:29:09.407Z 7 140-159 21 21 21 33 33 33 F F F
30012024-02-01T17:29:09.408Z 8 160-179 26 26 26 40 40 40 F F F
30022024-02-01T17:29:09.408Z 9 180-199 25 25 25 39 39 39 F F F
30032024-02-01T17:29:09.408Z 10 200-219 26 26 26 40 40 40 F F F
30042024-02-01T17:29:09.408Z 11 220-239 9 9 9 13 13 13 F F F
30052024-02-01T17:29:09.408Z 12 240-259 26 26 26 40 40 40 F F F
30062024-02-01T17:29:09.408Z 13 260-279 24 24 24 37 37 37 F F F
30072024-02-01T17:29:09.408Z 14 280-299 26 26 26 40 40 40 F F F
30082024-02-01T17:29:09.408Z 15 300-319 26 26 26 40 40 40 F F F
30092024-02-01T17:29:09.408Z 16 320-339 26 26 26 40 40 40 F F F
30102024-02-01T17:29:09.408Z 17 340-359 25 25 25 39 39 39 F F F
30112024-02-01T17:29:09.408Z 18 360-379 26 26 26 40 40 40 F F F
30122024-02-01T17:29:09.408Z 19 380-399 26 26 26 40 40 40 F F F
30132024-02-01T17:29:09.408Z 20 400-419 14 14 14 21 21 21 F F F
30142024-02-01T17:29:09.408Z 21 420-439 26 26 26 40 40 40 F F F
30152024-02-01T17:29:09.408Z 22 440-459 26 26 26 40 40 40 F F F
30162024-02-01T17:29:09.408Z 23 460-479 25 25 25 39 39 39 F F F
30172024-02-01T17:29:09.408Z 24 480-499 20 20 20 31 31 31 F F F
30182024-02-01T17:29:09.408Z 25 500-519 10 10 10 14 14 14 F F F
30192024-02-01T17:29:09.408Z 26 520-539 26 26 26 40 40 40 F F F
30202024-02-01T17:29:09.408Z 27 540-559 26 26 26 40 40 40 F F F
30212024-02-01T17:29:09.408Z 28 560-579 26 26 26 40 40 40 F F F
30222024-02-01T17:29:09.408Z 29 580-599 26 26 26 40 40 40 F F F
30232024-02-01T17:29:09.409ZMax gen: 26, Max flush: 40
30242024-02-01T17:29:09.409Z
30252024-02-01T17:29:09.409ZBegin loop 14
30262024-02-01T17:29:09.409ZDownstairs to restart: 2
30272024-02-01T17:29:09.409Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.57928929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
30282024-02-01T17:29:09.409Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.579833431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
30292024-02-01T17:29:09.409Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"Crucible runtime is spawned
30302024-02-01T17:29:09.409Zname":"crucible","level":30,"time":"2024-02-01T17:29:08.580050293Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
30312024-02-01T17:29:09.409Z{"msg":"Crucible 135d46b6-cfb5-43f8-b77b-b0b9ac887a5e has session id: 1c990c10-e267-4286-b171-677fbd92d6ca","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580154848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30322024-02-01T17:29:09.409Z{"msg":"Upstairs opts: Upstairs UUID: 135d46b6-cfb5-43f8-b77b-b0b9ac887a5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580238786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30332024-02-01T17:29:09.409Z{"msg":"Crucible stats registered with UUID: 135d46b6-cfb5-43f8-b77b-b0b9ac887a5e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580311485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30342024-02-01T17:29:09.410Z{"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580447593Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
30352024-02-01T17:29:09.410Z{"msg":"135d46b6-cfb5-43f8-b77b-b0b9ac887a5e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580489172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30362024-02-01T17:29:09.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580558173Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30372024-02-01T17:29:09.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.58066734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30382024-02-01T17:29:09.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580732231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30392024-02-01T17:29:09.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580760732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30402024-02-01T17:29:09.411Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580783036Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30412024-02-01T17:29:09.411Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580810867Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30422024-02-01T17:29:09.411Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580879416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30432024-02-01T17:29:09.411Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.580956106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30442024-02-01T17:29:09.411Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.581115127Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30452024-02-01T17:29:09.411Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.581197703Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30462024-02-01T17:29:09.412Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.581318981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30472024-02-01T17:29:09.412Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.581403112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"io task","client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30482024-02-01T17:29:09.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.582232703Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30492024-02-01T17:29:09.412Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.582320202Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30502024-02-01T17:29:09.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.582350048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30512024-02-01T17:29:09.412Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.582373101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30522024-02-01T17:29:09.412Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.583123366Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30532024-02-01T17:29:09.412Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.583214352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30542024-02-01T17:29:09.413Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.583260714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30552024-02-01T17:29:09.413Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.583333524Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30562024-02-01T17:29:09.413Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584146046Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30572024-02-01T17:29:09.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584225688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30582024-02-01T17:29:09.413Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584256183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30592024-02-01T17:29:09.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.58427961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30602024-02-01T17:29:09.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584337501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30612024-02-01T17:29:09.413Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584368122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30622024-02-01T17:29:09.413Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.584440519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30632024-02-01T17:29:09.413Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585100029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30642024-02-01T17:29:09.414Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585165034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30652024-02-01T17:29:09.414Z{"msg":"[0]R flush_numbers[0..12]: [39, 37, 40, 34, 40, 35, 35, 33, 40, 39, 40, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585193404Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30662024-02-01T17:29:09.414Z{"msg":"[0]R generation[0..12]: [25, 24, 26, 22, 26, 23, 23, 21, 26, 25, 26, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585236325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30672024-02-01T17:29:09.414Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585262767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30682024-02-01T17:29:09.414Z{"msg":"[1]R flush_numbers[0..12]: [39, 37, 40, 34, 40, 35, 35, 33, 40, 39, 40, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585288361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30692024-02-01T17:29:09.414Z{"msg":"[1]R generation[0..12]: [25, 24, 26, 22, 26, 23, 23, 21, 26, 25, 26, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.58531207Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30702024-02-01T17:29:09.414Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585355154Z","Wait for a query_work_queue command to finish before sending IO
30712024-02-01T17:29:09.414Zhostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30722024-02-01T17:29:09.414Z{"msg":"[2]R flush_numbers[0..12]: [39, 37, 40, 34, 40, 35, 35, 33, 40, 39, 40, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585384053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30732024-02-01T17:29:09.414Z{"msg":"[2]R generation[0..12]: [25, 24, 26, 22, 26, 23, 23, 21, 26, 25, 26, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585407553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30742024-02-01T17:29:09.414Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585475773Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30752024-02-01T17:29:09.414Z{"msg":"Max found gen is 27","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585538857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30762024-02-01T17:29:09.414Z{"msg":"Generation requested: 28 >= found:27","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585606642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30772024-02-01T17:29:09.414Z{"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585682582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30782024-02-01T17:29:09.414Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585711771Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"":"downstairs","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30792024-02-01T17:29:09.414Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585779323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30802024-02-01T17:29:09.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30812024-02-01T17:29:09.414Z,"time":"2024-02-01T17:29:08.585849523Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"0","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30822024-02-01T17:29:09.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.58588735Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"1","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30832024-02-01T17:29:09.415Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585948865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"client":"2","":"downstairs","session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30842024-02-01T17:29:09.415Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.585978153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30852024-02-01T17:29:09.415Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.586019485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30862024-02-01T17:29:09.415Z{"msg":"135d46b6-cfb5-43f8-b77b-b0b9ac887a5e is now active with session: 1c990c10-e267-4286-b171-677fbd92d6ca","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.586043012Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30872024-02-01T17:29:09.415Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.586081935Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241,"session_id":"1c990c10-e267-4286-b171-677fbd92d6ca"}
30882024-02-01T17:29:09.415Z{"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:08.586104333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
30892024-02-01T17:29:09.415ZLoading write count information from file "/tmp/test_repair_verify.data"
30902024-02-01T17:29:09.415ZRun Repair workload
30912024-02-01T17:29:09.415Z01/30 Write block 22 len 40960 data: 3 3 3 3 3 3 2 2 3 3
30922024-02-01T17:29:09.416Z02/30 Read block 193 len 32768
30932024-02-01T17:29:09.416Z03/30 Write block 0 len 28672 data: 2 2 2 3 3 3 3
30942024-02-01T17:29:09.421Z04/30 Write block 359 len 24576 data: 4 3 4 4 5 4
30952024-02-01T17:29:09.421Z05/30 Read block 64 len 12288
30962024-02-01T17:29:09.421Z06/30 Read block 225 len 36864
30972024-02-01T17:29:09.421Z07/30 Write block 194 len 16384 data: 2 2 2 3
30982024-02-01T17:29:09.421Z08/30 Write block 206 len 16384 data: 7 7 6 4
30992024-02-01T17:29:09.421Z09/30 Read block 74 len 36864
31002024-02-01T17:29:09.422Z10/30 Write block 299 len 8192 data: 6 6
31012024-02-01T17:29:09.422Z11/30 Read block 15 len 36864
31022024-02-01T17:29:09.422Z12/30 Read block 22 len 16384
31032024-02-01T17:29:09.422Z13/30 Write block 555 len 36864 data: 4 3 4 5 5 5 5 5 6
31042024-02-01T17:29:09.423Z14/30 Read block 291 len 36864
31052024-02-01T17:29:09.423Z15/30 Write block 315 len 40960 data: 5 5 5 4 4 4 3 3 2 2
31062024-02-01T17:29:09.423Z16/30 Read block 225 len 4096
31072024-02-01T17:29:09.423Z17/30 Write block 330 len 36864 data: 2 3 4 5 7 7 7 5 5
31082024-02-01T17:29:09.423Z18/30 Read block 95 len 32768
31092024-02-01T17:29:09.423Z19/30 Read block 450 len 36864
31102024-02-01T17:29:09.424Z20/30 Read block 82 len 16384
31112024-02-01T17:29:09.424Z21/30 Write block 575 len 32768 data: 2 2 2 3 3 3 3 3
31122024-02-01T17:29:09.424Z22/30 Read block 378 len 12288
31132024-02-01T17:29:09.424Z23/30 Read block 327 len 40960
31142024-02-01T17:29:09.424Z24/30 Read block 75 len 40960
31152024-02-01T17:29:09.424Z25/30 Write block 435 len 16384 data: 7 6 7 5
31162024-02-01T17:29:09.424Z26/30 Read block 96 len 40960
31172024-02-01T17:29:09.424Z27/30 Read block 346 len 32768
31182024-02-01T17:29:09.424Z28/30 Write block 196 len 8192 data: 3 4
31192024-02-01T17:29:09.424Z29/30 Write block 466 len 8192 data: 4 4
31202024-02-01T17:29:09.424Z30/30 Write block 133 len 20480 data: 3 3 3 3 4
31212024-02-01T17:29:09.424Z----------------------------------------------------------------
31222024-02-01T17:29:09.424Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
31232024-02-01T17:29:09.424ZStates: Active Active Active
31242024-02-01T17:29:09.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
31252024-02-01T17:29:09.424Z 1 Acked 1000 Write 10 Done Done Done false
31262024-02-01T17:29:09.424Z 2 Acked 1001 Read 8 Done Done Done false
31272024-02-01T17:29:09.424Z 3 Acked 1002 Write 7 Done Done Done false
31282024-02-01T17:29:09.424Z 4 Acked 1003 Write 6 Done Done Done false
31292024-02-01T17:29:09.424Z 5 Acked 1004 Read 3 Done Done Done false
31302024-02-01T17:29:09.424Z 6 Acked 1005 Read 9 Done Done Done false
31312024-02-01T17:29:09.424Z 7 Acked 1006 Write 4 Done Done Done false
31322024-02-01T17:29:09.424Z 8 Acked 1007 Write 4 Done Done Sent false
31332024-02-01T17:29:09.424Z 9 Acked 1008 Read 9 Done Done Sent false
31342024-02-01T17:29:09.425Z 10 Acked 1009 Write 2 Done Done Sent false
31352024-02-01T17:29:09.425Z 11 Acked 1010 Read 9 Done Done Sent false
31362024-02-01T17:29:09.425Z 12 Acked 1011 Read 4 Done Done Sent false
31372024-02-01T17:29:09.425Z 13 Acked 1012 Write 9 Done Done Sent false
31382024-02-01T17:29:09.425Z 14 Acked 1013 Read 9 Done Done Sent false
31392024-02-01T17:29:09.425Z 15 Acked 1014 Write 10 Done Done Sent false
31402024-02-01T17:29:09.425Z 16 Acked 1015 Read 1 Done Done Sent false
31412024-02-01T17:29:09.425Z 17 Acked 1016 Write 9 Done Done Sent false
31422024-02-01T17:29:09.425Z 18 Acked 1017 Read 8 Done Done Sent false
31432024-02-01T17:29:09.425Z 19 Acked 1018 Read 9 Done Done Sent false
31442024-02-01T17:29:09.425Z 20 Acked 1019 Read 4 Done Done Sent false
31452024-02-01T17:29:09.425Z 21 Acked 1020 Write 8 Done Done Sent false
31462024-02-01T17:29:09.425Z 22 Acked 1021 Read 3 Done Done Sent false
31472024-02-01T17:29:09.425Z 23 Acked 1022 Read 10 Done Done Sent false
31482024-02-01T17:29:09.425Z 24 Acked 1023 Read 10 Done Done Sent false
31492024-02-01T17:29:09.425Z 25 Acked 1024 Write 4 Done Done Sent false
31502024-02-01T17:29:09.425Z 26 Acked 1025 Read 10 Done Done Sent false
31512024-02-01T17:29:09.425Z 27 Acked 1026 Read 8 Done Done Sent false
31522024-02-01T17:29:09.425Z 28 Acked 1027 Write 2 Done Done Sent false
31532024-02-01T17:29:09.425Z 29 Acked 1028 Write 2 Done Done Sent false
31542024-02-01T17:29:09.425Z 30 Acked 1029 Write 5 Sent Sent Sent false
31552024-02-01T17:29:09.425Z STATES DS:0 DS:1 DS:2 TOTAL
31562024-02-01T17:29:09.425Z New 0 0 0 0
31572024-02-01T17:29:09.425Z Sent 1 1 23 25
31582024-02-01T17:29:09.426Z Done 29 29 7 65
31592024-02-01T17:29:09.426Z Skipped 0 0 0 0
31602024-02-01T17:29:09.426Z Error 0 0 0 0
31612024-02-01T17:29:09.426ZLast Flush: 0 0 0
31622024-02-01T17:29:09.426ZDownstairs last five completed:
31632024-02-01T17:29:09.426ZUpstairs last five completed: 30 29 28 27 26
31642024-02-01T17:29:09.426Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:08.634749732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1241}
31652024-02-01T17:29:09.426ZWrote out file "/tmp/test_repair_verify.data"
31662024-02-01T17:29:09.426Z
31672024-02-01T17:29:09.426ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31682024-02-01T17:29:09.426ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31692024-02-01T17:29:09.426Z 0 000-019 25 25 25 39 39 39 T T T
31702024-02-01T17:29:09.426Z 1 020-039 24 24 24 37 37 37 T T T
31712024-02-01T17:29:09.426Z 2 040-059 26 26 26 40 40 40 F F F
31722024-02-01T17:29:09.426Z 3 060-079 22 22 22 34 34 34 F F F
31732024-02-01T17:29:09.426Z 4 080-099 26 26 26 40 40 40 F F F
31742024-02-01T17:29:09.426Z 5 100-119 23 23 23 35 35 35 F F F
31752024-02-01T17:29:09.426Z 6 120-139 23 23 23 35 35 35 T T F <---
31762024-02-01T17:29:09.428Z 7 140-159 21 21 21 33 33 33 F F F
31772024-02-01T17:29:09.428Z 8 160-179 26 26 26 40 40 40 F F F
31782024-02-01T17:29:09.428Z 9 180-199 25 25 25 39 39 39 T T T
31792024-02-01T17:29:09.428Z 10 200-219 26 26 26 40 40 40 T T F <---
31802024-02-01T17:29:09.428Z 11 220-239 9 9 9 13 13 13 F F F
31812024-02-01T17:29:09.428Z 12 240-259 26 26 26 40 40 40 F F F
31822024-02-01T17:29:09.428Z 13 260-279 24 24 24 37 37 37 F F F
31832024-02-01T17:29:09.428Z 14 280-299 26 26 26 40 40 40 T T F <---
31842024-02-01T17:29:09.429Z 15 300-319 26 26 26 40 40 40 T T F <---
31852024-02-01T17:29:09.429Z 16 320-339 26 26 26 40 40 40 T T F <---
31862024-02-01T17:29:09.429Z 17 340-359 25 25 25 39 39 39 T T T
31872024-02-01T17:29:09.429Z 18 360-379 26 26 26 40 40 40 T T T
31882024-02-01T17:29:09.429Z 19 380-399 26 26 26 40 40 40 F F F
31892024-02-01T17:29:09.429Z 20 400-419 14 14 14 21 21 21 F F F
31902024-02-01T17:29:09.429Z 21 420-439 26 26 26 40 40 40 T T F <---
31912024-02-01T17:29:09.429Z 22 440-459 26 26 26 40 40 40 F F F
31922024-02-01T17:29:09.429Z 23 460-479 25 25 25 39 39 39 T T F <---
31932024-02-01T17:29:09.429Z 24 480-499 20 20 20 31 31 31 F F F
31942024-02-01T17:29:09.433Z 25 500-519 10 10 10 14 14 14 F F F
31952024-02-01T17:29:09.433Z 26 520-539 26 26 26 40 40 40 F F F
31962024-02-01T17:29:09.433Z 27 540-559 26 26 26 40 40 40 T T F <---
31972024-02-01T17:29:09.433Z 28 560-579 26 26 26 40 40 40 T T F <---
31982024-02-01T17:29:09.433Z 29 580-599 26 26 26 40 40 40 T T F <---
31992024-02-01T17:29:09.433ZMax gen: 26, Max flush: 40
32002024-02-01T17:29:09.433ZError: Difference in extent metadata found!
32012024-02-01T17:29:09.434ZA Difference in extent metadata is expected here
32022024-02-01T17:29:09.434ZOn loop 14
32032024-02-01T17:29:09.434Z
32042024-02-01T17:29:09.434ZVerifying data now
32052024-02-01T17:29:16.232ZNo change after verify
32062024-02-01T17:29:16.232ZLoop: 14 Downstairs dump after verify (and repair):
32072024-02-01T17:29:16.423ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
32082024-02-01T17:29:16.423Z 0 000-019 27 27 27 41 41 41 F F F
32092024-02-01T17:29:16.423Z 1 020-039 27 27 27 41 41 41 F F F
32102024-02-01T17:29:16.423Z 2 040-059 26 26 26 40 40 40 F F F
32112024-02-01T17:29:16.424Z 3 060-079 22 22 22 34 34 34 F F F
32122024-02-01T17:29:16.424Z 4 080-099 26 26 26 40 40 40 F F F
32132024-02-01T17:29:16.424Z 5 100-119 23 23 23 35 35 35 F F F
32142024-02-01T17:29:16.424Z 6 120-139 27 27 27 41 41 41 F F F
32152024-02-01T17:29:16.424Z 7 140-159 21 21 21 33 33 33 F F F
32162024-02-01T17:29:16.424Z 8 160-179 26 26 26 40 40 40 F F F
32172024-02-01T17:29:16.424Z 9 180-199 27 27 27 41 41 41 F F F
32182024-02-01T17:29:16.424Z 10 200-219 27 27 27 41 41 41 F F F
32192024-02-01T17:29:16.425Z 11 220-239 9 9 9 13 13 13 F F F
32202024-02-01T17:29:16.425Z 12 240-259 26 26 26 40 40 40 F F F
32212024-02-01T17:29:16.425Z 13 260-279 24 24 24 37 37 37 F F F
32222024-02-01T17:29:16.425Z 14 280-299 27 27 27 41 41 41 F F F
32232024-02-01T17:29:16.425Z 15 300-319 27 27 27 41 41 41 F F F
32242024-02-01T17:29:16.425Z 16 320-339 27 27 27 41 41 41 F F F
32252024-02-01T17:29:16.425Z 17 340-359 27 27 27 41 41 41 F F F
32262024-02-01T17:29:16.425Z 18 360-379 27 27 27 41 41 41 F F F
32272024-02-01T17:29:16.425Z 19 380-399 26 26 26 40 40 40 F F F
32282024-02-01T17:29:16.425Z 20 400-419 14 14 14 21 21 21 F F F
32292024-02-01T17:29:16.426Z 21 420-439 27 27 27 41 41 41 F F F
32302024-02-01T17:29:16.426Z 22 440-459 26 26 26 40 40 40 F F F
32312024-02-01T17:29:16.426Z 23 460-479 27 27 27 41 41 41 F F F
32322024-02-01T17:29:16.426Z 24 480-499 20 20 20 31 31 31 F F F
32332024-02-01T17:29:16.426Z 25 500-519 10 10 10 14 14 14 F F F
32342024-02-01T17:29:16.426Z 26 520-539 26 26 26 40 40 40 F F F
32352024-02-01T17:29:16.426Z 27 540-559 27 27 27 41 41 41 F F F
32362024-02-01T17:29:16.426Z 28 560-579 27 27 27 41 41 41 F F F
32372024-02-01T17:29:16.426Z 29 580-599 27 27 27 41 41 41 F F F
32382024-02-01T17:29:16.426ZMax gen: 27, Max flush: 41
32392024-02-01T17:29:16.426Z
32402024-02-01T17:29:16.426ZBegin loop 15
32412024-02-01T17:29:16.426ZDownstairs to restart: 0
32422024-02-01T17:29:16.427Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038029911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
32432024-02-01T17:29:16.427Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038545361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
32442024-02-01T17:29:16.427Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038621539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
32452024-02-01T17:29:16.428Z{"msg":"Crucible 4a44180b-3277-4987-b332-3f8d76d4f468 has session id: 1ed8fb78-504e-423e-9dfe-4b769cdc7592","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038704892Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32462024-02-01T17:29:16.429Z{"msg":"Crucible runtime is spawned
32472024-02-01T17:29:16.429ZUpstairs opts: Upstairs UUID: 4a44180b-3277-4987-b332-3f8d76d4f468, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038769477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32482024-02-01T17:29:16.429Z{"msg":"Crucible stats registered with UUID: 4a44180b-3277-4987-b332-3f8d76d4f468","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.038831499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32492024-02-01T17:29:16.429Z{"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.03887269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
32502024-02-01T17:29:16.429Z{"msg":"4a44180b-3277-4987-b332-3f8d76d4f468 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039022873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32512024-02-01T17:29:16.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.0390807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32522024-02-01T17:29:16.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039133065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32532024-02-01T17:29:16.429Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.03917572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32542024-02-01T17:29:16.429Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039217963Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32552024-02-01T17:29:16.430Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039257041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32562024-02-01T17:29:16.430Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039340861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32572024-02-01T17:29:16.430Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039422022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32582024-02-01T17:29:16.430Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039472853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32592024-02-01T17:29:16.430Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.03952031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32602024-02-01T17:29:16.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039655276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32612024-02-01T17:29:16.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039727824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32622024-02-01T17:29:16.430Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.039844038Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"io task","client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32632024-02-01T17:29:16.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04089295Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32642024-02-01T17:29:16.430Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04106392Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32652024-02-01T17:29:16.430Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.041130005Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32662024-02-01T17:29:16.431Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04115786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32672024-02-01T17:29:16.431Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.041473183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32682024-02-01T17:29:16.431Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.041538909Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32692024-02-01T17:29:16.431Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.042099778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32702024-02-01T17:29:16.431Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.042195804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32712024-02-01T17:29:16.431Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.042562515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32722024-02-01T17:29:16.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.042911492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32732024-02-01T17:29:16.431Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043012808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32742024-02-01T17:29:16.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043162572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32752024-02-01T17:29:16.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04319088Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32762024-02-01T17:29:16.431Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043234511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32772024-02-01T17:29:16.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043259778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32782024-02-01T17:29:16.432Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04345237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32792024-02-01T17:29:16.432Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04351027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32802024-02-01T17:29:16.432Z{"msg":"[0]R flush_numbers[0..12]: [41, 41, 40, 34, 40, 35, 41, 33, 40, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043537746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32812024-02-01T17:29:16.432Z{"msg":"[0]R generation[0..12]: [27, 27, 26, 22, 26, 23, 27, 21, 26, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043580663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32822024-02-01T17:29:16.432Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043607332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32832024-02-01T17:29:16.432Z{"msg":"[1]R flush_numbers[0..12]: [41, 41, 40, 34, 40, 35, 41, 33, 40, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043649053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32842024-02-01T17:29:16.432Z{"msg":"[1]R generation[0..12]: [27, 27, 26, 22, 26, 23, 27, 21, 26, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04367453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32852024-02-01T17:29:16.435Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043717367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32862024-02-01T17:29:16.435Z{"msg":"[2]R flush_numbers[0..12]: [41, 41, 40, 34, 40, 35, 41, 33, 40, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043741513Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32872024-02-01T17:29:16.435ZWait for a query_work_queue command to finish before sending IO
32882024-02-01T17:29:16.435Z{"msg":"[2]R generation[0..12]: [27, 27, 26, 22, 26, 23, 27, 21, 26, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043787304Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32892024-02-01T17:29:16.435Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043813615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32902024-02-01T17:29:16.438Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043857394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32912024-02-01T17:29:16.438Z{"msg":"Generation requested: 30 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043881838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32922024-02-01T17:29:16.438Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043904992Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32932024-02-01T17:29:16.438Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043951297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"":"downstairs","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32942024-02-01T17:29:16.438Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.043979442Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32952024-02-01T17:29:16.440Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044001703Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"0","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32962024-02-01T17:29:16.440Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044023747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"1","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32972024-02-01T17:29:16.440Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044073542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"client":"2","":"downstairs","session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32982024-02-01T17:29:16.440Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044099425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
32992024-02-01T17:29:16.440Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044145962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
33002024-02-01T17:29:16.440Z{"msg":"4a44180b-3277-4987-b332-3f8d76d4f468 is now active with session: 1ed8fb78-504e-423e-9dfe-4b769cdc7592","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044169805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
33012024-02-01T17:29:16.440Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.04418993Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249,"session_id":"1ed8fb78-504e-423e-9dfe-4b769cdc7592"}
33022024-02-01T17:29:16.440Z{"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:15.044234114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
33032024-02-01T17:29:16.440ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
33042024-02-01T17:29:16.441ZLoading write count information from file "/tmp/test_repair_verify.data"
33052024-02-01T17:29:16.441ZRun Repair workload
33062024-02-01T17:29:16.441Z01/30 Write block 333 len 24576 data: 6 8 8 8 6 6
33072024-02-01T17:29:16.627Z02/30 Read block 322 len 16384
33082024-02-01T17:29:16.627Z03/30 Read block 533 len 12288
33092024-02-01T17:29:16.627Z04/30 Read block 444 len 20480
33102024-02-01T17:29:16.627Z05/30 Read block 562 len 4096
33112024-02-01T17:29:16.627Z06/30 Write block 401 len 4096 data: 2
33122024-02-01T17:29:16.627Z07/30 Write block 73 len 4096 data: 3
33132024-02-01T17:29:16.627Z08/30 Write block 419 len 36864 data: 3 2 2 3 3 2 3 4 3
33142024-02-01T17:29:16.627Z09/30 Read block 422 len 40960
33152024-02-01T17:29:16.627Z10/30 Write block 485 len 32768 data: 3 3 4 4 4 4 4 5
33162024-02-01T17:29:16.627Z11/30 Write block 270 len 28672 data: 3 3 3 3 2 3 3
33172024-02-01T17:29:16.628Z12/30 Read block 355 len 16384
33182024-02-01T17:29:16.628Z13/30 Read block 570 len 32768
33192024-02-01T17:29:16.628Z14/30 Read block 487 len 4096
33202024-02-01T17:29:16.628Z15/30 Write block 254 len 8192 data: 4 4
33212024-02-01T17:29:16.628Z16/30 Read block 77 len 12288
33222024-02-01T17:29:16.628Z17/30 Read block 595 len 12288
33232024-02-01T17:29:16.628Z18/30 Read block 4 len 4096
33242024-02-01T17:29:16.628Z19/30 Write block 308 len 4096 data: 3
33252024-02-01T17:29:16.628Z20/30 Read block 78 len 40960
33262024-02-01T17:29:16.628Z21/30 Read block 583 len 20480
33272024-02-01T17:29:16.628Z22/30 Flush
33282024-02-01T17:29:16.628Z23/30 Write block 166 len 32768 data: 2 2 3 4 5 6 6 6
33292024-02-01T17:29:16.628Z24/30 Write block 352 len 28672 data: 5 4 7 5 4 3 4
33302024-02-01T17:29:16.628Z25/30 Read block 472 len 28672
33312024-02-01T17:29:16.628Z26/30 Read block 489 len 32768
33322024-02-01T17:29:16.628Z27/30 Read block 370 len 4096
33332024-02-01T17:29:16.628Z28/30 Read block 315 len 4096
33342024-02-01T17:29:16.628Z29/30 Write block 74 len 8192 data: 3 3
33352024-02-01T17:29:16.628Z30/30 Write block 69 len 28672 data: 3 3 3 3 4 4 4
33362024-02-01T17:29:16.628Z----------------------------------------------------------------
33372024-02-01T17:29:16.628Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
33382024-02-01T17:29:16.629ZStates: Active Active Active
33392024-02-01T17:29:16.629ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33402024-02-01T17:29:16.629Z 1 Acked 1000 Write 6 Sent Done Done false
33412024-02-01T17:29:16.629Z 2 Acked 1001 Read 4 Sent Done Done false
33422024-02-01T17:29:16.629Z 3 Acked 1002 Read 3 Sent Done Done false
33432024-02-01T17:29:16.629Z 4 Acked 1003 Read 5 Sent Done Done false
33442024-02-01T17:29:16.629Z 5 Acked 1004 Read 1 Sent Done Done false
33452024-02-01T17:29:16.629Z 6 Acked 1005 Write 1 Sent Done Done false
33462024-02-01T17:29:16.629Z 7 Acked 1006 Write 1 Sent Done Done false
33472024-02-01T17:29:16.629Z 8 Acked 1007 Write 9 Sent Done Done false
33482024-02-01T17:29:16.629Z 9 Acked 1008 Read 10 Sent Done Done false
33492024-02-01T17:29:16.629Z 10 Acked 1009 Write 8 Sent Done Done false
33502024-02-01T17:29:16.629Z 11 Acked 1010 Write 7 Sent Done Done false
33512024-02-01T17:29:16.629Z 12 Acked 1011 Read 4 Sent Done Done false
33522024-02-01T17:29:16.629Z 13 Acked 1012 Read 8 Sent Done Done false
33532024-02-01T17:29:16.629Z 14 Acked 1013 Read 1 Sent Done Done false
33542024-02-01T17:29:16.629Z 15 Acked 1014 Write 2 Sent Done Done false
33552024-02-01T17:29:16.629Z 16 Acked 1015 Read 3 Sent Done Done false
33562024-02-01T17:29:16.629Z 17 Acked 1016 Read 3 Sent Done Done false
33572024-02-01T17:29:16.630Z 18 Acked 1017 Read 1 Sent Done Done false
33582024-02-01T17:29:16.630Z 19 Acked 1018 Write 1 Sent Done Done false
33592024-02-01T17:29:16.630Z 20 Acked 1019 Read 10 Sent Done Done false
33602024-02-01T17:29:16.630Z 21 Acked 1020 Read 5 Sent Done Done false
33612024-02-01T17:29:16.630Z 22 Acked 1021 Flush 0 Sent Done Done false
33622024-02-01T17:29:16.630Z 23 Acked 1022 Write 8 Sent Done Done false
33632024-02-01T17:29:16.630Z 24 Acked 1023 Write 7 Sent Done Done false
33642024-02-01T17:29:16.630Z 25 Acked 1024 Read 7 Sent Done Done false
33652024-02-01T17:29:16.630Z 26 Acked 1025 Read 8 Sent Done Done false
33662024-02-01T17:29:16.631Z 27 Acked 1026 Read 1 Sent Done Done false
33672024-02-01T17:29:16.631Z 28 Acked 1027 Read 1 Sent Done Done false
33682024-02-01T17:29:16.631Z 29 Acked 1028 Write 2 Sent Done Done false
33692024-02-01T17:29:16.631Z 30 Acked 1029 Write 7 Sent Sent Sent false
33702024-02-01T17:29:16.631Z STATES DS:0 DS:1 DS:2 TOTAL
33712024-02-01T17:29:16.631Z New 0 0 0 0
33722024-02-01T17:29:16.631Z Sent 30 1 1 32
33732024-02-01T17:29:16.631Z Done 0 29 29 58
33742024-02-01T17:29:16.631Z Skipped 0 0 0 0
33752024-02-01T17:29:16.631Z Error 0 0 0 0
33762024-02-01T17:29:16.631ZLast Flush: 0 1021 1021
33772024-02-01T17:29:16.631ZDownstairs last five completed:
33782024-02-01T17:29:16.631ZUpstairs last five completed: 30 29 28 27 26
33792024-02-01T17:29:16.631Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:15.086731558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1249}
33802024-02-01T17:29:16.631ZWrote out file "/tmp/test_repair_verify.data"
33812024-02-01T17:29:16.631Z
33822024-02-01T17:29:16.631ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33832024-02-01T17:29:16.631ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33842024-02-01T17:29:16.631Z 0 000-019 27 27 27 41 41 41 F F F
33852024-02-01T17:29:16.632Z 1 020-039 27 27 27 41 41 41 F F F
33862024-02-01T17:29:16.633Z 2 040-059 26 26 26 40 40 40 F F F
33872024-02-01T17:29:16.633Z 3 060-079 22 30 30 34 42 42 F T T <---
33882024-02-01T17:29:16.633Z 4 080-099 26 26 26 40 40 40 F F F
33892024-02-01T17:29:16.633Z 5 100-119 23 23 23 35 35 35 F F F
33902024-02-01T17:29:16.633Z 6 120-139 27 27 27 41 41 41 F F F
33912024-02-01T17:29:16.633Z 7 140-159 21 21 21 33 33 33 F F F
33922024-02-01T17:29:16.633Z 8 160-179 26 26 26 40 40 40 F T T <---
33932024-02-01T17:29:16.633Z 9 180-199 27 27 27 41 41 41 F F F
33942024-02-01T17:29:16.633Z 10 200-219 27 27 27 41 41 41 F F F
33952024-02-01T17:29:16.633Z 11 220-239 9 9 9 13 13 13 F F F
33962024-02-01T17:29:16.633Z 12 240-259 26 30 30 40 42 42 F F F <---
33972024-02-01T17:29:16.633Z 13 260-279 24 30 30 37 42 42 F F F <---
33982024-02-01T17:29:16.633Z 14 280-299 27 27 27 41 41 41 F F F
33992024-02-01T17:29:16.633Z 15 300-319 27 30 30 41 42 42 F F F <---
34002024-02-01T17:29:16.633Z 16 320-339 27 30 30 41 42 42 F F F <---
34012024-02-01T17:29:16.633Z 17 340-359 27 27 27 41 41 41 F T T <---
34022024-02-01T17:29:16.633Z 18 360-379 27 27 27 41 41 41 F F F
34032024-02-01T17:29:16.633Z 19 380-399 26 26 26 40 40 40 F F F
34042024-02-01T17:29:16.634Z 20 400-419 14 30 30 21 42 42 F F F <---
34052024-02-01T17:29:16.634Z 21 420-439 27 30 30 41 42 42 F F F <---
34062024-02-01T17:29:16.634Z 22 440-459 26 26 26 40 40 40 F F F
34072024-02-01T17:29:16.656Z 23 460-479 27 27 27 41 41 41 F F F
34082024-02-01T17:29:16.656Z 24 480-499 20 30 30 31 42 42 F F F <---
34092024-02-01T17:29:16.656Z 25 500-519 10 10 10 14 14 14 F F F
34102024-02-01T17:29:16.657Z 26 520-539 26 26 26 40 40 40 F F F
34112024-02-01T17:29:16.657Z 27 540-559 27 27 27 41 41 41 F F F
34122024-02-01T17:29:16.657Z 28 560-579 27 27 27 41 41 41 F F F
34132024-02-01T17:29:16.657Z 29 580-599 27 27 27 41 41 41 F F F
34142024-02-01T17:29:16.657ZMax gen: 30, Max flush: 42
34152024-02-01T17:29:16.657ZError: Difference in extent metadata found!
34162024-02-01T17:29:16.657ZA Difference in extent metadata is expected here
34172024-02-01T17:29:16.657ZOn loop 15
34182024-02-01T17:29:16.657Z
34192024-02-01T17:29:16.657ZVerifying data now
34202024-02-01T17:29:20.755ZNo change after verify
34212024-02-01T17:29:20.755ZLoop: 15 Downstairs dump after verify (and repair):
34222024-02-01T17:29:20.778ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34232024-02-01T17:29:20.778Z 0 000-019 27 27 27 41 41 41 F F F
34242024-02-01T17:29:20.781Z 1 020-039 27 27 27 41 41 41 F F F
34252024-02-01T17:29:20.781Z 2 040-059 26 26 26 40 40 40 F F F
34262024-02-01T17:29:20.781Z 3 060-079 31 31 31 43 43 43 F F F
34272024-02-01T17:29:20.781Z 4 080-099 26 26 26 40 40 40 F F F
34282024-02-01T17:29:20.781Z 5 100-119 23 23 23 35 35 35 F F F
34292024-02-01T17:29:20.781Z 6 120-139 27 27 27 41 41 41 F F F
34302024-02-01T17:29:20.781Z 7 140-159 21 21 21 33 33 33 F F F
34312024-02-01T17:29:20.781Z 8 160-179 31 31 31 43 43 43 F F F
34322024-02-01T17:29:20.781Z 9 180-199 27 27 27 41 41 41 F F F
34332024-02-01T17:29:20.782Z 10 200-219 27 27 27 41 41 41 F F F
34342024-02-01T17:29:20.782Z 11 220-239 9 9 9 13 13 13 F F F
34352024-02-01T17:29:20.782Z 12 240-259 30 30 30 42 42 42 F F F
34362024-02-01T17:29:20.782Z 13 260-279 30 30 30 42 42 42 F F F
34372024-02-01T17:29:20.782Z 14 280-299 27 27 27 41 41 41 F F F
34382024-02-01T17:29:20.782Z 15 300-319 30 30 30 42 42 42 F F F
34392024-02-01T17:29:20.782Z 16 320-339 30 30 30 42 42 42 F F F
34402024-02-01T17:29:20.782Z 17 340-359 31 31 31 43 43 43 F F F
34412024-02-01T17:29:20.782Z 18 360-379 27 27 27 41 41 41 F F F
34422024-02-01T17:29:20.782Z 19 380-399 26 26 26 40 40 40 F F F
34432024-02-01T17:29:20.782Z 20 400-419 30 30 30 42 42 42 F F F
34442024-02-01T17:29:20.782Z 21 420-439 30 30 30 42 42 42 F F F
34452024-02-01T17:29:20.782Z 22 440-459 26 26 26 40 40 40 F F F
34462024-02-01T17:29:20.782Z 23 460-479 27 27 27 41 41 41 F F F
34472024-02-01T17:29:20.782Z 24 480-499 30 30 30 42 42 42 F F F
34482024-02-01T17:29:20.782Z 25 500-519 10 10 10 14 14 14 F F F
34492024-02-01T17:29:20.782Z 26 520-539 26 26 26 40 40 40 F F F
34502024-02-01T17:29:20.782Z 27 540-559 27 27 27 41 41 41 F F F
34512024-02-01T17:29:20.782Z 28 560-579 27 27 27 41 41 41 F F F
34522024-02-01T17:29:20.782Z 29 580-599 27 27 27 41 41 41 F F F
34532024-02-01T17:29:20.783ZMax gen: 31, Max flush: 43
34542024-02-01T17:29:20.789Z
34552024-02-01T17:29:20.789ZBegin loop 16
34562024-02-01T17:29:20.789ZDownstairs to restart: 0
34572024-02-01T17:29:20.813Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.075687932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257}
34582024-02-01T17:29:20.824Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076274671Z","hostname":"Crucible runtime is spawned
34592024-02-01T17:29:20.824Zip-10-150-1-139.us-west-2.compute.internal","pid":1257}
34602024-02-01T17:29:20.824Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076406664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257}
34612024-02-01T17:29:20.824Z{"msg":"Crucible 86a8266c-5f57-47bf-a7a2-cd169e65b955 has session id: 5d54fd92-2267-4f40-9984-c96789e2586b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076450746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34622024-02-01T17:29:20.824Z{"msg":"Upstairs opts: Upstairs UUID: 86a8266c-5f57-47bf-a7a2-cd169e65b955, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076527959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34632024-02-01T17:29:20.824Z{"msg":"Crucible stats registered with UUID: 86a8266c-5f57-47bf-a7a2-cd169e65b955","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076571985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34642024-02-01T17:29:20.824Z{"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076641859Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257}
34652024-02-01T17:29:20.824Z{"msg":"86a8266c-5f57-47bf-a7a2-cd169e65b955 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076678659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34662024-02-01T17:29:20.824Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076753976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34672024-02-01T17:29:20.824Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076865209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34682024-02-01T17:29:20.825Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.076955061Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34692024-02-01T17:29:20.826Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.07700352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34702024-02-01T17:29:20.826Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077086327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34712024-02-01T17:29:20.826Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077129308Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34722024-02-01T17:29:20.826Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077169631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34732024-02-01T17:29:20.826Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077213795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34742024-02-01T17:29:20.826Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.07725792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34752024-02-01T17:29:20.826Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077363618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34762024-02-01T17:29:20.826Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.077412214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34772024-02-01T17:29:20.826Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.07750837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"io task","client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34782024-02-01T17:29:20.826Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.078493074Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34792024-02-01T17:29:20.827Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.078538833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34802024-02-01T17:29:20.827Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.078563763Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34812024-02-01T17:29:20.827Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.078714601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34822024-02-01T17:29:20.828Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.079402698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34832024-02-01T17:29:20.828Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.079465796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34842024-02-01T17:29:20.828Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.079494663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34852024-02-01T17:29:20.828Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.079612341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34862024-02-01T17:29:20.828Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080432532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34872024-02-01T17:29:20.828Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080550373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34882024-02-01T17:29:20.828Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080578831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34892024-02-01T17:29:20.828Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080630169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34902024-02-01T17:29:20.828Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080655447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34912024-02-01T17:29:20.828Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080678211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34922024-02-01T17:29:20.828Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.080700944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34932024-02-01T17:29:20.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081381118Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34942024-02-01T17:29:20.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081442353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34952024-02-01T17:29:20.834Z{"msg":"[0]R flush_numbers[0..12]: [41, 41, 40, 43, 40, 35, 41, 33, 43, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081470183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34962024-02-01T17:29:20.834Z{"msg":"[0]R generation[0..12]: [27, 27, 26, 31, 26, 23, 27, 21, 31, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.08151326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34972024-02-01T17:29:20.835Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081540208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34982024-02-01T17:29:20.835Z{"msg":"[1]R flush_numbers[0..12]: [41, 41, 40, 43, 40, 35, 41, 33, 43, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081581968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
34992024-02-01T17:29:20.837Z{"msg":"[1]R generation[0..12]: [27, 27, 26, 31, 26, 23, 27, 21, 31, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081606918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35002024-02-01T17:29:20.837Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
35012024-02-01T17:29:20.837Z,"time":"2024-02-01T17:29:20.08164946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35022024-02-01T17:29:20.837Z{"msg":"[2]R flush_numbers[0..12]: [41, 41, 40, 43, 40, 35, 41, 33, 43, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081675421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35032024-02-01T17:29:20.837Z{"msg":"[2]R generation[0..12]: [27, 27, 26, 31, 26, 23, 27, 21, 31, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081698807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35042024-02-01T17:29:20.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.08172333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35052024-02-01T17:29:20.837Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081745161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35062024-02-01T17:29:20.837Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081767186Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35072024-02-01T17:29:20.838Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081808286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35082024-02-01T17:29:20.838Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.08183233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"":"downstairs","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35092024-02-01T17:29:20.838Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081874048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35102024-02-01T17:29:20.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081896014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"0","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35112024-02-01T17:29:20.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.08193651Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"1","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35122024-02-01T17:29:20.838Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081960434Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"client":"2","":"downstairs","session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35132024-02-01T17:29:20.838Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.081982912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35142024-02-01T17:29:20.838Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.082032114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35152024-02-01T17:29:20.838Z{"msg":"86a8266c-5f57-47bf-a7a2-cd169e65b955 is now active with session: 5d54fd92-2267-4f40-9984-c96789e2586b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.082055687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35162024-02-01T17:29:20.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.082075704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257,"session_id":"5d54fd92-2267-4f40-9984-c96789e2586b"}
35172024-02-01T17:29:20.839Z{"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:20.082114797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257}
35182024-02-01T17:29:20.839ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35192024-02-01T17:29:20.839ZLoading write count information from file "/tmp/test_repair_verify.data"
35202024-02-01T17:29:20.839ZRun Repair workload
35212024-02-01T17:29:20.839Z01/30 Write block 466 len 12288 data: 5 5 4
35222024-02-01T17:29:20.839Z02/30 Read block 436 len 24576
35232024-02-01T17:29:20.839Z03/30 Read block 353 len 28672
35242024-02-01T17:29:20.839Z04/30 Read block 389 len 36864
35252024-02-01T17:29:20.839Z05/30 Read block 491 len 28672
35262024-02-01T17:29:20.839Z06/30 Write block 295 len 4096 data: 5
35272024-02-01T17:29:20.839Z07/30 Write block 276 len 4096 data: 4
35282024-02-01T17:29:20.839Z08/30 Read block 226 len 4096
35292024-02-01T17:29:20.839Z09/30 Write block 176 len 12288 data: 8 8 5
35302024-02-01T17:29:20.839Z10/30 Write block 292 len 28672 data: 4 5 6 6 6 5 6
35312024-02-01T17:29:20.839Z11/30 Flush
35322024-02-01T17:29:20.847Z12/30 Write block 530 len 4096 data: 4
35332024-02-01T17:29:20.847Z13/30 Read block 76 len 40960
35342024-02-01T17:29:20.847Z14/30 Flush
35352024-02-01T17:29:20.847Z15/30 Write block 86 len 28672 data: 3 2 2 2 2 2 3
35362024-02-01T17:29:20.847Z16/30 Read block 23 len 12288
35372024-02-01T17:29:20.847Z17/30 Read block 518 len 16384
35382024-02-01T17:29:20.847Z18/30 Flush
35392024-02-01T17:29:20.852Z19/30 Write block 402 len 32768 data: 2 2 2 2 2 2 2 2
35402024-02-01T17:29:20.852Z20/30 Write block 17 len 36864 data: 4 4 3 3 3 4 4 4 4
35412024-02-01T17:29:20.852Z21/30 Read block 485 len 12288
35422024-02-01T17:29:20.858Z22/30 Read block 446 len 12288
35432024-02-01T17:29:20.858Z23/30 Write block 267 len 12288 data: 2 2 3
35442024-02-01T17:29:20.858Z24/30 Read block 237 len 36864
35452024-02-01T17:29:20.858Z25/30 Read block 291 len 4096
35462024-02-01T17:29:20.858Z26/30 Read block 538 len 24576
35472024-02-01T17:29:20.865Z27/30 Write block 506 len 40960 data: 2 2 2 2 2 2 2 2 2 2
35482024-02-01T17:29:20.865Z28/30 Write block 357 len 24576 data: 4 5 5 4 5 5
35492024-02-01T17:29:20.865Z29/30 Read block 222 len 4096
35502024-02-01T17:29:20.865Z30/30 Read block 180 len 20480
35512024-02-01T17:29:20.872Z----------------------------------------------------------------
35522024-02-01T17:29:20.872Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
35532024-02-01T17:29:20.873ZStates: Active Active Active
35542024-02-01T17:29:20.873ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35552024-02-01T17:29:20.873Z 1 Acked 1000 Write 3 Sent Done Done false
35562024-02-01T17:29:20.874Z 2 Acked 1001 Read 6 Sent Done Done false
35572024-02-01T17:29:20.874Z 3 Acked 1002 Read 7 Sent Done Done false
35582024-02-01T17:29:20.874Z 4 Acked 1003 Read 9 Sent Done Done false
35592024-02-01T17:29:20.874Z 5 Acked 1004 Read 7 Sent Done Done false
35602024-02-01T17:29:20.874Z 6 Acked 1005 Write 1 Sent Done Done false
35612024-02-01T17:29:20.874Z 7 Acked 1006 Write 1 Sent Done Done false
35622024-02-01T17:29:20.875Z 8 Acked 1007 Read 1 Sent Done Done false
35632024-02-01T17:29:20.875Z 9 Acked 1008 Write 3 Sent Done Done false
35642024-02-01T17:29:20.875Z 10 Acked 1009 Write 7 Sent Done Done false
35652024-02-01T17:29:20.875Z 11 Acked 1010 Flush 0 Sent Done Done false
35662024-02-01T17:29:20.876Z 12 Acked 1011 Write 1 Sent Done Done false
35672024-02-01T17:29:20.876Z 13 Acked 1012 Read 10 Sent Done Done false
35682024-02-01T17:29:20.876Z 14 Acked 1013 Flush 0 Sent Done Done false
35692024-02-01T17:29:20.876Z 15 Acked 1014 Write 7 Sent Done Done false
35702024-02-01T17:29:20.876Z 16 Acked 1015 Read 3 Sent Done Done false
35712024-02-01T17:29:20.878Z 17 Acked 1016 Read 4 Sent Done Done false
35722024-02-01T17:29:20.878Z 18 Acked 1017 Flush 0 Sent Done Done false
35732024-02-01T17:29:20.879Z 19 Acked 1018 Write 8 Sent Done Done false
35742024-02-01T17:29:20.879Z 20 Acked 1019 Write 9 Sent Done Done false
35752024-02-01T17:29:20.879Z 21 Acked 1020 Read 3 Sent Done Done false
35762024-02-01T17:29:20.879Z 22 Acked 1021 Read 3 Sent Done Done false
35772024-02-01T17:29:20.879Z 23 Acked 1022 Write 3 Sent Done Done false
35782024-02-01T17:29:20.881Z 24 Acked 1023 Read 9 Sent Done Done false
35792024-02-01T17:29:20.881Z 25 Acked 1024 Read 1 Sent Done Done false
35802024-02-01T17:29:20.881Z 26 Acked 1025 Read 6 Sent Done Done false
35812024-02-01T17:29:20.881Z 27 Acked 1026 Write 10 Sent Done Done false
35822024-02-01T17:29:20.881Z 28 Acked 1027 Write 6 Sent Done Done false
35832024-02-01T17:29:20.881Z 29 Acked 1028 Read 1 Sent Done Done false
35842024-02-01T17:29:20.881Z 30 Acked 1029 Read 5 Sent Sent Done false
35852024-02-01T17:29:20.881Z STATES DS:0 DS:1 DS:2 TOTAL
35862024-02-01T17:29:20.881Z New 0 0 0 0
35872024-02-01T17:29:20.882Z Sent 30 1 0 31
35882024-02-01T17:29:20.882Z Done 0 29 30 59
35892024-02-01T17:29:20.882Z Skipped 0 0 0 0
35902024-02-01T17:29:20.882Z Error 0 0 0 0
35912024-02-01T17:29:20.882ZLast Flush: 0 1017 1017
35922024-02-01T17:29:20.882ZDownstairs last five completed:
35932024-02-01T17:29:20.882ZUpstairs last five completed: 30 29 28 27 26
35942024-02-01T17:29:20.882Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:20.128074387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1257}
35952024-02-01T17:29:20.883ZWrote out file "/tmp/test_repair_verify.data"
35962024-02-01T17:29:20.893Z
35972024-02-01T17:29:20.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35982024-02-01T17:29:20.900ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35992024-02-01T17:29:20.900Z 0 000-019 27 27 27 41 41 41 F T T <---
36002024-02-01T17:29:20.900Z 1 020-039 27 27 27 41 41 41 F T T <---
36012024-02-01T17:29:20.900Z 2 040-059 26 26 26 40 40 40 F F F
36022024-02-01T17:29:20.900Z 3 060-079 31 31 31 43 43 43 F F F
36032024-02-01T17:29:20.904Z 4 080-099 26 32 32 40 46 46 F F F <---
36042024-02-01T17:29:20.904Z 5 100-119 23 23 23 35 35 35 F F F
36052024-02-01T17:29:20.904Z 6 120-139 27 27 27 41 41 41 F F F
36062024-02-01T17:29:20.904Z 7 140-159 21 21 21 33 33 33 F F F
36072024-02-01T17:29:20.904Z 8 160-179 31 32 32 43 44 44 F F F <---
36082024-02-01T17:29:20.904Z 9 180-199 27 27 27 41 41 41 F F F
36092024-02-01T17:29:20.904Z 10 200-219 27 27 27 41 41 41 F F F
36102024-02-01T17:29:20.904Z 11 220-239 9 9 9 13 13 13 F F F
36112024-02-01T17:29:20.904Z 12 240-259 30 30 30 42 42 42 F F F
36122024-02-01T17:29:20.904Z 13 260-279 30 32 32 42 44 44 F T T <---
36132024-02-01T17:29:20.904Z 14 280-299 27 32 32 41 44 44 F F F <---
36142024-02-01T17:29:20.904Z 15 300-319 30 30 30 42 42 42 F F F
36152024-02-01T17:29:20.904Z 16 320-339 30 30 30 42 42 42 F F F
36162024-02-01T17:29:20.905Z 17 340-359 31 31 31 43 43 43 F T T <---
36172024-02-01T17:29:20.905Z 18 360-379 27 27 27 41 41 41 F T T <---
36182024-02-01T17:29:20.905Z 19 380-399 26 26 26 40 40 40 F F F
36192024-02-01T17:29:20.906Z 20 400-419 30 30 30 42 42 42 F T T <---
36202024-02-01T17:29:20.906Z 21 420-439 30 30 30 42 42 42 F F F
36212024-02-01T17:29:20.906Z 22 440-459 26 26 26 40 40 40 F F F
36222024-02-01T17:29:20.906Z 23 460-479 27 32 32 41 44 44 F F F <---
36232024-02-01T17:29:20.906Z 24 480-499 30 30 30 42 42 42 F F F
36242024-02-01T17:29:20.906Z 25 500-519 10 10 10 14 14 14 F T T <---
36252024-02-01T17:29:20.906Z 26 520-539 26 32 32 40 45 45 F F F <---
36262024-02-01T17:29:20.906Z 27 540-559 27 27 27 41 41 41 F F F
36272024-02-01T17:29:20.906Z 28 560-579 27 27 27 41 41 41 F F F
36282024-02-01T17:29:20.906Z 29 580-599 27 27 27 41 41 41 F F F
36292024-02-01T17:29:20.906ZMax gen: 32, Max flush: 46
36302024-02-01T17:29:20.906ZError: Difference in extent metadata found!
36312024-02-01T17:29:20.910ZA Difference in extent metadata is expected here
36322024-02-01T17:29:20.910ZOn loop 16
36332024-02-01T17:29:20.910Z
36342024-02-01T17:29:20.910ZVerifying data now
36352024-02-01T17:29:25.968ZNo change after verify
36362024-02-01T17:29:25.968ZLoop: 16 Downstairs dump after verify (and repair):
36372024-02-01T17:29:25.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36382024-02-01T17:29:25.987Z 0 000-019 33 33 33 47 47 47 F F F
36392024-02-01T17:29:25.990Z 1 020-039 33 33 33 47 47 47 F F F
36402024-02-01T17:29:25.990Z 2 040-059 26 26 26 40 40 40 F F F
36412024-02-01T17:29:25.990Z 3 060-079 31 31 31 43 43 43 F F F
36422024-02-01T17:29:25.991Z 4 080-099 32 32 32 46 46 46 F F F
36432024-02-01T17:29:25.991Z 5 100-119 23 23 23 35 35 35 F F F
36442024-02-01T17:29:25.991Z 6 120-139 27 27 27 41 41 41 F F F
36452024-02-01T17:29:25.991Z 7 140-159 21 21 21 33 33 33 F F F
36462024-02-01T17:29:25.991Z 8 160-179 32 32 32 44 44 44 F F F
36472024-02-01T17:29:25.991Z 9 180-199 27 27 27 41 41 41 F F F
36482024-02-01T17:29:25.991Z 10 200-219 27 27 27 41 41 41 F F F
36492024-02-01T17:29:25.991Z 11 220-239 9 9 9 13 13 13 F F F
36502024-02-01T17:29:25.991Z 12 240-259 30 30 30 42 42 42 F F F
36512024-02-01T17:29:25.991Z 13 260-279 33 33 33 47 47 47 F F F
36522024-02-01T17:29:25.991Z 14 280-299 32 32 32 44 44 44 F F F
36532024-02-01T17:29:25.991Z 15 300-319 30 30 30 42 42 42 F F F
36542024-02-01T17:29:25.991Z 16 320-339 30 30 30 42 42 42 F F F
36552024-02-01T17:29:25.991Z 17 340-359 33 33 33 47 47 47 F F F
36562024-02-01T17:29:25.991Z 18 360-379 33 33 33 47 47 47 F F F
36572024-02-01T17:29:25.991Z 19 380-399 26 26 26 40 40 40 F F F
36582024-02-01T17:29:25.991Z 20 400-419 33 33 33 47 47 47 F F F
36592024-02-01T17:29:25.991Z 21 420-439 30 30 30 42 42 42 F F F
36602024-02-01T17:29:25.991Z 22 440-459 26 26 26 40 40 40 F F F
36612024-02-01T17:29:25.991Z 23 460-479 32 32 32 44 44 44 F F F
36622024-02-01T17:29:25.991Z 24 480-499 30 30 30 42 42 42 F F F
36632024-02-01T17:29:25.991Z 25 500-519 33 33 33 47 47 47 F F F
36642024-02-01T17:29:25.992Z 26 520-539 32 32 32 45 45 45 F F F
36652024-02-01T17:29:25.992Z 27 540-559 27 27 27 41 41 41 F F F
36662024-02-01T17:29:25.992Z 28 560-579 27 27 27 41 41 41 F F F
36672024-02-01T17:29:25.992Z 29 580-599 27 27 27 41 41 41 F F F
36682024-02-01T17:29:25.992ZMax gen: 33, Max flush: 47
36692024-02-01T17:29:25.995Z
36702024-02-01T17:29:25.995ZBegin loop 17
36712024-02-01T17:29:25.995ZDownstairs to restart: 1
36722024-02-01T17:29:26.024Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.287446058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
36732024-02-01T17:29:26.028ZCrucible runtime is spawned
36742024-02-01T17:29:26.028Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288226044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
36752024-02-01T17:29:26.028Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288312898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
36762024-02-01T17:29:26.029Z{"msg":"Crucible d7846648-a2f9-4abe-9393-b459f92e8ef9 has session id: e589f4b3-d5f0-45a2-9abd-297b977103e7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288344375Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36772024-02-01T17:29:26.029Z{"msg":"Upstairs opts: Upstairs UUID: d7846648-a2f9-4abe-9393-b459f92e8ef9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288397056Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36782024-02-01T17:29:26.029Z{"msg":"Crucible stats registered with UUID: d7846648-a2f9-4abe-9393-b459f92e8ef9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288447972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36792024-02-01T17:29:26.029Z{"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.28849375Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
36802024-02-01T17:29:26.030Z{"msg":"d7846648-a2f9-4abe-9393-b459f92e8ef9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288524569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36812024-02-01T17:29:26.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288581259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36822024-02-01T17:29:26.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288662139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36832024-02-01T17:29:26.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288744562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36842024-02-01T17:29:26.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288792587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36852024-02-01T17:29:26.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288864298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36862024-02-01T17:29:26.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.288911464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36872024-02-01T17:29:26.030Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289009094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36882024-02-01T17:29:26.030Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289055286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36892024-02-01T17:29:26.030Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289104087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36902024-02-01T17:29:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289149682Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36912024-02-01T17:29:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289368902Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36922024-02-01T17:29:26.032Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.289467499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"io task","client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36932024-02-01T17:29:26.032Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.290132046Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36942024-02-01T17:29:26.032Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.290255698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36952024-02-01T17:29:26.032Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.290343081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36962024-02-01T17:29:26.032Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.290411641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36972024-02-01T17:29:26.032Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.290931595Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36982024-02-01T17:29:26.032Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.291056831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
36992024-02-01T17:29:26.032Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.291307101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37002024-02-01T17:29:26.033Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.291391762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37012024-02-01T17:29:26.037Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292005792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37022024-02-01T17:29:26.037Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292375325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37032024-02-01T17:29:26.037Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292462219Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37042024-02-01T17:29:26.038Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292509499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37052024-02-01T17:29:26.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292579402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37062024-02-01T17:29:26.038Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292652168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37072024-02-01T17:29:26.038Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.292723671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37082024-02-01T17:29:26.038Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.29300707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37092024-02-01T17:29:26.038Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293076068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37102024-02-01T17:29:26.038Z{"msg":"[0]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 35, 41, 33, 44, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293109698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37112024-02-01T17:29:26.038Z{"msg":"[0]R generation[0..12]: [33, 33, 26, 31, 32, 23, 27, 21, 32, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293170796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37122024-02-01T17:29:26.038Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293230443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37132024-02-01T17:29:26.038Z{"msg":"[1]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 35, 41, 33, 44, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293289987Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37142024-02-01T17:29:26.038Z{"msg":"[1]R generation[0..12]: [33, 33, 26, 31, 32, 23, 27, 21, 32, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293363136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37152024-02-01T17:29:26.038Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293398295Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid"Wait for a query_work_queue command to finish before sending IO
37162024-02-01T17:29:26.038Z:1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37172024-02-01T17:29:26.038Z{"msg":"[2]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 35, 41, 33, 44, 41, 41, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293463092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37182024-02-01T17:29:26.038Z{"msg":"[2]R generation[0..12]: [33, 33, 26, 31, 32, 23, 27, 21, 32, 27, 27, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293495881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37192024-02-01T17:29:26.038Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293550275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37202024-02-01T17:29:26.038Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293580874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37212024-02-01T17:29:26.038Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293634817Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37222024-02-01T17:29:26.038Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293659981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37232024-02-01T17:29:26.038Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293734142Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"":"downstairs","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37242024-02-01T17:29:26.038Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293777801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37252024-02-01T17:29:26.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293814422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"0","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37262024-02-01T17:29:26.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293856009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"1","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37272024-02-01T17:29:26.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293920247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"client":"2","":"downstairs","session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37282024-02-01T17:29:26.039Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.293962955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37292024-02-01T17:29:26.039Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.294020204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37302024-02-01T17:29:26.040Z{"msg":"d7846648-a2f9-4abe-9393-b459f92e8ef9 is now active with session: e589f4b3-d5f0-45a2-9abd-297b977103e7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.294061862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37312024-02-01T17:29:26.040Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.294112013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265,"session_id":"e589f4b3-d5f0-45a2-9abd-297b977103e7"}
37322024-02-01T17:29:26.040Z{"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:25.294151058Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37332024-02-01T17:29:26.040Zhostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
37342024-02-01T17:29:26.040ZLoading write count information from file "/tmp/test_repair_verify.data"
37352024-02-01T17:29:26.040ZRun Repair workload
37362024-02-01T17:29:26.040Z01/30 Flush
37372024-02-01T17:29:26.040Z02/30 Write block 438 len 24576 data: 6 6 5 5 5 4
37382024-02-01T17:29:26.040Z03/30 Read block 326 len 16384
37392024-02-01T17:29:26.043Z04/30 Write block 208 len 32768 data: 7 5 2 3 2 2 2 2
37402024-02-01T17:29:26.044Z05/30 Read block 193 len 32768
37412024-02-01T17:29:26.044Z06/30 Read block 155 len 40960
37422024-02-01T17:29:26.044Z07/30 Write block 464 len 4096 data: 3
37432024-02-01T17:29:26.044Z08/30 Read block 57 len 24576
37442024-02-01T17:29:26.047Z09/30 Read block 379 len 16384
37452024-02-01T17:29:26.047Z10/30 Write block 456 len 16384 data: 4 4 4 5
37462024-02-01T17:29:26.047Z11/30 Write block 206 len 24576 data: 8 8 8 6 3 4
37472024-02-01T17:29:26.050Z12/30 Read block 150 len 4096
37482024-02-01T17:29:26.050Z13/30 Write block 122 len 16384 data: 5 4 4 4
37492024-02-01T17:29:26.050Z14/30 Read block 424 len 8192
37502024-02-01T17:29:26.053Z15/30 Write block 479 len 32768 data: 3 3 3 3 4 4 4 4
37512024-02-01T17:29:26.053Z16/30 Write block 160 len 24576 data: 3 4 4 3 3 3
37522024-02-01T17:29:26.056Z17/30 Read block 590 len 12288
37532024-02-01T17:29:26.056Z18/30 Write block 370 len 24576 data: 3 2 2 3 4 5
37542024-02-01T17:29:26.059Z19/30 Read block 53 len 24576
37552024-02-01T17:29:26.062Z20/30 Write block 111 len 36864 data: 3 3 4 4 4 3 3 3 4
37562024-02-01T17:29:26.062Z21/30 Read block 294 len 12288
37572024-02-01T17:29:26.062Z22/30 Read block 219 len 12288
37582024-02-01T17:29:26.065Z23/30 Read block 235 len 36864
37592024-02-01T17:29:26.065Z24/30 Read block 211 len 28672
37602024-02-01T17:29:26.068Z25/30 Read block 301 len 32768
37612024-02-01T17:29:26.071Z26/30 Write block 588 len 40960 data: 4 5 5 4 3 3 3 3 2 3
37622024-02-01T17:29:26.071Z27/30 Read block 236 len 36864
37632024-02-01T17:29:26.074Z28/30 Read block 26 len 24576
37642024-02-01T17:29:26.074Z29/30 Write block 537 len 16384 data: 4 3 3 5
37652024-02-01T17:29:26.077Z30/30 Write block 128 len 16384 data: 3 3 3 3
37662024-02-01T17:29:26.077Z----------------------------------------------------------------
37672024-02-01T17:29:26.077Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:29
37682024-02-01T17:29:26.077ZStates: Active Active Active
37692024-02-01T17:29:26.078ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37702024-02-01T17:29:26.078Z 2 Acked 1001 Write 6 Done Done Done false
37712024-02-01T17:29:26.078Z 3 Acked 1002 Read 4 Done Done Done false
37722024-02-01T17:29:26.078Z 4 Acked 1003 Write 8 Done Sent Done false
37732024-02-01T17:29:26.078Z 5 Acked 1004 Read 8 Done Sent Done false
37742024-02-01T17:29:26.078Z 6 Acked 1005 Read 10 Done Sent Done false
37752024-02-01T17:29:26.079Z 7 Acked 1006 Write 1 Done Sent Done false
37762024-02-01T17:29:26.079Z 8 Acked 1007 Read 6 Done Sent Done false
37772024-02-01T17:29:26.079Z 9 Acked 1008 Read 4 Done Sent Done false
37782024-02-01T17:29:26.079Z 10 Acked 1009 Write 4 Done Sent Done false
37792024-02-01T17:29:26.079Z 11 Acked 1010 Write 6 Done Sent Done false
37802024-02-01T17:29:26.079Z 12 Acked 1011 Read 1 Done Sent Done false
37812024-02-01T17:29:26.079Z 13 Acked 1012 Write 4 Done Sent Done false
37822024-02-01T17:29:26.079Z 14 Acked 1013 Read 2 Done Sent Done false
37832024-02-01T17:29:26.079Z 15 Acked 1014 Write 8 Done Sent Done false
37842024-02-01T17:29:26.079Z 16 Acked 1015 Write 6 Done Sent Done false
37852024-02-01T17:29:26.080Z 17 Acked 1016 Read 3 Done Sent Done false
37862024-02-01T17:29:26.080Z 18 Acked 1017 Write 6 Done Sent Done false
37872024-02-01T17:29:26.080Z 19 Acked 1018 Read 6 Done Sent Done false
37882024-02-01T17:29:26.080Z 20 Acked 1019 Write 9 Done Sent Done false
37892024-02-01T17:29:26.080Z 21 Acked 1020 Read 3 Done Sent Done false
37902024-02-01T17:29:26.080Z 22 Acked 1021 Read 3 Done Sent Done false
37912024-02-01T17:29:26.080Z 23 Acked 1022 Read 9 Done Sent Done false
37922024-02-01T17:29:26.080Z 24 Acked 1023 Read 7 Done Sent Done false
37932024-02-01T17:29:26.080Z 25 Acked 1024 Read 8 Done Sent Done false
37942024-02-01T17:29:26.080Z 26 Acked 1025 Write 10 Done Sent Done false
37952024-02-01T17:29:26.080Z 27 Acked 1026 Read 9 Done Sent Done false
37962024-02-01T17:29:26.080Z 28 Acked 1027 Read 6 Done Sent Done false
37972024-02-01T17:29:26.080Z 29 Acked 1028 Write 4 Done Sent Done false
37982024-02-01T17:29:26.080Z 30 Acked 1029 Write 4 Sent Sent Sent false
37992024-02-01T17:29:26.080Z STATES DS:0 DS:1 DS:2 TOTAL
38002024-02-01T17:29:26.080Z New 0 0 0 0
38012024-02-01T17:29:26.080Z Sent 1 27 1 29
38022024-02-01T17:29:26.080Z Done 28 2 28 58
38032024-02-01T17:29:26.080Z Skipped 0 0 0 0
38042024-02-01T17:29:26.080Z Error 0 0 0 0
38052024-02-01T17:29:26.080ZLast Flush: 1000 1000 1000
38062024-02-01T17:29:26.080ZDownstairs last five completed: 1000
38072024-02-01T17:29:26.080ZUpstairs last five completed: 30 29 28 27 26
38082024-02-01T17:29:26.080Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:25.340189654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1265}
38092024-02-01T17:29:26.080ZWrote out file "/tmp/test_repair_verify.data"
38102024-02-01T17:29:26.084Z
38112024-02-01T17:29:26.087ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38122024-02-01T17:29:26.117ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38132024-02-01T17:29:26.117Z 0 000-019 33 33 33 47 47 47 F F F
38142024-02-01T17:29:26.117Z 1 020-039 33 33 33 47 47 47 F F F
38152024-02-01T17:29:26.117Z 2 040-059 26 26 26 40 40 40 F F F
38162024-02-01T17:29:26.117Z 3 060-079 31 31 31 43 43 43 F F F
38172024-02-01T17:29:26.117Z 4 080-099 32 32 32 46 46 46 F F F
38182024-02-01T17:29:26.117Z 5 100-119 23 23 23 35 35 35 T F T <---
38192024-02-01T17:29:26.123Z 6 120-139 27 27 27 41 41 41 T F T <---
38202024-02-01T17:29:26.123Z 7 140-159 21 21 21 33 33 33 F F F
38212024-02-01T17:29:26.123Z 8 160-179 32 32 32 44 44 44 T F T <---
38222024-02-01T17:29:26.123Z 9 180-199 27 27 27 41 41 41 F F F
38232024-02-01T17:29:26.123Z 10 200-219 27 27 27 41 41 41 T F T <---
38242024-02-01T17:29:26.123Z 11 220-239 9 9 9 13 13 13 F F F
38252024-02-01T17:29:26.123Z 12 240-259 30 30 30 42 42 42 F F F
38262024-02-01T17:29:26.123Z 13 260-279 33 33 33 47 47 47 F F F
38272024-02-01T17:29:26.123Z 14 280-299 32 32 32 44 44 44 F F F
38282024-02-01T17:29:26.123Z 15 300-319 30 30 30 42 42 42 F F F
38292024-02-01T17:29:26.123Z 16 320-339 30 30 30 42 42 42 F F F
38302024-02-01T17:29:26.123Z 17 340-359 33 33 33 47 47 47 F F F
38312024-02-01T17:29:26.123Z 18 360-379 33 33 33 47 47 47 T F T <---
38322024-02-01T17:29:26.123Z 19 380-399 26 26 26 40 40 40 F F F
38332024-02-01T17:29:26.123Z 20 400-419 33 33 33 47 47 47 F F F
38342024-02-01T17:29:26.123Z 21 420-439 30 30 30 42 42 42 T T T
38352024-02-01T17:29:26.124Z 22 440-459 26 26 26 40 40 40 T T T
38362024-02-01T17:29:26.124Z 23 460-479 32 32 32 44 44 44 T F T <---
38372024-02-01T17:29:26.124Z 24 480-499 30 30 30 42 42 42 T F T <---
38382024-02-01T17:29:26.124Z 25 500-519 33 33 33 47 47 47 F F F
38392024-02-01T17:29:26.124Z 26 520-539 32 32 32 45 45 45 T F T <---
38402024-02-01T17:29:26.124Z 27 540-559 27 27 27 41 41 41 T F T <---
38412024-02-01T17:29:26.124Z 28 560-579 27 27 27 41 41 41 F F F
38422024-02-01T17:29:26.124Z 29 580-599 27 27 27 41 41 41 T F T <---
38432024-02-01T17:29:26.124ZMax gen: 33, Max flush: 47
38442024-02-01T17:29:26.124ZError: Difference in extent metadata found!
38452024-02-01T17:29:26.127ZA Difference in extent metadata is expected here
38462024-02-01T17:29:26.127ZOn loop 17
38472024-02-01T17:29:26.128Z
38482024-02-01T17:29:26.128ZVerifying data now
38492024-02-01T17:29:27.172ZNo change after verify
38502024-02-01T17:29:27.172ZLoop: 17 Downstairs dump after verify (and repair):
38512024-02-01T17:29:27.195ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38522024-02-01T17:29:27.195Z 0 000-019 33 33 33 47 47 47 F F F
38532024-02-01T17:29:27.195Z 1 020-039 33 33 33 47 47 47 F F F
38542024-02-01T17:29:27.195Z 2 040-059 26 26 26 40 40 40 F F F
38552024-02-01T17:29:27.195Z 3 060-079 31 31 31 43 43 43 F F F
38562024-02-01T17:29:27.195Z 4 080-099 32 32 32 46 46 46 F F F
38572024-02-01T17:29:27.195Z 5 100-119 34 34 34 48 48 48 F F F
38582024-02-01T17:29:27.195Z 6 120-139 34 34 34 48 48 48 F F F
38592024-02-01T17:29:27.195Z 7 140-159 21 21 21 33 33 33 F F F
38602024-02-01T17:29:27.195Z 8 160-179 34 34 34 48 48 48 F F F
38612024-02-01T17:29:27.195Z 9 180-199 27 27 27 41 41 41 F F F
38622024-02-01T17:29:27.195Z 10 200-219 34 34 34 48 48 48 F F F
38632024-02-01T17:29:27.198Z 11 220-239 9 9 9 13 13 13 F F F
38642024-02-01T17:29:27.199Z 12 240-259 30 30 30 42 42 42 F F F
38652024-02-01T17:29:27.199Z 13 260-279 33 33 33 47 47 47 F F F
38662024-02-01T17:29:27.199Z 14 280-299 32 32 32 44 44 44 F F F
38672024-02-01T17:29:27.199Z 15 300-319 30 30 30 42 42 42 F F F
38682024-02-01T17:29:27.199Z 16 320-339 30 30 30 42 42 42 F F F
38692024-02-01T17:29:27.199Z 17 340-359 33 33 33 47 47 47 F F F
38702024-02-01T17:29:27.199Z 18 360-379 34 34 34 48 48 48 F F F
38712024-02-01T17:29:27.200Z 19 380-399 26 26 26 40 40 40 F F F
38722024-02-01T17:29:27.200Z 20 400-419 33 33 33 47 47 47 F F F
38732024-02-01T17:29:27.200Z 21 420-439 34 34 34 48 48 48 F F F
38742024-02-01T17:29:27.200Z 22 440-459 34 34 34 48 48 48 F F F
38752024-02-01T17:29:27.200Z 23 460-479 34 34 34 48 48 48 F F F
38762024-02-01T17:29:27.200Z 24 480-499 34 34 34 48 48 48 F F F
38772024-02-01T17:29:27.200Z 25 500-519 33 33 33 47 47 47 F F F
38782024-02-01T17:29:27.200Z 26 520-539 34 34 34 48 48 48 F F F
38792024-02-01T17:29:27.200Z 27 540-559 34 34 34 48 48 48 F F F
38802024-02-01T17:29:27.200Z 28 560-579 27 27 27 41 41 41 F F F
38812024-02-01T17:29:27.200Z 29 580-599 34 34 34 48 48 48 F F F
38822024-02-01T17:29:27.200ZMax gen: 34, Max flush: 48
38832024-02-01T17:29:27.203Z
38842024-02-01T17:29:27.203ZBegin loop 18
38852024-02-01T17:29:27.203ZDownstairs to restart: 2
38862024-02-01T17:29:27.230Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.4948195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
38872024-02-01T17:29:27.230Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495284527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
38882024-02-01T17:29:27.234Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495350505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
38892024-02-01T17:29:27.235Z{"msg":"Crucible 906f662b-21cd-4683-8693-6886f0eb371e has session id: 2c367889-c538-449a-8058-22a569a3d6e7","v":0,"name":"crucible","level":30Crucible runtime is spawned
38902024-02-01T17:29:27.235Z,"time":"2024-02-01T17:29:26.495446034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38912024-02-01T17:29:27.235Z{"msg":"Upstairs opts: Upstairs UUID: 906f662b-21cd-4683-8693-6886f0eb371e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495634635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38922024-02-01T17:29:27.235Z{"msg":"Crucible stats registered with UUID: 906f662b-21cd-4683-8693-6886f0eb371e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495686061Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38932024-02-01T17:29:27.235Z{"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495710458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
38942024-02-01T17:29:27.235Z{"msg":"906f662b-21cd-4683-8693-6886f0eb371e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495749003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38952024-02-01T17:29:27.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495771943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38962024-02-01T17:29:27.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.4958711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38972024-02-01T17:29:27.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.49589766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38982024-02-01T17:29:27.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.495944001Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
38992024-02-01T17:29:27.235Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496007217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39002024-02-01T17:29:27.235Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496035468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39012024-02-01T17:29:27.236Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.49605808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39022024-02-01T17:29:27.236Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496090342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39032024-02-01T17:29:27.236Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496406358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39042024-02-01T17:29:27.236Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496540388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39052024-02-01T17:29:27.236Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496699927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39062024-02-01T17:29:27.236Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.496843249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"io task","client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39072024-02-01T17:29:27.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.497382286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39082024-02-01T17:29:27.236Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.497451336Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39092024-02-01T17:29:27.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.497668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39102024-02-01T17:29:27.236Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.497757555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39112024-02-01T17:29:27.236Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.498293589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39122024-02-01T17:29:27.236Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.498357112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39132024-02-01T17:29:27.241Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.498403511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39142024-02-01T17:29:27.241Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.498495958Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39152024-02-01T17:29:27.241Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.49933326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39162024-02-01T17:29:27.242Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39172024-02-01T17:29:27.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499420865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39182024-02-01T17:29:27.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499444316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39192024-02-01T17:29:27.242Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499496582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39202024-02-01T17:29:27.242Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499521628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39212024-02-01T17:29:27.242Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.499546948Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39222024-02-01T17:29:27.242Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500203813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39232024-02-01T17:29:27.242Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500270665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39242024-02-01T17:29:27.242Z{"msg":"[0]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 48, 33, 48, 41, 48, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500332611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39252024-02-01T17:29:27.242Z{"msg":"[0]R generation[0..12]: [33, 33, 26, 31, 32, 34, 34, 21, 34, 27, 34, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500393707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39262024-02-01T17:29:27.243Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500423257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39272024-02-01T17:29:27.243Z{"msg":"[1]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 48, 33, 48, 41, 48, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500485016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39282024-02-01T17:29:27.243Z{"msg":"[1]R generation[0..12]: [33, 33, 26, 31, 32, 34, 34, 21, 34, 27, 34, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500512779Z"Wait for a query_work_queue command to finish before sending IO
39292024-02-01T17:29:27.243Z,"hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39302024-02-01T17:29:27.243Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500639964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39312024-02-01T17:29:27.244Z{"msg":"[2]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 48, 33, 48, 41, 48, 13]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500680272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39322024-02-01T17:29:27.244Z{"msg":"[2]R generation[0..12]: [33, 33, 26, 31, 32, 34, 34, 21, 34, 27, 34, 9]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500737195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39332024-02-01T17:29:27.244Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500764421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39342024-02-01T17:29:27.244Z{"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.5008332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39352024-02-01T17:29:27.244Z{"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500860583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39362024-02-01T17:29:27.244Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500882172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39372024-02-01T17:29:27.244Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.50090378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"":"downstairs","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39382024-02-01T17:29:27.244Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500930756Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39392024-02-01T17:29:27.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.500984427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"0","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39402024-02-01T17:29:27.244Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501026249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"1","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39412024-02-01T17:29:27.245Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501057568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"client":"2","":"downstairs","session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39422024-02-01T17:29:27.245Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501115003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39432024-02-01T17:29:27.245Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501139327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39442024-02-01T17:29:27.245Z{"msg":"906f662b-21cd-4683-8693-6886f0eb371e is now active with session: 2c367889-c538-449a-8058-22a569a3d6e7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501195681Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39452024-02-01T17:29:27.245Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501220402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273,"session_id":"2c367889-c538-449a-8058-22a569a3d6e7"}
39462024-02-01T17:29:27.245Z{"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:26.501274629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
39472024-02-01T17:29:27.245ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
39482024-02-01T17:29:27.245ZLoading write count information from file "/tmp/test_repair_verify.data"
39492024-02-01T17:29:27.246ZRun Repair workload
39502024-02-01T17:29:27.246Z01/30 Write block 152 len 40960 data: 4 4 3 4 3 3 3 3 4 5
39512024-02-01T17:29:27.249Z02/30 Read block 318 len 20480
39522024-02-01T17:29:27.249Z03/30 Write block 388 len 36864 data: 3 4 4 4 4 4 2 2 2
39532024-02-01T17:29:27.249Z04/30 Write block 511 len 12288 data: 3 3 3
39542024-02-01T17:29:27.249Z05/30 Write block 265 len 28672 data: 3 3 3 3 4 4 4
39552024-02-01T17:29:27.249Z06/30 Read block 244 len 28672
39562024-02-01T17:29:27.250Z07/30 Write block 254 len 4096 data: 5
39572024-02-01T17:29:27.253Z08/30 Read block 597 len 4096
39582024-02-01T17:29:27.253Z09/30 Read block 78 len 20480
39592024-02-01T17:29:27.254Z10/30 Flush
39602024-02-01T17:29:27.256Z11/30 Write block 358 len 36864 data: 6 6 5 6 6 6 5 4 4
39612024-02-01T17:29:27.274Z12/30 Write block 123 len 4096 data: 5
39622024-02-01T17:29:27.274Z13/30 Flush
39632024-02-01T17:29:27.274Z14/30 Write block 465 len 36864 data: 4 6 6 5 3 4 4 4 2
39642024-02-01T17:29:27.274Z15/30 Write block 179 len 36864 data: 5 4 6 5 6 6 7 5 4
39652024-02-01T17:29:27.274Z16/30 Read block 51 len 36864
39662024-02-01T17:29:27.274Z17/30 Read block 376 len 32768
39672024-02-01T17:29:27.274Z18/30 Write block 416 len 40960 data: 5 3 3 4 3 3 4 4 3 4
39682024-02-01T17:29:27.274Z19/30 Read block 577 len 40960
39692024-02-01T17:29:27.274Z20/30 Write block 224 len 4096 data: 2
39702024-02-01T17:29:27.274Z21/30 Write block 477 len 32768 data: 3 3 4 4 4 4 5 5
39712024-02-01T17:29:27.278Z22/30 Write block 207 len 36864 data: 9 9 7 4 5 3 3 3 3
39722024-02-01T17:29:27.278Z23/30 Read block 17 len 20480
39732024-02-01T17:29:27.278Z24/30 Write block 263 len 32768 data: 3 3 4 4 4 4 5 5
39742024-02-01T17:29:27.281Z25/30 Read block 429 len 28672
39752024-02-01T17:29:27.281Z26/30 Write block 271 len 4096 data: 5
39762024-02-01T17:29:27.281Z27/30 Read block 129 len 12288
39772024-02-01T17:29:27.284Z28/30 Read block 384 len 36864
39782024-02-01T17:29:27.284Z29/30 Read block 297 len 40960
39792024-02-01T17:29:27.287Z30/30 Read block 401 len 36864
39802024-02-01T17:29:27.288Z----------------------------------------------------------------
39812024-02-01T17:29:27.288Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
39822024-02-01T17:29:27.288ZStates: Active Active Active
39832024-02-01T17:29:27.288ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
39842024-02-01T17:29:27.288Z 1 Acked 1000 Write 10 Done Done Done false
39852024-02-01T17:29:27.288Z 2 Acked 1001 Read 5 Done Done Done false
39862024-02-01T17:29:27.288Z 3 Acked 1002 Write 9 Done Done Done false
39872024-02-01T17:29:27.288Z 4 Acked 1003 Write 3 Done Done Done false
39882024-02-01T17:29:27.288Z 5 Acked 1004 Write 7 Done Done Done false
39892024-02-01T17:29:27.288Z 6 Acked 1005 Read 7 Done Done Done false
39902024-02-01T17:29:27.288Z 7 Acked 1006 Write 1 Done Done Sent false
39912024-02-01T17:29:27.288Z 8 Acked 1007 Read 1 Done Done Sent false
39922024-02-01T17:29:27.288Z 9 Acked 1008 Read 5 Done Done Sent false
39932024-02-01T17:29:27.288Z 10 Acked 1009 Flush 0 Done Done Sent false
39942024-02-01T17:29:27.288Z 11 Acked 1010 Write 9 Done Done Sent false
39952024-02-01T17:29:27.288Z 12 Acked 1011 Write 1 Done Done Sent false
39962024-02-01T17:29:27.288Z 13 Acked 1012 Flush 0 Done Done Sent false
39972024-02-01T17:29:27.288Z 14 Acked 1013 Write 9 Done Done Sent false
39982024-02-01T17:29:27.288Z 15 Acked 1014 Write 9 Done Done Sent false
39992024-02-01T17:29:27.288Z 16 Acked 1015 Read 9 Done Done Sent false
40002024-02-01T17:29:27.288Z 17 Acked 1016 Read 8 Done Done Sent false
40012024-02-01T17:29:27.288Z 18 Acked 1017 Write 10 Done Done Sent false
40022024-02-01T17:29:27.288Z 19 Acked 1018 Read 10 Done Done Sent false
40032024-02-01T17:29:27.292Z 20 Acked 1019 Write 1 Done Done Sent false
40042024-02-01T17:29:27.292Z 21 Acked 1020 Write 8 Done Done Sent false
40052024-02-01T17:29:27.293Z 22 Acked 1021 Write 9 Done Done Sent false
40062024-02-01T17:29:27.293Z 23 Acked 1022 Read 5 Done Done Sent false
40072024-02-01T17:29:27.293Z 24 Acked 1023 Write 8 Done Done Sent false
40082024-02-01T17:29:27.293Z 25 Acked 1024 Read 7 Done Done Sent false
40092024-02-01T17:29:27.293Z 26 Acked 1025 Write 1 Done Done Sent false
40102024-02-01T17:29:27.293Z 27 Acked 1026 Read 3 Done Done Sent false
40112024-02-01T17:29:27.293Z 28 Acked 1027 Read 9 Done Done Sent false
40122024-02-01T17:29:27.293Z 29 Acked 1028 Read 10 Done Done Sent false
40132024-02-01T17:29:27.293Z 30 Acked 1029 Read 9 Done Sent Sent false
40142024-02-01T17:29:27.293Z STATES DS:0 DS:1 DS:2 TOTAL
40152024-02-01T17:29:27.293Z New 0 0 0 0
40162024-02-01T17:29:27.293Z Sent 0 1 24 25
40172024-02-01T17:29:27.293Z Done 30 29 6 65
40182024-02-01T17:29:27.293Z Skipped 0 0 0 0
40192024-02-01T17:29:27.293Z Error 0 0 0 0
40202024-02-01T17:29:27.293ZLast Flush: 1012 1012 0
40212024-02-01T17:29:27.294ZDownstairs last five completed:
40222024-02-01T17:29:27.294ZUpstairs last five completed: 30 29 28 27 26
40232024-02-01T17:29:27.294Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:26.552640204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1273}
40242024-02-01T17:29:27.294ZWrote out file "/tmp/test_repair_verify.data"
40252024-02-01T17:29:27.297Z
40262024-02-01T17:29:27.301ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40272024-02-01T17:29:27.330ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40282024-02-01T17:29:27.330Z 0 000-019 33 33 33 47 47 47 F F F
40292024-02-01T17:29:27.331Z 1 020-039 33 33 33 47 47 47 F F F
40302024-02-01T17:29:27.331Z 2 040-059 26 26 26 40 40 40 F F F
40312024-02-01T17:29:27.331Z 3 060-079 31 31 31 43 43 43 F F F
40322024-02-01T17:29:27.334Z 4 080-099 32 32 32 46 46 46 F F F
40332024-02-01T17:29:27.334Z 5 100-119 34 34 34 48 48 48 F F F
40342024-02-01T17:29:27.334Z 6 120-139 36 36 34 50 50 48 F F F <---
40352024-02-01T17:29:27.334Z 7 140-159 36 36 21 49 49 33 F F T <---
40362024-02-01T17:29:27.334Z 8 160-179 36 36 34 49 49 48 T T T <---
40372024-02-01T17:29:27.334Z 9 180-199 27 27 27 41 41 41 T T F <---
40382024-02-01T17:29:27.334Z 10 200-219 34 34 34 48 48 48 T T F <---
40392024-02-01T17:29:27.334Z 11 220-239 9 9 9 13 13 13 T T F <---
40402024-02-01T17:29:27.334Z 12 240-259 36 36 30 49 49 42 F F F <---
40412024-02-01T17:29:27.334Z 13 260-279 36 36 33 49 49 47 T T T <---
40422024-02-01T17:29:27.334Z 14 280-299 32 32 32 44 44 44 F F F
40432024-02-01T17:29:27.334Z 15 300-319 30 30 30 42 42 42 F F F
40442024-02-01T17:29:27.334Z 16 320-339 30 30 30 42 42 42 F F F
40452024-02-01T17:29:27.334Z 17 340-359 36 36 33 50 50 47 F F F <---
40462024-02-01T17:29:27.334Z 18 360-379 36 36 34 50 50 48 F F F <---
40472024-02-01T17:29:27.334Z 19 380-399 36 36 26 49 49 40 F F T <---
40482024-02-01T17:29:27.334Z 20 400-419 33 33 33 47 47 47 T T F <---
40492024-02-01T17:29:27.335Z 21 420-439 34 34 34 48 48 48 T T F <---
40502024-02-01T17:29:27.335Z 22 440-459 34 34 34 48 48 48 F F F
40512024-02-01T17:29:27.335Z 23 460-479 34 34 34 48 48 48 T T F <---
40522024-02-01T17:29:27.335Z 24 480-499 34 34 34 48 48 48 T T F <---
40532024-02-01T17:29:27.336Z 25 500-519 36 36 33 49 49 47 F F T <---
40542024-02-01T17:29:27.336Z 26 520-539 34 34 34 48 48 48 F F F
40552024-02-01T17:29:27.336Z 27 540-559 34 34 34 48 48 48 F F F
40562024-02-01T17:29:27.336Z 28 560-579 27 27 27 41 41 41 F F F
40572024-02-01T17:29:27.336Z 29 580-599 34 34 34 48 48 48 F F F
40582024-02-01T17:29:27.336ZMax gen: 36, Max flush: 50
40592024-02-01T17:29:27.336ZError: Difference in extent metadata found!
40602024-02-01T17:29:27.339ZA Difference in extent metadata is expected here
40612024-02-01T17:29:27.339ZOn loop 18
40622024-02-01T17:29:27.339Z
40632024-02-01T17:29:27.339ZVerifying data now
40642024-02-01T17:29:33.711ZNo change after verify
40652024-02-01T17:29:33.711ZLoop: 18 Downstairs dump after verify (and repair):
40662024-02-01T17:29:33.717ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
40672024-02-01T17:29:33.718Z 0 000-019 33 33 33 47 47 47 F F F
40682024-02-01T17:29:33.718Z 1 020-039 33 33 33 47 47 47 F F F
40692024-02-01T17:29:33.718Z 2 040-059 26 26 26 40 40 40 F F F
40702024-02-01T17:29:33.718Z 3 060-079 31 31 31 43 43 43 F F F
40712024-02-01T17:29:33.718Z 4 080-099 32 32 32 46 46 46 F F F
40722024-02-01T17:29:33.719Z 5 100-119 34 34 34 48 48 48 F F F
40732024-02-01T17:29:33.719Z 6 120-139 36 36 36 50 50 50 F F F
40742024-02-01T17:29:33.719Z 7 140-159 36 36 36 49 49 49 F F F
40752024-02-01T17:29:33.719Z 8 160-179 37 37 37 51 51 51 F F F
40762024-02-01T17:29:33.719Z 9 180-199 37 37 37 51 51 51 F F F
40772024-02-01T17:29:33.719Z 10 200-219 37 37 37 51 51 51 F F F
40782024-02-01T17:29:33.720Z 11 220-239 37 37 37 51 51 51 F F F
40792024-02-01T17:29:33.720Z 12 240-259 36 36 36 49 49 49 F F F
40802024-02-01T17:29:33.720Z 13 260-279 37 37 37 51 51 51 F F F
40812024-02-01T17:29:33.720Z 14 280-299 32 32 32 44 44 44 F F F
40822024-02-01T17:29:33.720Z 15 300-319 30 30 30 42 42 42 F F F
40832024-02-01T17:29:33.720Z 16 320-339 30 30 30 42 42 42 F F F
40842024-02-01T17:29:33.721Z 17 340-359 36 36 36 50 50 50 F F F
40852024-02-01T17:29:33.721Z 18 360-379 36 36 36 50 50 50 F F F
40862024-02-01T17:29:33.721Z 19 380-399 36 36 36 49 49 49 F F F
40872024-02-01T17:29:33.721Z 20 400-419 37 37 37 51 51 51 F F F
40882024-02-01T17:29:33.721Z 21 420-439 37 37 37 51 51 51 F F F
40892024-02-01T17:29:33.721Z 22 440-459 34 34 34 48 48 48 F F F
40902024-02-01T17:29:33.721Z 23 460-479 37 37 37 51 51 51 F F F
40912024-02-01T17:29:33.721Z 24 480-499 37 37 37 51 51 51 F F F
40922024-02-01T17:29:33.721Z 25 500-519 36 36 36 49 49 49 F F F
40932024-02-01T17:29:33.721Z 26 520-539 34 34 34 48 48 48 F F F
40942024-02-01T17:29:33.721Z 27 540-559 34 34 34 48 48 48 F F F
40952024-02-01T17:29:33.721Z 28 560-579 27 27 27 41 41 41 F F F
40962024-02-01T17:29:33.721Z 29 580-599 34 34 34 48 48 48 F F F
40972024-02-01T17:29:33.722ZMax gen: 37, Max flush: 51
40982024-02-01T17:29:33.722Z
40992024-02-01T17:29:33.722ZBegin loop 19
41002024-02-01T17:29:33.722ZDownstairs to restart: 2
41012024-02-01T17:29:33.739Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.003399128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
41022024-02-01T17:29:33.750Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
41032024-02-01T17:29:33.750Z,"time":"2024-02-01T17:29:33.00398311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
41042024-02-01T17:29:33.750Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004169845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
41052024-02-01T17:29:33.750Z{"msg":"Crucible 9a7afb5a-9b5a-4f21-93a8-929a0ec1c81c has session id: 9c953c86-9a55-4353-a429-4057fbde04b9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004246672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41062024-02-01T17:29:33.750Z{"msg":"Upstairs opts: Upstairs UUID: 9a7afb5a-9b5a-4f21-93a8-929a0ec1c81c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.00432821Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41072024-02-01T17:29:33.750Z{"msg":"Crucible stats registered with UUID: 9a7afb5a-9b5a-4f21-93a8-929a0ec1c81c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004399633Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41082024-02-01T17:29:33.750Z{"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.0044672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
41092024-02-01T17:29:33.750Z{"msg":"9a7afb5a-9b5a-4f21-93a8-929a0ec1c81c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004547388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41102024-02-01T17:29:33.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.00458647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41112024-02-01T17:29:33.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004735394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41122024-02-01T17:29:33.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.004959329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41132024-02-01T17:29:33.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005053509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41142024-02-01T17:29:33.752Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.00512701Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41152024-02-01T17:29:33.752Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005199479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41162024-02-01T17:29:33.752Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005329502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41172024-02-01T17:29:33.752Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.00540941Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41182024-02-01T17:29:33.753Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005452661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41192024-02-01T17:29:33.753Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005524125Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41202024-02-01T17:29:33.754Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005616052Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41212024-02-01T17:29:33.754Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.005659463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"io task","client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41222024-02-01T17:29:33.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.006442606Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41232024-02-01T17:29:33.754Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.006515725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41242024-02-01T17:29:33.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.006562319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41252024-02-01T17:29:33.754Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.006666644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41262024-02-01T17:29:33.754Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.00729196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41272024-02-01T17:29:33.754Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.007385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41282024-02-01T17:29:33.754Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.007533486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41292024-02-01T17:29:33.754Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.007626794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41302024-02-01T17:29:33.963Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008450795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41312024-02-01T17:29:33.963Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008724403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41322024-02-01T17:29:33.963Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008784463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41332024-02-01T17:29:33.964Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008810796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41342024-02-01T17:29:33.964Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008831807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41352024-02-01T17:29:33.964Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008873222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41362024-02-01T17:29:33.964Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.008898116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41372024-02-01T17:29:33.964Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009268501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41382024-02-01T17:29:33.964Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009334043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41392024-02-01T17:29:33.964Z{"msg":"[0]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 50, 49, 51, 51, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009362847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41402024-02-01T17:29:33.964Z{"msg":"[0]R generation[0..12]: [33, 33, 26, 31, 32, 34, 36, 36, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009390053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41412024-02-01T17:29:33.964Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009434118Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41422024-02-01T17:29:33.964Z{"msg":"[1]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 50, 49, 51, 51, 51, 51]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009460476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41432024-02-01T17:29:33.964Z{"msg":"[1]R generation[0..12]: [33, 33, 26, 31, 32, 34, 36, 36, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009502079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41442024-02-01T17:29:33.964Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009528574Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41452024-02-01T17:29:33.964Z{"msg":"[2]R flush_numbers[0..12]: [47, 47, 40, 43, 46, 48, 50, 49, 51, 51, 51, 51]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
41462024-02-01T17:29:33.964Z,"time":"2024-02-01T17:29:33.009570359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41472024-02-01T17:29:33.964Z{"msg":"[2]R generation[0..12]: [33, 33, 26, 31, 32, 34, 36, 36, 37, 37, 37, 37]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009599272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41482024-02-01T17:29:33.965Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009656389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41492024-02-01T17:29:33.965Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009710275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41502024-02-01T17:29:33.965Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009743794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41512024-02-01T17:29:33.965Z{"msg":"Next flush: 52","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009820033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41522024-02-01T17:29:33.965Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009864398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"":"downstairs","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41532024-02-01T17:29:33.965Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009956214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41542024-02-01T17:29:33.965Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.009995545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"0","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41552024-02-01T17:29:33.966Z{"msg":"Transition from WaitQuorum to Active","v"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
41562024-02-01T17:29:33.966Z:0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010073204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"1","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41572024-02-01T17:29:33.966Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010110747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"client":"2","":"downstairs","session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41582024-02-01T17:29:33.966Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010134798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41592024-02-01T17:29:33.966Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010180848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41602024-02-01T17:29:33.966Z{"msg":"9a7afb5a-9b5a-4f21-93a8-929a0ec1c81c is now active with session: 9c953c86-9a55-4353-a429-4057fbde04b9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010206678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41612024-02-01T17:29:33.967Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010246765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281,"session_id":"9c953c86-9a55-4353-a429-4057fbde04b9"}
41622024-02-01T17:29:33.967Z{"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:33.010269623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
41632024-02-01T17:29:33.967ZLoading write count information from file "/tmp/test_repair_verify.data"
41642024-02-01T17:29:33.967ZRun Repair workload
41652024-02-01T17:29:33.967Z01/30 Write block 298 len 36864 data: 7 7 7 5 5 5 5 5 3
41662024-02-01T17:29:33.967Z02/30 Read block 28 len 4096
41672024-02-01T17:29:33.967Z03/30 Read block 345 len 40960
41682024-02-01T17:29:33.967Z04/30 Write block 327 len 24576 data: 2 2 2 3 4 5
41692024-02-01T17:29:33.967Z05/30 Read block 509 len 24576
41702024-02-01T17:29:33.967Z06/30 Read block 26 len 4096
41712024-02-01T17:29:33.967Z07/30 Read block 510 len 24576
41722024-02-01T17:29:33.967Z08/30 Write block 388 len 28672 data: 4 5 5 5 5 5 3
41732024-02-01T17:29:33.967Z09/30 Read block 266 len 28672
41742024-02-01T17:29:33.967Z10/30 Read block 300 len 16384
41752024-02-01T17:29:34.087Z11/30 Write block 295 len 16384 data: 7 7 6 8
41762024-02-01T17:29:34.087Z12/30 Write block 573 len 20480 data: 2 2 3 3 3
41772024-02-01T17:29:34.087Z13/30 Write block 569 len 4096 data: 4
41782024-02-01T17:29:34.088Z14/30 Read block 105 len 40960
41792024-02-01T17:29:34.088Z15/30 Read block 15 len 8192
41802024-02-01T17:29:34.088Z16/30 Write block 28 len 8192 data: 3 3
41812024-02-01T17:29:34.088Z17/30 Flush
41822024-02-01T17:29:34.088Z18/30 Write block 182 len 24576 data: 6 7 7 8 6 5
41832024-02-01T17:29:34.088Z19/30 Write block 218 len 40960 data: 2 2 2 2 2 2 3 2 2 2
41842024-02-01T17:29:34.088Z20/30 Write block 312 len 24576 data: 4 5 4 6 6 6
41852024-02-01T17:29:34.088Z21/30 Read block 140 len 36864
41862024-02-01T17:29:34.088Z22/30 Write block 541 len 36864 data: 7 7 6 6 5 5 4 4 4
41872024-02-01T17:29:34.088Z23/30 Write block 199 len 28672 data: 3 4 4 5 5 7 7
41882024-02-01T17:29:34.088Z24/30 Read block 294 len 28672
41892024-02-01T17:29:34.088Z25/30 Read block 429 len 32768
41902024-02-01T17:29:34.088Z26/30 Read block 401 len 20480
41912024-02-01T17:29:34.088Z27/30 Read block 414 len 28672
41922024-02-01T17:29:34.088Z28/30 Write block 66 len 8192 data: 2 3
41932024-02-01T17:29:34.088Z29/30 Read block 444 len 8192
41942024-02-01T17:29:34.088Z30/30 Write block 18 len 32768 data: 5 4 4 4 5 5 5 5
41952024-02-01T17:29:34.088Z----------------------------------------------------------------
41962024-02-01T17:29:34.088Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
41972024-02-01T17:29:34.088ZStates: Active Active Active
41982024-02-01T17:29:34.088ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
41992024-02-01T17:29:34.088Z 1 Acked 1000 Write 9 Done Done Done false
42002024-02-01T17:29:34.088Z 2 Acked 1001 Read 1 Done Done Done false
42012024-02-01T17:29:34.088Z 3 Acked 1002 Read 10 Done Done Done false
42022024-02-01T17:29:34.089Z 4 Acked 1003 Write 6 Done Done Sent false
42032024-02-01T17:29:34.089Z 5 Acked 1004 Read 6 Done Done Sent false
42042024-02-01T17:29:34.089Z 6 Acked 1005 Read 1 Done Done Sent false
42052024-02-01T17:29:34.089Z 7 Acked 1006 Read 6 Done Done Sent false
42062024-02-01T17:29:34.089Z 8 Acked 1007 Write 7 Done Done Sent false
42072024-02-01T17:29:34.089Z 9 Acked 1008 Read 7 Done Done Sent false
42082024-02-01T17:29:34.089Z 10 Acked 1009 Read 4 Done Done Sent false
42092024-02-01T17:29:34.089Z 11 Acked 1010 Write 4 Done Done Sent false
42102024-02-01T17:29:34.089Z 12 Acked 1011 Write 5 Done Done Sent false
42112024-02-01T17:29:34.089Z 13 Acked 1012 Write 1 Done Done Sent false
42122024-02-01T17:29:34.089Z 14 Acked 1013 Read 10 Done Done Sent false
42132024-02-01T17:29:34.089Z 15 Acked 1014 Read 2 Done Done Sent false
42142024-02-01T17:29:34.089Z 16 Acked 1015 Write 2 Done Done Sent false
42152024-02-01T17:29:34.089Z 17 Acked 1016 Flush 0 Done Done Sent false
42162024-02-01T17:29:34.089Z 18 Acked 1017 Write 6 Done Done Sent false
42172024-02-01T17:29:34.089Z 19 Acked 1018 Write 10 Done Done Sent false
42182024-02-01T17:29:34.089Z 20 Acked 1019 Write 6 Done Done Sent false
42192024-02-01T17:29:34.089Z 21 Acked 1020 Read 9 Done Done Sent false
42202024-02-01T17:29:34.089Z 22 Acked 1021 Write 9 Done Done Sent false
42212024-02-01T17:29:34.089Z 23 Acked 1022 Write 7 Done Done Sent false
42222024-02-01T17:29:34.089Z 24 Acked 1023 Read 7 Done Done Sent false
42232024-02-01T17:29:34.089Z 25 Acked 1024 Read 8 Done Done Sent false
42242024-02-01T17:29:34.089Z 26 Acked 1025 Read 5 Done Done Sent false
42252024-02-01T17:29:34.089Z 27 Acked 1026 Read 7 Done Done Sent false
42262024-02-01T17:29:34.090Z 28 Acked 1027 Write 2 Done Done Sent false
42272024-02-01T17:29:34.090Z 29 Acked 1028 Read 2 Done Done Sent false
42282024-02-01T17:29:34.090Z 30 Acked 1029 Write 8 Sent Sent Sent false
42292024-02-01T17:29:34.090Z STATES DS:0 DS:1 DS:2 TOTAL
42302024-02-01T17:29:34.090Z New 0 0 0 0
42312024-02-01T17:29:34.090Z Sent 1 1 27 29
42322024-02-01T17:29:34.090Z Done 29 29 3 61
42332024-02-01T17:29:34.090Z Skipped 0 0 0 0
42342024-02-01T17:29:34.090Z Error 0 0 0 0
42352024-02-01T17:29:34.090ZLast Flush: 1016 1016 0
42362024-02-01T17:29:34.090ZDownstairs last five completed:
42372024-02-01T17:29:34.090ZUpstairs last five completed: 30 29 28 27 26
42382024-02-01T17:29:34.090Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:33.055815018Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1281}
42392024-02-01T17:29:34.090ZWrote out file "/tmp/test_repair_verify.data"
42402024-02-01T17:29:34.090Z
42412024-02-01T17:29:34.090ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
42422024-02-01T17:29:34.090ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42432024-02-01T17:29:34.090Z 0 000-019 33 33 33 47 47 47 T T F <---
42442024-02-01T17:29:34.090Z 1 020-039 38 38 33 52 52 47 T T F <---
42452024-02-01T17:29:34.090Z 2 040-059 26 26 26 40 40 40 F F F
42462024-02-01T17:29:34.090Z 3 060-079 31 31 31 43 43 43 T T F <---
42472024-02-01T17:29:34.090Z 4 080-099 32 32 32 46 46 46 F F F
42482024-02-01T17:29:34.090Z 5 100-119 34 34 34 48 48 48 F F F
42492024-02-01T17:29:34.090Z 6 120-139 36 36 36 50 50 50 F F F
42502024-02-01T17:29:34.091Z 7 140-159 36 36 36 49 49 49 F F F
42512024-02-01T17:29:34.091Z 8 160-179 37 37 37 51 51 51 F F F
42522024-02-01T17:29:34.092Z 9 180-199 37 37 37 51 51 51 T T F <---
42532024-02-01T17:29:34.092Z 10 200-219 37 37 37 51 51 51 T T F <---
42542024-02-01T17:29:34.092Z 11 220-239 37 37 37 51 51 51 T T F <---
42552024-02-01T17:29:34.092Z 12 240-259 36 36 36 49 49 49 F F F
42562024-02-01T17:29:34.092Z 13 260-279 37 37 37 51 51 51 F F F
42572024-02-01T17:29:34.092Z 14 280-299 38 38 32 52 52 44 F F T <---
42582024-02-01T17:29:34.092Z 15 300-319 38 38 30 52 52 42 T T T <---
42592024-02-01T17:29:34.092Z 16 320-339 38 38 30 52 52 42 F F F <---
42602024-02-01T17:29:34.092Z 17 340-359 36 36 36 50 50 50 F F F
42612024-02-01T17:29:34.092Z 18 360-379 36 36 36 50 50 50 F F F
42622024-02-01T17:29:34.092Z 19 380-399 38 38 36 52 52 49 F F F <---
42632024-02-01T17:29:34.092Z 20 400-419 37 37 37 51 51 51 F F F
42642024-02-01T17:29:34.092Z 21 420-439 37 37 37 51 51 51 F F F
42652024-02-01T17:29:34.092Z 22 440-459 34 34 34 48 48 48 F F F
42662024-02-01T17:29:34.092Z 23 460-479 37 37 37 51 51 51 F F F
42672024-02-01T17:29:34.092Z 24 480-499 37 37 37 51 51 51 F F F
42682024-02-01T17:29:34.092Z 25 500-519 36 36 36 49 49 49 F F F
42692024-02-01T17:29:34.092Z 26 520-539 34 34 34 48 48 48 F F F
42702024-02-01T17:29:34.092Z 27 540-559 34 34 34 48 48 48 T T F <---
42712024-02-01T17:29:34.092Z 28 560-579 38 38 27 52 52 41 F F F <---
42722024-02-01T17:29:34.092Z 29 580-599 34 34 34 48 48 48 F F F
42732024-02-01T17:29:34.092ZMax gen: 38, Max flush: 52
42742024-02-01T17:29:34.092ZError: Difference in extent metadata found!
42752024-02-01T17:29:34.136ZA Difference in extent metadata is expected here
42762024-02-01T17:29:34.136ZOn loop 19
42772024-02-01T17:29:34.136Z
42782024-02-01T17:29:34.136ZVerifying data now
42792024-02-01T17:29:34.881ZNo change after verify
42802024-02-01T17:29:34.881ZLoop: 19 Downstairs dump after verify (and repair):
42812024-02-01T17:29:34.904ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
42822024-02-01T17:29:34.904Z 0 000-019 39 39 39 53 53 53 F F F
42832024-02-01T17:29:34.904Z 1 020-039 39 39 39 53 53 53 F F F
42842024-02-01T17:29:34.904Z 2 040-059 26 26 26 40 40 40 F F F
42852024-02-01T17:29:34.904Z 3 060-079 39 39 39 53 53 53 F F F
42862024-02-01T17:29:34.907Z 4 080-099 32 32 32 46 46 46 F F F
42872024-02-01T17:29:34.907Z 5 100-119 34 34 34 48 48 48 F F F
42882024-02-01T17:29:34.907Z 6 120-139 36 36 36 50 50 50 F F F
42892024-02-01T17:29:34.907Z 7 140-159 36 36 36 49 49 49 F F F
42902024-02-01T17:29:34.907Z 8 160-179 37 37 37 51 51 51 F F F
42912024-02-01T17:29:34.908Z 9 180-199 39 39 39 53 53 53 F F F
42922024-02-01T17:29:34.908Z 10 200-219 39 39 39 53 53 53 F F F
42932024-02-01T17:29:34.908Z 11 220-239 39 39 39 53 53 53 F F F
42942024-02-01T17:29:34.908Z 12 240-259 36 36 36 49 49 49 F F F
42952024-02-01T17:29:34.908Z 13 260-279 37 37 37 51 51 51 F F F
42962024-02-01T17:29:34.908Z 14 280-299 38 38 38 52 52 52 F F F
42972024-02-01T17:29:34.908Z 15 300-319 39 39 39 53 53 53 F F F
42982024-02-01T17:29:34.908Z 16 320-339 38 38 38 52 52 52 F F F
42992024-02-01T17:29:34.908Z 17 340-359 36 36 36 50 50 50 F F F
43002024-02-01T17:29:34.908Z 18 360-379 36 36 36 50 50 50 F F F
43012024-02-01T17:29:34.908Z 19 380-399 38 38 38 52 52 52 F F F
43022024-02-01T17:29:34.908Z 20 400-419 37 37 37 51 51 51 F F F
43032024-02-01T17:29:34.908Z 21 420-439 37 37 37 51 51 51 F F F
43042024-02-01T17:29:34.908Z 22 440-459 34 34 34 48 48 48 F F F
43052024-02-01T17:29:34.908Z 23 460-479 37 37 37 51 51 51 F F F
43062024-02-01T17:29:34.908Z 24 480-499 37 37 37 51 51 51 F F F
43072024-02-01T17:29:34.908Z 25 500-519 36 36 36 49 49 49 F F F
43082024-02-01T17:29:34.908Z 26 520-539 34 34 34 48 48 48 F F F
43092024-02-01T17:29:34.908Z 27 540-559 39 39 39 53 53 53 F F F
43102024-02-01T17:29:34.908Z 28 560-579 38 38 38 52 52 52 F F F
43112024-02-01T17:29:34.908Z 29 580-599 34 34 34 48 48 48 F F F
43122024-02-01T17:29:34.908ZMax gen: 39, Max flush: 53
43132024-02-01T17:29:34.911Z
43142024-02-01T17:29:34.911ZBegin loop 20
43152024-02-01T17:29:34.912ZDownstairs to restart: 2
43162024-02-01T17:29:34.939Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.203208101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
43172024-02-01T17:29:34.939Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.203716724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
43182024-02-01T17:29:34.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.203782468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
43192024-02-01T17:29:34.943Z{"msg":"Crucible 81483968-8a66-4480-8b5f-559a8d769957 has session id: 7875960c-f0f5-48a8-a7c1-cbb7e6c5d960","v":0,"name":"crucible","level":30Crucible runtime is spawned
43202024-02-01T17:29:34.943Z,"time":"2024-02-01T17:29:34.203857217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43212024-02-01T17:29:34.943Z{"msg":"Upstairs opts: Upstairs UUID: 81483968-8a66-4480-8b5f-559a8d769957, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.20404198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43222024-02-01T17:29:34.943Z{"msg":"Crucible stats registered with UUID: 81483968-8a66-4480-8b5f-559a8d769957","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204124887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43232024-02-01T17:29:34.943Z{"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204197236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
43242024-02-01T17:29:34.943Z{"msg":"81483968-8a66-4480-8b5f-559a8d769957 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204236829Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43252024-02-01T17:29:34.943Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204329449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43262024-02-01T17:29:34.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204383684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43272024-02-01T17:29:34.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204457485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43282024-02-01T17:29:34.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204537928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43292024-02-01T17:29:34.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204614538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43302024-02-01T17:29:34.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204687042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43312024-02-01T17:29:34.944Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204718399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43322024-02-01T17:29:34.944Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204786714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43332024-02-01T17:29:34.944Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204860744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43342024-02-01T17:29:34.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.204984385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43352024-02-01T17:29:34.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.205204163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43362024-02-01T17:29:34.944Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.205307275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"io task","client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43372024-02-01T17:29:34.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.205773917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43382024-02-01T17:29:34.944Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.205860639Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43392024-02-01T17:29:34.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.205975405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43402024-02-01T17:29:34.944Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.206106045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43412024-02-01T17:29:34.944Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.206220659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43422024-02-01T17:29:34.944Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.206265536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43432024-02-01T17:29:34.949Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.207092008Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43442024-02-01T17:29:34.949Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.207156682Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43452024-02-01T17:29:34.949Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.207184405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43462024-02-01T17:29:34.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208114663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43472024-02-01T17:29:34.949Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208183516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43482024-02-01T17:29:34.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208299229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43492024-02-01T17:29:34.950Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208371399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43502024-02-01T17:29:34.950Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208418307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43512024-02-01T17:29:34.950Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208475832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43522024-02-01T17:29:34.950Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208516523Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43532024-02-01T17:29:34.950Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208559608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43542024-02-01T17:29:34.950Z{"msg":"[0]R flush_numbers[0..12]: [53, 53, 40, 53, 46, 48, 50, 49, 51, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208619962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43552024-02-01T17:29:34.950Z{"msg":"[0]R generation[0..12]: [39, 39, 26, 39, 32, 34, 36, 36, 37, 39, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208666085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43562024-02-01T17:29:34.950Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208720929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43572024-02-01T17:29:34.950Z{"msg":"[1]R flush_numbers[0..12]: [53, 53, 40, 53, 46, 48, 50, 49, 51, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208765298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43582024-02-01T17:29:34.950ZWait for a query_work_queue command to finish before sending IO
43592024-02-01T17:29:34.950Z{"msg":"[1]R generation[0..12]: [39, 39, 26, 39, 32, 34, 36, 36, 37, 39, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.20882356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43602024-02-01T17:29:34.950Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208869358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43612024-02-01T17:29:34.950Z{"msg":"[2]R flush_numbers[0..12]: [53, 53, 40, 53, 46, 48, 50, 49, 51, 53, 53, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.208926865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43622024-02-01T17:29:34.950Z{"msg":"[2]R generation[0..12]: [39, 39, 26, 39, 32, 34, 36, 36, 37, 39, 39, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.2089713Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43632024-02-01T17:29:34.950Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209016646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43642024-02-01T17:29:34.950Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209077756Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43652024-02-01T17:29:34.950Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.2091219Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43662024-02-01T17:29:34.950Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209175628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43672024-02-01T17:29:34.950Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209218137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"":"downstairs","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43682024-02-01T17:29:34.950Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209279891Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43692024-02-01T17:29:34.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209318869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"0","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43702024-02-01T17:29:34.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209381296Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"1","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43712024-02-01T17:29:34.950Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209423759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"client":"2","":"downstairs","session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43722024-02-01T17:29:34.950Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209481665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43732024-02-01T17:29:34.951Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
43742024-02-01T17:29:34.951Zname":"crucible","level":30,"time":"2024-02-01T17:29:34.209529409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43752024-02-01T17:29:34.951Z{"msg":"81483968-8a66-4480-8b5f-559a8d769957 is now active with session: 7875960c-f0f5-48a8-a7c1-cbb7e6c5d960","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209569964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43762024-02-01T17:29:34.951Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209608792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289,"session_id":"7875960c-f0f5-48a8-a7c1-cbb7e6c5d960"}
43772024-02-01T17:29:34.951Z{"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:34.209666599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
43782024-02-01T17:29:34.951ZLoading write count information from file "/tmp/test_repair_verify.data"
43792024-02-01T17:29:34.951ZRun Repair workload
43802024-02-01T17:29:34.951Z01/30 Write block 376 len 36864 data: 7 7 6 6 6 7 5 5 4
43812024-02-01T17:29:34.954Z02/30 Read block 401 len 36864
43822024-02-01T17:29:34.954Z03/30 Read block 554 len 16384
43832024-02-01T17:29:34.954Z04/30 Read block 289 len 36864
43842024-02-01T17:29:34.957Z05/30 Write block 22 len 12288 data: 6 6 6
43852024-02-01T17:29:34.957Z06/30 Read block 119 len 16384
43862024-02-01T17:29:34.957Z07/30 Read block 55 len 8192
43872024-02-01T17:29:34.972Z08/30 Flush
43882024-02-01T17:29:34.972Z09/30 Write block 488 len 28672 data: 5 5 5 5 6 5 4
43892024-02-01T17:29:34.972Z10/30 Read block 19 len 32768
43902024-02-01T17:29:34.976Z11/30 Write block 69 len 28672 data: 4 4 4 4 5 5 5
43912024-02-01T17:29:34.977Z12/30 Flush
43922024-02-01T17:29:34.977Z13/30 Write block 108 len 8192 data: 4 4
43932024-02-01T17:29:34.977Z14/30 Write block 25 len 12288 data: 6 4 4
43942024-02-01T17:29:34.977Z15/30 Read block 403 len 24576
43952024-02-01T17:29:34.977Z16/30 Read block 305 len 20480
43962024-02-01T17:29:34.977Z17/30 Write block 158 len 28672 data: 4 4 5 6 5 4 4
43972024-02-01T17:29:34.981Z18/30 Write block 371 len 28672 data: 3 3 4 5 6 8 8
43982024-02-01T17:29:34.981Z19/30 Read block 26 len 16384
43992024-02-01T17:29:34.981Z20/30 Write block 187 len 20480 data: 6 4 4 3 2
44002024-02-01T17:29:34.981Z21/30 Write block 502 len 32768 data: 3 3 2 2 3 3 3 3
44012024-02-01T17:29:34.981Z22/30 Read block 186 len 28672
44022024-02-01T17:29:34.984Z23/30 Write block 575 len 28672 data: 4 4 4 4 4 4 4
44032024-02-01T17:29:34.984Z24/30 Read block 246 len 20480
44042024-02-01T17:29:34.993Z25/30 Write block 68 len 40960 data: 3 5 5 5 5 6 6 6 2 2
44052024-02-01T17:29:34.993Z26/30 Flush
44062024-02-01T17:29:34.993Z27/30 Write block 418 len 36864 data: 4 5 4 4 5 5 4 5 5
44072024-02-01T17:29:34.997Z28/30 Write block 195 len 36864 data: 3 4 5 3 4 5 5 6 6
44082024-02-01T17:29:34.997Z29/30 Read block 446 len 24576
44092024-02-01T17:29:34.997Z30/30 Read block 189 len 8192
44102024-02-01T17:29:35.002Z----------------------------------------------------------------
44112024-02-01T17:29:35.002Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30
44122024-02-01T17:29:35.002ZStates: Active Active Active
44132024-02-01T17:29:35.002ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
44142024-02-01T17:29:35.002Z 1 Acked 1000 Write 9 Done Done Done false
44152024-02-01T17:29:35.002Z 2 Acked 1001 Read 9 Done Done Done false
44162024-02-01T17:29:35.003Z 3 Acked 1002 Read 4 Done Done Sent false
44172024-02-01T17:29:35.003Z 4 Acked 1003 Read 9 Done Done Sent false
44182024-02-01T17:29:35.003Z 5 Acked 1004 Write 3 Done Done Sent false
44192024-02-01T17:29:35.003Z 6 Acked 1005 Read 4 Done Done Sent false
44202024-02-01T17:29:35.003Z 7 Acked 1006 Read 2 Done Done Sent false
44212024-02-01T17:29:35.003Z 8 Acked 1007 Flush 0 Done Done Sent false
44222024-02-01T17:29:35.003Z 9 Acked 1008 Write 7 Done Done Sent false
44232024-02-01T17:29:35.003Z 10 Acked 1009 Read 8 Done Done Sent false
44242024-02-01T17:29:35.003Z 11 Acked 1010 Write 7 Done Done Sent false
44252024-02-01T17:29:35.003Z 12 Acked 1011 Flush 0 Done Done Sent false
44262024-02-01T17:29:35.003Z 13 Acked 1012 Write 2 Done Done Sent false
44272024-02-01T17:29:35.003Z 14 Acked 1013 Write 3 Done Done Sent false
44282024-02-01T17:29:35.003Z 15 Acked 1014 Read 6 Done Done Sent false
44292024-02-01T17:29:35.003Z 16 Acked 1015 Read 5 Done Done Sent false
44302024-02-01T17:29:35.003Z 17 Acked 1016 Write 7 Done Done Sent false
44312024-02-01T17:29:35.003Z 18 Acked 1017 Write 7 Done Done Sent false
44322024-02-01T17:29:35.004Z 19 Acked 1018 Read 4 Done Done Sent false
44332024-02-01T17:29:35.004Z 20 Acked 1019 Write 5 Done Done Sent false
44342024-02-01T17:29:35.004Z 21 Acked 1020 Write 8 Done Done Sent false
44352024-02-01T17:29:35.005Z 22 Acked 1021 Read 7 Done Done Sent false
44362024-02-01T17:29:35.005Z 23 Acked 1022 Write 7 Done Done Sent false
44372024-02-01T17:29:35.005Z 24 Acked 1023 Read 5 Done Done Sent false
44382024-02-01T17:29:35.005Z 25 Acked 1024 Write 10 Done Done Sent false
44392024-02-01T17:29:35.005Z 26 Acked 1025 Flush 0 Done Done Sent false
44402024-02-01T17:29:35.005Z 27 Acked 1026 Write 9 Done Done Sent false
44412024-02-01T17:29:35.005Z 28 Acked 1027 Write 9 Done Done Sent false
44422024-02-01T17:29:35.005Z 29 Acked 1028 Read 6 Done Done Sent false
44432024-02-01T17:29:35.005Z 30 Acked 1029 Read 2 Done Sent Sent false
44442024-02-01T17:29:35.005Z STATES DS:0 DS:1 DS:2 TOTAL
44452024-02-01T17:29:35.005Z New 0 0 0 0
44462024-02-01T17:29:35.005Z Sent 0 1 28 29
44472024-02-01T17:29:35.005Z Done 30 29 2 61
44482024-02-01T17:29:35.005Z Skipped 0 0 0 0
44492024-02-01T17:29:35.006Z Error 0 0 0 0
44502024-02-01T17:29:35.006ZLast Flush: 1025 1025 0
44512024-02-01T17:29:35.006ZDownstairs last five completed:
44522024-02-01T17:29:35.006ZUpstairs last five completed: 30 29 28 27 26
44532024-02-01T17:29:35.006Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:34.263438837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1289}
44542024-02-01T17:29:35.006ZWrote out file "/tmp/test_repair_verify.data"
44552024-02-01T17:29:35.009Z
44562024-02-01T17:29:35.012ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
44572024-02-01T17:29:35.046ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44582024-02-01T17:29:35.047Z 0 000-019 39 39 39 53 53 53 F F F
44592024-02-01T17:29:35.047Z 1 020-039 40 40 39 56 56 53 F F F <---
44602024-02-01T17:29:35.047Z 2 040-059 26 26 26 40 40 40 F F F
44612024-02-01T17:29:35.047Z 3 060-079 40 40 39 56 56 53 F F F <---
44622024-02-01T17:29:35.063Z 4 080-099 32 32 32 46 46 46 F F F
44632024-02-01T17:29:35.063Z 5 100-119 40 40 34 56 56 48 F F F <---
44642024-02-01T17:29:35.064Z 6 120-139 36 36 36 50 50 50 F F F
44652024-02-01T17:29:35.064Z 7 140-159 40 40 36 56 56 49 F F F <---
44662024-02-01T17:29:35.064Z 8 160-179 40 40 37 56 56 51 F F F <---
44672024-02-01T17:29:35.064Z 9 180-199 40 40 39 56 56 53 T T F <---
44682024-02-01T17:29:35.064Z 10 200-219 39 39 39 53 53 53 T T F <---
44692024-02-01T17:29:35.064Z 11 220-239 39 39 39 53 53 53 F F F
44702024-02-01T17:29:35.064Z 12 240-259 36 36 36 49 49 49 F F F
44712024-02-01T17:29:35.064Z 13 260-279 37 37 37 51 51 51 F F F
44722024-02-01T17:29:35.064Z 14 280-299 38 38 38 52 52 52 F F F
44732024-02-01T17:29:35.064Z 15 300-319 39 39 39 53 53 53 F F F
44742024-02-01T17:29:35.064Z 16 320-339 38 38 38 52 52 52 F F F
44752024-02-01T17:29:35.064Z 17 340-359 36 36 36 50 50 50 F F F
44762024-02-01T17:29:35.065Z 18 360-379 40 40 36 56 56 50 F F T <---
44772024-02-01T17:29:35.065Z 19 380-399 40 40 38 54 54 52 F F T <---
44782024-02-01T17:29:35.065Z 20 400-419 37 37 37 51 51 51 T T F <---
44792024-02-01T17:29:35.065Z 21 420-439 37 37 37 51 51 51 T T F <---
44802024-02-01T17:29:35.065Z 22 440-459 34 34 34 48 48 48 F F F
44812024-02-01T17:29:35.065Z 23 460-479 37 37 37 51 51 51 F F F
44822024-02-01T17:29:35.067Z 24 480-499 40 40 37 55 55 51 F F F <---
44832024-02-01T17:29:35.067Z 25 500-519 40 40 36 56 56 49 F F F <---
44842024-02-01T17:29:35.068Z 26 520-539 34 34 34 48 48 48 F F F
44852024-02-01T17:29:35.068Z 27 540-559 39 39 39 53 53 53 F F F
44862024-02-01T17:29:35.068Z 28 560-579 40 40 38 56 56 52 F F F <---
44872024-02-01T17:29:35.068Z 29 580-599 40 40 34 56 56 48 F F F <---
44882024-02-01T17:29:35.068ZMax gen: 40, Max flush: 56
44892024-02-01T17:29:35.068ZError: Difference in extent metadata found!
44902024-02-01T17:29:35.068ZA Difference in extent metadata is expected here
44912024-02-01T17:29:35.068ZOn loop 20
44922024-02-01T17:29:35.068Z
44932024-02-01T17:29:35.068ZVerifying data now
44942024-02-01T17:29:40.717ZNo change after verify
44952024-02-01T17:29:40.717ZLoop: 20 Downstairs dump after verify (and repair):
44962024-02-01T17:29:40.748ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44972024-02-01T17:29:40.748Z 0 000-019 39 39 39 53 53 53 F F F
44982024-02-01T17:29:40.748Z 1 020-039 40 40 40 56 56 56 F F F
44992024-02-01T17:29:40.748Z 2 040-059 26 26 26 40 40 40 F F F
45002024-02-01T17:29:40.748Z 3 060-079 40 40 40 56 56 56 F F F
45012024-02-01T17:29:40.748Z 4 080-099 32 32 32 46 46 46 F F F
45022024-02-01T17:29:40.748Z 5 100-119 40 40 40 56 56 56 F F F
45032024-02-01T17:29:40.749Z 6 120-139 36 36 36 50 50 50 F F F
45042024-02-01T17:29:40.749Z 7 140-159 40 40 40 56 56 56 F F F
45052024-02-01T17:29:40.749Z 8 160-179 40 40 40 56 56 56 F F F
45062024-02-01T17:29:40.749Z 9 180-199 41 41 41 57 57 57 F F F
45072024-02-01T17:29:40.749Z 10 200-219 41 41 41 57 57 57 F F F
45082024-02-01T17:29:40.749Z 11 220-239 39 39 39 53 53 53 F F F
45092024-02-01T17:29:40.749Z 12 240-259 36 36 36 49 49 49 F F F
45102024-02-01T17:29:40.750Z 13 260-279 37 37 37 51 51 51 F F F
45112024-02-01T17:29:40.750Z 14 280-299 38 38 38 52 52 52 F F F
45122024-02-01T17:29:40.750Z 15 300-319 39 39 39 53 53 53 F F F
45132024-02-01T17:29:40.750Z 16 320-339 38 38 38 52 52 52 F F F
45142024-02-01T17:29:40.750Z 17 340-359 36 36 36 50 50 50 F F F
45152024-02-01T17:29:40.750Z 18 360-379 40 40 40 56 56 56 F F F
45162024-02-01T17:29:40.751Z 19 380-399 40 40 40 54 54 54 F F F
45172024-02-01T17:29:40.751Z 20 400-419 41 41 41 57 57 57 F F F
45182024-02-01T17:29:40.751Z 21 420-439 41 41 41 57 57 57 F F F
45192024-02-01T17:29:40.751Z 22 440-459 34 34 34 48 48 48 F F F
45202024-02-01T17:29:40.751Z 23 460-479 37 37 37 51 51 51 F F F
45212024-02-01T17:29:40.751Z 24 480-499 40 40 40 55 55 55 F F F
45222024-02-01T17:29:40.751Z 25 500-519 40 40 40 56 56 56 F F F
45232024-02-01T17:29:40.751Z 26 520-539 34 34 34 48 48 48 F F F
45242024-02-01T17:29:40.752Z 27 540-559 39 39 39 53 53 53 F F F
45252024-02-01T17:29:40.752Z 28 560-579 40 40 40 56 56 56 F F F
45262024-02-01T17:29:40.752Z 29 580-599 40 40 40 56 56 56 F F F
45272024-02-01T17:29:40.752ZMax gen: 41, Max flush: 57
45282024-02-01T17:29:40.752Z
45292024-02-01T17:29:40.752ZBegin loop 21
45302024-02-01T17:29:40.752ZDownstairs to restart: 2
45312024-02-01T17:29:40.753Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.633404479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
45322024-02-01T17:29:40.753Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.633987704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
45332024-02-01T17:29:40.753Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634058926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
45342024-02-01T17:29:40.753Z{"Crucible runtime is spawned
45352024-02-01T17:29:40.754Zmsg":"Crucible 16866196-ddf6-4bd8-8986-dfd6ffa20005 has session id: 303064ee-5435-4e87-a166-0a7b2c951c12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634158263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45362024-02-01T17:29:40.754Z{"msg":"Upstairs opts: Upstairs UUID: 16866196-ddf6-4bd8-8986-dfd6ffa20005, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634229183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45372024-02-01T17:29:40.754Z{"msg":"Crucible stats registered with UUID: 16866196-ddf6-4bd8-8986-dfd6ffa20005","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634256568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45382024-02-01T17:29:40.754Z{"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634365451Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
45392024-02-01T17:29:40.754Z{"msg":"16866196-ddf6-4bd8-8986-dfd6ffa20005 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634386982Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45402024-02-01T17:29:40.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634429991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45412024-02-01T17:29:40.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634490182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45422024-02-01T17:29:40.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634538265Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45432024-02-01T17:29:40.755Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634581557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45442024-02-01T17:29:40.755Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634625006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45452024-02-01T17:29:40.756Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634690868Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45462024-02-01T17:29:40.756Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634734446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45472024-02-01T17:29:40.756Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634778427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45482024-02-01T17:29:40.756Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634822724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45492024-02-01T17:29:40.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.63489911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45502024-02-01T17:29:40.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.634988715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45512024-02-01T17:29:40.756Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.635036486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"io task","client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45522024-02-01T17:29:40.757Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.636363347Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45532024-02-01T17:29:40.758Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.636502255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45542024-02-01T17:29:40.758Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.636546798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45552024-02-01T17:29:40.758Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.636621984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45562024-02-01T17:29:40.758Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.637333797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45572024-02-01T17:29:40.758Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.637607207Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45582024-02-01T17:29:40.759Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.637940041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45592024-02-01T17:29:40.759Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638071905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45602024-02-01T17:29:40.759Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638206694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45612024-02-01T17:29:40.759Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638269812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45622024-02-01T17:29:40.759Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638296791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45632024-02-01T17:29:40.760Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638510621Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45642024-02-01T17:29:40.760Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638606759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45652024-02-01T17:29:40.760Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638684766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45662024-02-01T17:29:40.761Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.638911105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45672024-02-01T17:29:40.761Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639597112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45682024-02-01T17:29:40.761Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639669795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45692024-02-01T17:29:40.761Z{"msg":"[0]R flush_numbers[0..12]: [53, 56, 40, 56, 46, 56, 50, 56, 56, 57, 57, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639696985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45702024-02-01T17:29:40.761Z{"msg":"[0]R generation[0..12]: [39, 40, 26, 40, 32, 40, 36, 40, 40, 41, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639733437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45712024-02-01T17:29:40.761Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.63978722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45722024-02-01T17:29:40.762Z{"msg":"[1]R flush_numbers[0..12]: [53, 56, 40, 56, 46, 56, 50, 56, 56, 57, 57, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639814466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45732024-02-01T17:29:40.762Z{"msg":"[1]R generation[0..12]: [39, 40, 26, 40, 32, 40, 36, 40, 40, 41, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639872768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45742024-02-01T17:29:40.762Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639901942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45752024-02-01T17:29:40.762Z{"msg":"[2]R flush_numbers[0..12]: [53, 56, 40, 56, 46, 56, 50, 56, 56, 57, 57, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.639928517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45762024-02-01T17:29:40.763Z{"Wait for a query_work_queue command to finish before sending IO
45772024-02-01T17:29:40.763Zmsg":"[2]R generation[0..12]: [39, 40, 26, 40, 32, 40, 36, 40, 40, 41, 41, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.63998932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45782024-02-01T17:29:40.763Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640040612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45792024-02-01T17:29:40.763Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640101286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45802024-02-01T17:29:40.763Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640128343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45812024-02-01T17:29:40.763Z{"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640160752Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45822024-02-01T17:29:40.763Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640214739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"":"downstairs","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45832024-02-01T17:29:40.764Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640240083Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45842024-02-01T17:29:40.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640265205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"0","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45852024-02-01T17:29:40.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640329505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"1","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45862024-02-01T17:29:40.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640355335Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"client":"2","":"downstairs","session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45872024-02-01T17:29:40.764Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640385214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45882024-02-01T17:29:40.764Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.64042189Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45892024-02-01T17:29:40.764Z{"msg":"16866196-ddf6-4bd8-8986-dfd6ffa20005 is now active with session: 303064ee-5435-4e87-a166-0a7b2c951c12","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640487584Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45902024-02-01T17:29:40.765Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640511348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297,"session_id":"303064ee-5435-4e87-a166-0a7b2c951c12"}
45912024-02-01T17:29:40.765Z{"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:39.640532512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
45922024-02-01T17:29:40.765ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45932024-02-01T17:29:40.765ZLoading write count information from file "/tmp/test_repair_verify.data"
45942024-02-01T17:29:40.765ZRun Repair workload
45952024-02-01T17:29:40.768Z01/30 Write block 214 len 24576 data: 4 4 2 2 3 3
45962024-02-01T17:29:40.780Z02/30 Flush
45972024-02-01T17:29:40.780Z03/30 Write block 379 len 12288 data: 7 7 8
45982024-02-01T17:29:40.780Z04/30 Read block 174 len 24576
45992024-02-01T17:29:40.781Z05/30 Write block 44 len 12288 data: 4 4 3
46002024-02-01T17:29:40.781Z06/30 Read block 148 len 16384
46012024-02-01T17:29:40.781Z07/30 Read block 8 len 32768
46022024-02-01T17:29:40.781Z08/30 Read block 310 len 36864
46032024-02-01T17:29:40.781Z09/30 Flush
46042024-02-01T17:29:40.781Z10/30 Flush
46052024-02-01T17:29:40.782Z11/30 Write block 153 len 4096 data: 5
46062024-02-01T17:29:40.782Z12/30 Write block 403 len 4096 data: 3
46072024-02-01T17:29:40.782Z13/30 Read block 450 len 16384
46082024-02-01T17:29:40.782Z14/30 Write block 187 len 32768 data: 7 5 5 4 3 2 2 3
46092024-02-01T17:29:40.782Z15/30 Read block 169 len 20480
46102024-02-01T17:29:40.783Z16/30 Read block 300 len 20480
46112024-02-01T17:29:40.783Z17/30 Write block 546 len 8192 data: 6 5
46122024-02-01T17:29:40.783Z18/30 Read block 274 len 32768
46132024-02-01T17:29:40.783Z19/30 Read block 283 len 20480
46142024-02-01T17:29:40.783Z20/30 Read block 5 len 28672
46152024-02-01T17:29:40.783Z21/30 Write block 96 len 12288 data: 3 3 3
46162024-02-01T17:29:40.783Z22/30 Write block 257 len 12288 data: 4 5 5
46172024-02-01T17:29:40.784Z23/30 Write block 190 len 40960 data: 5 4 3 3 4 4 5 6 4 5
46182024-02-01T17:29:40.784Z24/30 Write block 97 len 24576 data: 4 4 4 4 4 4
46192024-02-01T17:29:40.784Z25/30 Read block 399 len 28672
46202024-02-01T17:29:40.784Z26/30 Write block 557 len 32768 data: 5 6 6 6 6 6 7 6
46212024-02-01T17:29:40.786Z27/30 Write block 46 len 12288 data: 4 3 4
46222024-02-01T17:29:40.786Z28/30 Read block 413 len 32768
46232024-02-01T17:29:40.786Z29/30 Read block 291 len 8192
46242024-02-01T17:29:40.786Z30/30 Read block 433 len 24576
46252024-02-01T17:29:40.786Z----------------------------------------------------------------
46262024-02-01T17:29:40.786Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
46272024-02-01T17:29:40.787ZStates: Active Active Active
46282024-02-01T17:29:40.823ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
46292024-02-01T17:29:40.823Z 1 Acked 1000 Write 6 Done Done Sent false
46302024-02-01T17:29:40.824Z 2 Acked 1001 Flush 0 Done Done Sent false
46312024-02-01T17:29:40.824Z 3 Acked 1002 Write 3 Done Done Sent false
46322024-02-01T17:29:40.824Z 4 Acked 1003 Read 6 Done Done Sent false
46332024-02-01T17:29:40.825Z 5 Acked 1004 Write 3 Done Done Sent false
46342024-02-01T17:29:40.835Z 6 Acked 1005 Read 4 Done Done Sent false
46352024-02-01T17:29:40.835Z 7 Acked 1006 Read 8 Done Done Sent false
46362024-02-01T17:29:40.835Z 8 Acked 1007 Read 9 Done Done Sent false
46372024-02-01T17:29:40.835Z 9 Acked 1008 Flush 0 Done Done Sent false
46382024-02-01T17:29:40.836Z 10 Acked 1009 Flush 0 Done Done Sent false
46392024-02-01T17:29:40.836Z 11 Acked 1010 Write 1 Done Done Sent false
46402024-02-01T17:29:40.836Z 12 Acked 1011 Write 1 Done Done Sent false
46412024-02-01T17:29:40.836Z 13 Acked 1012 Read 4 Done Done Sent false
46422024-02-01T17:29:40.837Z 14 Acked 1013 Write 8 Done Done Sent false
46432024-02-01T17:29:40.837Z 15 Acked 1014 Read 5 Done Done Sent false
46442024-02-01T17:29:40.837Z 16 Acked 1015 Read 5 Done Done Sent false
46452024-02-01T17:29:40.837Z 17 Acked 1016 Write 2 Done Done Sent false
46462024-02-01T17:29:40.837Z 18 Acked 1017 Read 8 Done Done Sent false
46472024-02-01T17:29:40.838Z 19 Acked 1018 Read 5 Done Done Sent false
46482024-02-01T17:29:40.838Z 20 Acked 1019 Read 7 Done Done Sent false
46492024-02-01T17:29:40.838Z 21 Acked 1020 Write 3 Done Done Sent false
46502024-02-01T17:29:40.838Z 22 Acked 1021 Write 3 Done Done Sent false
46512024-02-01T17:29:40.839Z 23 Acked 1022 Write 10 Done Done Sent false
46522024-02-01T17:29:40.839Z 24 Acked 1023 Write 6 Done Done Sent false
46532024-02-01T17:29:40.839Z 25 Acked 1024 Read 7 Done Done Sent false
46542024-02-01T17:29:40.839Z 26 Acked 1025 Write 8 Done Done Sent false
46552024-02-01T17:29:40.839Z 27 Acked 1026 Write 3 Done Done Sent false
46562024-02-01T17:29:40.840Z 28 Acked 1027 Read 8 Done Done Sent false
46572024-02-01T17:29:40.840Z 29 Acked 1028 Read 2 Done Done Sent false
46582024-02-01T17:29:40.840Z 30 Acked 1029 Read 6 Done Done Sent false
46592024-02-01T17:29:40.840Z STATES DS:0 DS:1 DS:2 TOTAL
46602024-02-01T17:29:40.841Z New 0 0 0 0
46612024-02-01T17:29:40.841Z Sent 0 0 30 30
46622024-02-01T17:29:40.841Z Done 30 30 0 60
46632024-02-01T17:29:40.841Z Skipped 0 0 0 0
46642024-02-01T17:29:40.841Z Error 0 0 0 0
46652024-02-01T17:29:40.842ZLast Flush: 1009 1009 0
46662024-02-01T17:29:40.842ZDownstairs last five completed:
46672024-02-01T17:29:40.842ZUpstairs last five completed: 30 29 28 27 26
46682024-02-01T17:29:40.842Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:39.68450065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1297}
46692024-02-01T17:29:40.842ZWrote out file "/tmp/test_repair_verify.data"
46702024-02-01T17:29:40.842Z
46712024-02-01T17:29:40.843ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46722024-02-01T17:29:40.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46732024-02-01T17:29:40.843Z 0 000-019 39 39 39 53 53 53 F F F
46742024-02-01T17:29:40.843Z 1 020-039 40 40 40 56 56 56 F F F
46752024-02-01T17:29:40.843Z 2 040-059 42 42 26 59 59 40 T T F <---
46762024-02-01T17:29:40.843Z 3 060-079 40 40 40 56 56 56 F F F
46772024-02-01T17:29:40.843Z 4 080-099 32 32 32 46 46 46 T T F <---
46782024-02-01T17:29:40.843Z 5 100-119 40 40 40 56 56 56 T T F <---
46792024-02-01T17:29:40.844Z 6 120-139 36 36 36 50 50 50 F F F
46802024-02-01T17:29:40.844Z 7 140-159 40 40 40 56 56 56 T T F <---
46812024-02-01T17:29:40.844Z 8 160-179 40 40 40 56 56 56 F F F
46822024-02-01T17:29:40.844Z 9 180-199 41 41 41 57 57 57 T T F <---
46832024-02-01T17:29:40.844Z 10 200-219 42 42 41 58 58 57 F F F <---
46842024-02-01T17:29:40.844Z 11 220-239 39 39 39 53 53 53 F F F
46852024-02-01T17:29:40.845Z 12 240-259 36 36 36 49 49 49 T T F <---
46862024-02-01T17:29:40.845Z 13 260-279 37 37 37 51 51 51 F F F
46872024-02-01T17:29:40.845Z 14 280-299 38 38 38 52 52 52 F F F
46882024-02-01T17:29:40.845Z 15 300-319 39 39 39 53 53 53 F F F
46892024-02-01T17:29:40.845Z 16 320-339 38 38 38 52 52 52 F F F
46902024-02-01T17:29:40.845Z 17 340-359 36 36 36 50 50 50 F F F
46912024-02-01T17:29:40.845Z 18 360-379 42 42 40 59 59 56 F F F <---
46922024-02-01T17:29:40.846Z 19 380-399 42 42 40 59 59 54 F F F <---
46932024-02-01T17:29:40.847Z 20 400-419 41 41 41 57 57 57 T T F <---
46942024-02-01T17:29:40.847Z 21 420-439 41 41 41 57 57 57 F F F
46952024-02-01T17:29:40.848Z 22 440-459 34 34 34 48 48 48 F F F
46962024-02-01T17:29:40.854Z 23 460-479 37 37 37 51 51 51 F F F
46972024-02-01T17:29:40.854Z 24 480-499 40 40 40 55 55 55 F F F
46982024-02-01T17:29:40.854Z 25 500-519 40 40 40 56 56 56 F F F
46992024-02-01T17:29:40.854Z 26 520-539 34 34 34 48 48 48 F F F
47002024-02-01T17:29:40.854Z 27 540-559 39 39 39 53 53 53 T T F <---
47012024-02-01T17:29:40.854Z 28 560-579 40 40 40 56 56 56 T T F <---
47022024-02-01T17:29:40.854Z 29 580-599 40 40 40 56 56 56 F F F
47032024-02-01T17:29:40.854ZMax gen: 42, Max flush: 59
47042024-02-01T17:29:40.855ZError: Difference in extent metadata found!
47052024-02-01T17:29:40.855ZA Difference in extent metadata is expected here
47062024-02-01T17:29:40.855ZOn loop 21
47072024-02-01T17:29:40.855Z
47082024-02-01T17:29:40.855ZVerifying data now
47092024-02-01T17:29:41.514ZNo change after verify
47102024-02-01T17:29:41.514ZLoop: 21 Downstairs dump after verify (and repair):
47112024-02-01T17:29:41.535ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
47122024-02-01T17:29:41.535Z 0 000-019 39 39 39 53 53 53 F F F
47132024-02-01T17:29:41.539Z 1 020-039 40 40 40 56 56 56 F F F
47142024-02-01T17:29:41.539Z 2 040-059 43 43 43 60 60 60 F F F
47152024-02-01T17:29:41.539Z 3 060-079 40 40 40 56 56 56 F F F
47162024-02-01T17:29:41.539Z 4 080-099 43 43 43 60 60 60 F F F
47172024-02-01T17:29:41.540Z 5 100-119 43 43 43 60 60 60 F F F
47182024-02-01T17:29:41.540Z 6 120-139 36 36 36 50 50 50 F F F
47192024-02-01T17:29:41.540Z 7 140-159 43 43 43 60 60 60 F F F
47202024-02-01T17:29:41.540Z 8 160-179 40 40 40 56 56 56 F F F
47212024-02-01T17:29:41.540Z 9 180-199 43 43 43 60 60 60 F F F
47222024-02-01T17:29:41.540Z 10 200-219 42 42 42 58 58 58 F F F
47232024-02-01T17:29:41.540Z 11 220-239 39 39 39 53 53 53 F F F
47242024-02-01T17:29:41.540Z 12 240-259 43 43 43 60 60 60 F F F
47252024-02-01T17:29:41.540Z 13 260-279 37 37 37 51 51 51 F F F
47262024-02-01T17:29:41.540Z 14 280-299 38 38 38 52 52 52 F F F
47272024-02-01T17:29:41.540Z 15 300-319 39 39 39 53 53 53 F F F
47282024-02-01T17:29:41.540Z 16 320-339 38 38 38 52 52 52 F F F
47292024-02-01T17:29:41.541Z 17 340-359 36 36 36 50 50 50 F F F
47302024-02-01T17:29:41.541Z 18 360-379 42 42 42 59 59 59 F F F
47312024-02-01T17:29:41.542Z 19 380-399 42 42 42 59 59 59 F F F
47322024-02-01T17:29:41.542Z 20 400-419 43 43 43 60 60 60 F F F
47332024-02-01T17:29:41.542Z 21 420-439 41 41 41 57 57 57 F F F
47342024-02-01T17:29:41.542Z 22 440-459 34 34 34 48 48 48 F F F
47352024-02-01T17:29:41.542Z 23 460-479 37 37 37 51 51 51 F F F
47362024-02-01T17:29:41.542Z 24 480-499 40 40 40 55 55 55 F F F
47372024-02-01T17:29:41.542Z 25 500-519 40 40 40 56 56 56 F F F
47382024-02-01T17:29:41.542Z 26 520-539 34 34 34 48 48 48 F F F
47392024-02-01T17:29:41.542Z 27 540-559 43 43 43 60 60 60 F F F
47402024-02-01T17:29:41.542Z 28 560-579 43 43 43 60 60 60 F F F
47412024-02-01T17:29:41.543Z 29 580-599 40 40 40 56 56 56 F F F
47422024-02-01T17:29:41.543ZMax gen: 43, Max flush: 60
47432024-02-01T17:29:41.543Z
47442024-02-01T17:29:41.543ZBegin loop 22
47452024-02-01T17:29:41.543ZDownstairs to restart: 0
47462024-02-01T17:29:41.572Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.830603551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
47472024-02-01T17:29:41.572Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831040344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
47482024-02-01T17:29:41.573Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831070916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
47492024-02-01T17:29:41.573Z{"msg":"Crucible 307755f8-1c5b-49c8-a005-b9d0251de2c9 has session id: 6c8853d5-c22b-49cc-90d8-548d4bfa15e0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831093622Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47502024-02-01T17:29:41.573Z{Crucible runtime is spawned
47512024-02-01T17:29:41.573Z"msg":"Upstairs opts: Upstairs UUID: 307755f8-1c5b-49c8-a005-b9d0251de2c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831197585Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47522024-02-01T17:29:41.588Z{"msg":"Crucible stats registered with UUID: 307755f8-1c5b-49c8-a005-b9d0251de2c9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831241349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47532024-02-01T17:29:41.588Z{"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831310553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
47542024-02-01T17:29:41.588Z{"msg":"307755f8-1c5b-49c8-a005-b9d0251de2c9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831495644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47552024-02-01T17:29:41.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.83157005Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47562024-02-01T17:29:41.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831632766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47572024-02-01T17:29:41.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831675543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47582024-02-01T17:29:41.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831714596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47592024-02-01T17:29:41.589Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.83174381Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47602024-02-01T17:29:41.589Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831769145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47612024-02-01T17:29:41.589Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831806138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47622024-02-01T17:29:41.589Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831853077Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47632024-02-01T17:29:41.589Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.831899707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47642024-02-01T17:29:41.589Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.832129855Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47652024-02-01T17:29:41.589Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.832234602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47662024-02-01T17:29:41.589Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.83232124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"io task","client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47672024-02-01T17:29:41.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.833493086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47682024-02-01T17:29:41.590Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.833553249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47692024-02-01T17:29:41.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.833598128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47702024-02-01T17:29:41.590Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.833690577Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47712024-02-01T17:29:41.590Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.834058399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47722024-02-01T17:29:41.590Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.834119208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47732024-02-01T17:29:41.590Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.834812776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47742024-02-01T17:29:41.591Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.834905585Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47752024-02-01T17:29:41.591Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.835266153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47762024-02-01T17:29:41.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.835718156Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47772024-02-01T17:29:41.591Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.835804531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47782024-02-01T17:29:41.591Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.835835601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47792024-02-01T17:29:41.591Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836055497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47802024-02-01T17:29:41.592Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836128512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47812024-02-01T17:29:41.592Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836154653Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47822024-02-01T17:29:41.592Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836212902Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47832024-02-01T17:29:41.592Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836246985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47842024-02-01T17:29:41.592Z{"msg":"[0]R flush_numbers[0..12]: [53, 56, 60, 56, 60, 60, 50, 60, 56, 60, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.83630378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47852024-02-01T17:29:41.593Z{"msg":"[0]R generation[0..12]: [39, 40, 43, 40, 43, 43, 36, 43, 40, 43, 42, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836363059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47862024-02-01T17:29:41.593Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836392675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47872024-02-01T17:29:41.593Z{"msg":"[1]R flush_numbers[0..12]: [53, 56, 60, 56, 60, 60, 50, 60, 56, 60, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836451707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47882024-02-01T17:29:41.593Z{"msg":"[1]R generation[0..12]: [39, 40, 43, 40, 43, 43, 36, 43, 40, 43, 42, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836483814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47892024-02-01T17:29:41.593ZWait for a query_work_queue command to finish before sending IO
47902024-02-01T17:29:41.593Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836545457Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47912024-02-01T17:29:41.593Z{"msg":"[2]R flush_numbers[0..12]: [53, 56, 60, 56, 60, 60, 50, 60, 56, 60, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836606604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47922024-02-01T17:29:41.593Z{"msg":"[2]R generation[0..12]: [39, 40, 43, 40, 43, 43, 36, 43, 40, 43, 42, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836697085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47932024-02-01T17:29:41.594Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836726201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47942024-02-01T17:29:41.594Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836782961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47952024-02-01T17:29:41.594Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836810199Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47962024-02-01T17:29:41.594Z{"msg":"Next flush: 61","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836868983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47972024-02-01T17:29:41.594Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836895833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"":"downstairs","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47982024-02-01T17:29:41.594Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836957387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
47992024-02-01T17:29:41.595Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.836982434Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"0","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48002024-02-01T17:29:41.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837013996Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"1","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48012024-02-01T17:29:41.596Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837053107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"client":"2","":"downstairs","session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48022024-02-01T17:29:41.596Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837109009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48032024-02-01T17:29:41.596Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837133814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48042024-02-01T17:29:41.596Z{"msg":"307755f8-1c5b-49c8-a005-b9d0251de2c9 is now active with session: 6c8853d5-c22b-49cc-90d8-548d4bfa15e0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837189501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48052024-02-01T17:29:41.596Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837214751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305,"session_id":"6c8853d5-c22b-49cc-90d8-548d4bfa15e0"}
48062024-02-01T17:29:41.596Z{"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:40.837271294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
48072024-02-01T17:29:41.596ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
48082024-02-01T17:29:41.596ZLoading write count information from file "/tmp/test_repair_verify.data"
48092024-02-01T17:29:41.597ZRun Repair workload
48102024-02-01T17:29:41.597Z01/30 Write block 377 len 12288 data: 9 7 8
48112024-02-01T17:29:41.597Z02/30 Read block 476 len 32768
48122024-02-01T17:29:41.597Z03/30 Write block 152 len 28672 data: 5 6 4 5 4 4 5
48132024-02-01T17:29:41.597Z04/30 Write block 183 len 4096 data: 8
48142024-02-01T17:29:41.609Z05/30 Write block 326 len 40960 data: 2 3 3 3 4 5 6 7 9 9
48152024-02-01T17:29:41.609Z06/30 Read block 255 len 12288
48162024-02-01T17:29:41.609Z07/30 Read block 98 len 36864
48172024-02-01T17:29:41.609Z08/30 Read block 198 len 20480
48182024-02-01T17:29:41.609Z09/30 Read block 505 len 36864
48192024-02-01T17:29:41.609Z10/30 Write block 375 len 28672 data: 7 9 10 8 9 8 9
48202024-02-01T17:29:41.609Z11/30 Write block 152 len 20480 data: 6 7 5 6 5
48212024-02-01T17:29:41.609Z12/30 Read block 527 len 16384
48222024-02-01T17:29:41.610Z13/30 Read block 412 len 16384
48232024-02-01T17:29:41.610Z14/30 Read block 123 len 16384
48242024-02-01T17:29:41.610Z15/30 Write block 401 len 36864 data: 3 3 4 3 3 3 3 3 3
48252024-02-01T17:29:41.610Z16/30 Read block 300 len 8192
48262024-02-01T17:29:41.610Z17/30 Read block 162 len 12288
48272024-02-01T17:29:41.610Z18/30 Write block 88 len 12288 data: 3 3 3
48282024-02-01T17:29:41.610Z19/30 Write block 20 len 28672 data: 5 5 7 7 7 7 5
48292024-02-01T17:29:41.610Z20/30 Write block 544 len 36864 data: 7 6 7 6 5 5 5 5 5
48302024-02-01T17:29:41.610Z21/30 Read block 431 len 4096
48312024-02-01T17:29:41.617Z22/30 Write block 54 len 28672 data: 6 7 6 5 4 4 5
48322024-02-01T17:29:41.617Z23/30 Write block 380 len 28672 data: 9 10 6 6 5 3 2
48332024-02-01T17:29:41.617Z24/30 Write block 390 len 24576 data: 6 6 6 6 4 3
48342024-02-01T17:29:41.617Z25/30 Read block 291 len 4096
48352024-02-01T17:29:41.617Z26/30 Read block 65 len 36864
48362024-02-01T17:29:41.617Z27/30 Read block 420 len 8192
48372024-02-01T17:29:41.618Z28/30 Read block 60 len 4096
48382024-02-01T17:29:41.618Z29/30 Write block 135 len 20480 data: 4 4 5 4 4
48392024-02-01T17:29:41.618Z30/30 Read block 445 len 16384
48402024-02-01T17:29:41.652Z----------------------------------------------------------------
48412024-02-01T17:29:41.653Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
48422024-02-01T17:29:41.653ZStates: Active Active Active
48432024-02-01T17:29:41.653ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48442024-02-01T17:29:41.653Z 1 Acked 1000 Write 3 Sent Done Done false
48452024-02-01T17:29:41.653Z 2 Acked 1001 Read 8 Sent Done Done false
48462024-02-01T17:29:41.654Z 3 Acked 1002 Write 7 Sent Done Done false
48472024-02-01T17:29:41.654Z 4 Acked 1003 Write 1 Sent Done Done false
48482024-02-01T17:29:41.654Z 5 Acked 1004 Write 10 Sent Done Done false
48492024-02-01T17:29:41.654Z 6 Acked 1005 Read 3 Sent Done Done false
48502024-02-01T17:29:41.654Z 7 Acked 1006 Read 9 Sent Done Done false
48512024-02-01T17:29:41.655Z 8 Acked 1007 Read 5 Sent Done Done false
48522024-02-01T17:29:41.655Z 9 Acked 1008 Read 9 Sent Done Done false
48532024-02-01T17:29:41.655Z 10 Acked 1009 Write 7 Sent Done Done false
48542024-02-01T17:29:41.655Z 11 Acked 1010 Write 5 Sent Done Done false
48552024-02-01T17:29:41.655Z 12 Acked 1011 Read 4 Sent Done Done false
48562024-02-01T17:29:41.656Z 13 Acked 1012 Read 4 Sent Done Done false
48572024-02-01T17:29:41.657Z 14 Acked 1013 Read 4 Sent Done Done false
48582024-02-01T17:29:41.657Z 15 Acked 1014 Write 9 Sent Done Done false
48592024-02-01T17:29:41.657Z 16 Acked 1015 Read 2 Sent Done Done false
48602024-02-01T17:29:41.657Z 17 Acked 1016 Read 3 Sent Done Done false
48612024-02-01T17:29:41.658Z 18 Acked 1017 Write 3 Sent Done Done false
48622024-02-01T17:29:41.658Z 19 Acked 1018 Write 7 Sent Done Done false
48632024-02-01T17:29:41.658Z 20 Acked 1019 Write 9 Sent Done Done false
48642024-02-01T17:29:41.658Z 21 Acked 1020 Read 1 Sent Done Done false
48652024-02-01T17:29:41.659Z 22 Acked 1021 Write 7 Sent Done Done false
48662024-02-01T17:29:41.659Z 23 Acked 1022 Write 7 Sent Done Done false
48672024-02-01T17:29:41.659Z 24 Acked 1023 Write 6 Sent Done Done false
48682024-02-01T17:29:41.659Z 25 Acked 1024 Read 1 Sent Done Done false
48692024-02-01T17:29:41.659Z 26 Acked 1025 Read 9 Sent Done Done false
48702024-02-01T17:29:41.660Z 27 Acked 1026 Read 2 Sent Done Done false
48712024-02-01T17:29:41.660Z 28 Acked 1027 Read 1 Sent Done Done false
48722024-02-01T17:29:41.660Z 29 Acked 1028 Write 5 Sent Done Done false
48732024-02-01T17:29:41.660Z 30 Acked 1029 Read 4 Sent Done Sent false
48742024-02-01T17:29:41.660Z STATES DS:0 DS:1 DS:2 TOTAL
48752024-02-01T17:29:41.661Z New 0 0 0 0
48762024-02-01T17:29:41.661Z Sent 30 0 1 31
48772024-02-01T17:29:41.661Z Done 0 30 29 59
48782024-02-01T17:29:41.661Z Skipped 0 0 0 0
48792024-02-01T17:29:41.662Z Error 0 0 0 0
48802024-02-01T17:29:41.662ZLast Flush: 0 0 0
48812024-02-01T17:29:41.662ZDownstairs last five completed:
48822024-02-01T17:29:41.662ZUpstairs last five completed: 30 29 28 27 26
48832024-02-01T17:29:41.663Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:40.880787468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1305}
48842024-02-01T17:29:41.663ZWrote out file "/tmp/test_repair_verify.data"
48852024-02-01T17:29:41.663Z
48862024-02-01T17:29:41.663ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48872024-02-01T17:29:41.668ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48882024-02-01T17:29:41.668Z 0 000-019 39 39 39 53 53 53 F F F
48892024-02-01T17:29:41.668Z 1 020-039 40 40 40 56 56 56 F T T <---
48902024-02-01T17:29:41.668Z 2 040-059 43 43 43 60 60 60 F T T <---
48912024-02-01T17:29:41.668Z 3 060-079 40 40 40 56 56 56 F T T <---
48922024-02-01T17:29:41.668Z 4 080-099 43 43 43 60 60 60 F T T <---
48932024-02-01T17:29:41.668Z 5 100-119 43 43 43 60 60 60 F F F
48942024-02-01T17:29:41.668Z 6 120-139 36 36 36 50 50 50 F T T <---
48952024-02-01T17:29:41.668Z 7 140-159 43 43 43 60 60 60 F T T <---
48962024-02-01T17:29:41.668Z 8 160-179 40 40 40 56 56 56 F F F
48972024-02-01T17:29:41.668Z 9 180-199 43 43 43 60 60 60 F T T <---
48982024-02-01T17:29:41.668Z 10 200-219 42 42 42 58 58 58 F F F
48992024-02-01T17:29:41.669Z 11 220-239 39 39 39 53 53 53 F F F
49002024-02-01T17:29:41.669Z 12 240-259 43 43 43 60 60 60 F F F
49012024-02-01T17:29:41.669Z 13 260-279 37 37 37 51 51 51 F F F
49022024-02-01T17:29:41.669Z 14 280-299 38 38 38 52 52 52 F F F
49032024-02-01T17:29:41.669Z 15 300-319 39 39 39 53 53 53 F F F
49042024-02-01T17:29:41.669Z 16 320-339 38 38 38 52 52 52 F T T <---
49052024-02-01T17:29:41.669Z 17 340-359 36 36 36 50 50 50 F F F
49062024-02-01T17:29:41.669Z 18 360-379 42 42 42 59 59 59 F T T <---
49072024-02-01T17:29:41.669Z 19 380-399 42 42 42 59 59 59 F T T <---
49082024-02-01T17:29:41.669Z 20 400-419 43 43 43 60 60 60 F T T <---
49092024-02-01T17:29:41.669Z 21 420-439 41 41 41 57 57 57 F F F
49102024-02-01T17:29:41.669Z 22 440-459 34 34 34 48 48 48 F F F
49112024-02-01T17:29:41.669Z 23 460-479 37 37 37 51 51 51 F F F
49122024-02-01T17:29:41.669Z 24 480-499 40 40 40 55 55 55 F F F
49132024-02-01T17:29:41.669Z 25 500-519 40 40 40 56 56 56 F F F
49142024-02-01T17:29:41.669Z 26 520-539 34 34 34 48 48 48 F F F
49152024-02-01T17:29:41.669Z 27 540-559 43 43 43 60 60 60 F T T <---
49162024-02-01T17:29:41.669Z 28 560-579 43 43 43 60 60 60 F F F
49172024-02-01T17:29:41.669Z 29 580-599 40 40 40 56 56 56 F F F
49182024-02-01T17:29:41.669ZMax gen: 43, Max flush: 60
49192024-02-01T17:29:41.670ZError: Difference in extent metadata found!
49202024-02-01T17:29:41.670ZA Difference in extent metadata is expected here
49212024-02-01T17:29:41.670ZOn loop 22
49222024-02-01T17:29:41.670Z
49232024-02-01T17:29:41.670ZVerifying data now
49242024-02-01T17:29:46.747ZNo change after verify
49252024-02-01T17:29:46.748ZLoop: 22 Downstairs dump after verify (and repair):
49262024-02-01T17:29:47.193ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49272024-02-01T17:29:47.193Z 0 000-019 39 39 39 53 53 53 F F F
49282024-02-01T17:29:47.194Z 1 020-039 44 44 44 61 61 61 F F F
49292024-02-01T17:29:47.194Z 2 040-059 44 44 44 61 61 61 F F F
49302024-02-01T17:29:47.194Z 3 060-079 44 44 44 61 61 61 F F F
49312024-02-01T17:29:47.194Z 4 080-099 44 44 44 61 61 61 F F F
49322024-02-01T17:29:47.194Z 5 100-119 43 43 43 60 60 60 F F F
49332024-02-01T17:29:47.194Z 6 120-139 44 44 44 61 61 61 F F F
49342024-02-01T17:29:47.195Z 7 140-159 44 44 44 61 61 61 F F F
49352024-02-01T17:29:47.195Z 8 160-179 40 40 40 56 56 56 F F F
49362024-02-01T17:29:47.195Z 9 180-199 44 44 44 61 61 61 F F F
49372024-02-01T17:29:47.195Z 10 200-219 42 42 42 58 58 58 F F F
49382024-02-01T17:29:47.195Z 11 220-239 39 39 39 53 53 53 F F F
49392024-02-01T17:29:47.195Z 12 240-259 43 43 43 60 60 60 F F F
49402024-02-01T17:29:47.195Z 13 260-279 37 37 37 51 51 51 F F F
49412024-02-01T17:29:47.195Z 14 280-299 38 38 38 52 52 52 F F F
49422024-02-01T17:29:47.195Z 15 300-319 39 39 39 53 53 53 F F F
49432024-02-01T17:29:47.195Z 16 320-339 44 44 44 61 61 61 F F F
49442024-02-01T17:29:47.195Z 17 340-359 36 36 36 50 50 50 F F F
49452024-02-01T17:29:47.196Z 18 360-379 44 44 44 61 61 61 F F F
49462024-02-01T17:29:47.196Z 19 380-399 44 44 44 61 61 61 F F F
49472024-02-01T17:29:47.196Z 20 400-419 44 44 44 61 61 61 F F F
49482024-02-01T17:29:47.196Z 21 420-439 41 41 41 57 57 57 F F F
49492024-02-01T17:29:47.196Z 22 440-459 34 34 34 48 48 48 F F F
49502024-02-01T17:29:47.196Z 23 460-479 37 37 37 51 51 51 F F F
49512024-02-01T17:29:47.196Z 24 480-499 40 40 40 55 55 55 F F F
49522024-02-01T17:29:47.196Z 25 500-519 40 40 40 56 56 56 F F F
49532024-02-01T17:29:47.196Z 26 520-539 34 34 34 48 48 48 F F F
49542024-02-01T17:29:47.196Z 27 540-559 44 44 44 61 61 61 F F F
49552024-02-01T17:29:47.196Z 28 560-579 43 43 43 60 60 60 F F F
49562024-02-01T17:29:47.196Z 29 580-599 40 40 40 56 56 56 F F F
49572024-02-01T17:29:47.196ZMax gen: 44, Max flush: 61
49582024-02-01T17:29:47.196Z
49592024-02-01T17:29:47.197ZBegin loop 23
49602024-02-01T17:29:47.197ZDownstairs to restart: 2
49612024-02-01T17:29:47.197Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.05657875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
49622024-02-01T17:29:47.197Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057077086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
49632024-02-01T17:29:47.197Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057115592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
49642024-02-01T17:29:47.197Z{"msg":"Crucible 4287c9c1-926e-4098-846e-9a4db602d882 has session id: 7b5f88b2-e211-4613-9c0f-a90a49d789b2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057184998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49652024-02-01T17:29:47.197Z{"msg":"Upstairs opts: Upstairs UUID: 4287c9c1-926e-4098-846e-9a4db602d882, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057240098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49662024-02-01T17:29:47.197Z{"msg":"Crucible stats registered with UUID: 4287c9c1-926e-4098-846e-9a4db602d882","v":0,"name":"crucible","level":Crucible runtime is spawned
49672024-02-01T17:29:47.197Z30,"time":"2024-02-01T17:29:46.057355964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49682024-02-01T17:29:47.197Z{"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057447165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
49692024-02-01T17:29:47.197Z{"msg":"4287c9c1-926e-4098-846e-9a4db602d882 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057558995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49702024-02-01T17:29:47.197Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057614362Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49712024-02-01T17:29:47.198Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057649281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49722024-02-01T17:29:47.198Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057699743Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49732024-02-01T17:29:47.199Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057746011Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49742024-02-01T17:29:47.199Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057833155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49752024-02-01T17:29:47.199Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057874527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49762024-02-01T17:29:47.199Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.057917512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49772024-02-01T17:29:47.199Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.058070125Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49782024-02-01T17:29:47.199Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.058209423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49792024-02-01T17:29:47.199Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.058303899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49802024-02-01T17:29:47.200Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.058350934Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49812024-02-01T17:29:47.200Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.058425076Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"io task","client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49822024-02-01T17:29:47.200Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059365651Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49832024-02-01T17:29:47.200Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059446673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49842024-02-01T17:29:47.200Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059544748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49852024-02-01T17:29:47.200Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059594103Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49862024-02-01T17:29:47.201Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059801603Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49872024-02-01T17:29:47.201Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.059916562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49882024-02-01T17:29:47.201Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.060464139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49892024-02-01T17:29:47.201Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.060535528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49902024-02-01T17:29:47.201Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.060885214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49912024-02-01T17:29:47.201Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061510372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49922024-02-01T17:29:47.202Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061604093Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49932024-02-01T17:29:47.202Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061662825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49942024-02-01T17:29:47.202Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061689297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49952024-02-01T17:29:47.203Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061711808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49962024-02-01T17:29:47.203Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061753537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49972024-02-01T17:29:47.203Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061776437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49982024-02-01T17:29:47.203Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.0618169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
49992024-02-01T17:29:47.203Z{"msg":"[0]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061850631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50002024-02-01T17:29:47.203Z{"msg":"[0]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 42, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061893007Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50012024-02-01T17:29:47.203Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061919378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50022024-02-01T17:29:47.205Z{"msg":"[1]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061960619Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50032024-02-01T17:29:47.205Z{"msg":"[1]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 42, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.061985464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50042024-02-01T17:29:47.205Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.06204696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50052024-02-01T17:29:47.205Z{"msg":"[2]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 58, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062078357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50062024-02-01T17:29:47.205Z{"msg":"[2]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 42, 39]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
50072024-02-01T17:29:47.205Z,"time":"2024-02-01T17:29:46.06212034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50082024-02-01T17:29:47.205Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062149863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50092024-02-01T17:29:47.205Z{"msg":"Max found gen is 45","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062190148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50102024-02-01T17:29:47.205Z{"msg":"Generation requested: 46 >= found:45","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062214424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50112024-02-01T17:29:47.205Z{"msg":"Next flush: 62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062235466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50122024-02-01T17:29:47.206Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062256659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"":"downstairs","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50132024-02-01T17:29:47.206Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062278626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50142024-02-01T17:29:47.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062324951Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"0","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50152024-02-01T17:29:47.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062349113Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"1","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50162024-02-01T17:29:47.206Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.06237069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"client":"2","":"downstairs","session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50172024-02-01T17:29:47.206Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062420196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50182024-02-01T17:29:47.206Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.06244293Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50192024-02-01T17:29:47.207Z{"msg":"4287c9c1-926e-4098-846e-9a4db602d882 is now active with session: 7b5f88b2-e211-4613-9c0f-a90a49d789b2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062482957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50202024-02-01T17:29:47.207Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062504814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313,"session_id":"7b5f88b2-e211-4613-9c0f-a90a49d789b2"}
50212024-02-01T17:29:47.207Z{"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:46.062544453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
50222024-02-01T17:29:47.207ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50232024-02-01T17:29:47.207ZLoading write count information from file "/tmp/test_repair_verify.data"
50242024-02-01T17:29:47.207ZRun Repair workload
50252024-02-01T17:29:47.207Z01/30 Write block 250 len 12288 data: 5 4 3
50262024-02-01T17:29:47.228Z02/30 Read block 335 len 32768
50272024-02-01T17:29:47.228Z03/30 Write block 272 len 28672 data: 4 4 3 4 5 3 3
50282024-02-01T17:29:47.228Z04/30 Write block 582 len 20480 data: 4 3 3 3 4
50292024-02-01T17:29:47.228Z05/30 Write block 324 len 12288 data: 3 2 3
50302024-02-01T17:29:47.228Z06/30 Flush
50312024-02-01T17:29:47.228Z07/30 Write block 583 len 36864 data: 4 4 4 5 4 5 6 6 5
50322024-02-01T17:29:47.228Z08/30 Read block 509 len 4096
50332024-02-01T17:29:47.228Z09/30 Flush
50342024-02-01T17:29:47.228Z10/30 Write block 484 len 40960 data: 6 5 5 5 6 6 6 6 7 6
50352024-02-01T17:29:47.229Z11/30 Read block 161 len 32768
50362024-02-01T17:29:47.229Z12/30 Write block 203 len 8192 data: 7 8
50372024-02-01T17:29:47.229Z13/30 Read block 474 len 16384
50382024-02-01T17:29:47.229Z14/30 Read block 539 len 12288
50392024-02-01T17:29:47.229Z15/30 Read block 269 len 12288
50402024-02-01T17:29:47.229Z16/30 Read block 76 len 40960
50412024-02-01T17:29:47.229Z17/30 Flush
50422024-02-01T17:29:47.229Z18/30 Write block 481 len 8192 data: 5 5
50432024-02-01T17:29:47.229Z19/30 Read block 578 len 32768
50442024-02-01T17:29:47.229Z20/30 Read block 510 len 36864
50452024-02-01T17:29:47.229Z21/30 Write block 541 len 28672 data: 8 8 7 8 7 8 7
50462024-02-01T17:29:47.229Z22/30 Read block 336 len 20480
50472024-02-01T17:29:47.229Z23/30 Write block 459 len 32768 data: 6 5 4 4 3 4 5 7
50482024-02-01T17:29:47.229Z24/30 Flush
50492024-02-01T17:29:47.229Z25/30 Write block 513 len 40960 data: 4 3 3 2 2 2 2 3 3 4
50502024-02-01T17:29:47.230Z26/30 Write block 298 len 8192 data: 9 8
50512024-02-01T17:29:47.230Z27/30 Write block 202 len 8192 data: 7 8
50522024-02-01T17:29:47.230Z28/30 Read block 444 len 16384
50532024-02-01T17:29:47.230Z29/30 Read block 533 len 8192
50542024-02-01T17:29:47.230Z30/30 Read block 21 len 36864
50552024-02-01T17:29:47.230Z----------------------------------------------------------------
50562024-02-01T17:29:47.230Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
50572024-02-01T17:29:47.230ZStates: Active Active Active
50582024-02-01T17:29:47.230ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50592024-02-01T17:29:47.230Z 1 Acked 1000 Write 3 Done Done Done false
50602024-02-01T17:29:47.230Z 2 Acked 1001 Read 8 Done Done Done false
50612024-02-01T17:29:47.230Z 3 Acked 1002 Write 7 Done Done Done false
50622024-02-01T17:29:47.230Z 4 Acked 1003 Write 5 Done Done Done false
50632024-02-01T17:29:47.230Z 5 Acked 1004 Write 3 Done Done Sent false
50642024-02-01T17:29:47.230Z 6 Acked 1005 Flush 0 Done Done Sent false
50652024-02-01T17:29:47.230Z 7 Acked 1006 Write 9 Done Done Sent false
50662024-02-01T17:29:47.231Z 8 Acked 1007 Read 1 Done Done Sent false
50672024-02-01T17:29:47.231Z 9 Acked 1008 Flush 0 Done Done Sent false
50682024-02-01T17:29:47.231Z 10 Acked 1009 Write 10 Done Done Sent false
50692024-02-01T17:29:47.231Z 11 Acked 1010 Read 8 Done Done Sent false
50702024-02-01T17:29:47.231Z 12 Acked 1011 Write 2 Done Done Sent false
50712024-02-01T17:29:47.231Z 13 Acked 1012 Read 4 Done Done Sent false
50722024-02-01T17:29:47.231Z 14 Acked 1013 Read 3 Done Done Sent false
50732024-02-01T17:29:47.231Z 15 Acked 1014 Read 3 Done Done Sent false
50742024-02-01T17:29:47.231Z 16 Acked 1015 Read 10 Done Done Sent false
50752024-02-01T17:29:47.231Z 17 Acked 1016 Flush 0 Done Done Sent false
50762024-02-01T17:29:47.231Z 18 Acked 1017 Write 2 Done Done Sent false
50772024-02-01T17:29:47.231Z 19 Acked 1018 Read 8 Done Done Sent false
50782024-02-01T17:29:47.231Z 20 Acked 1019 Read 9 Done Done Sent false
50792024-02-01T17:29:47.231Z 21 Acked 1020 Write 7 Done Done Sent false
50802024-02-01T17:29:47.231Z 22 Acked 1021 Read 5 Done Done Sent false
50812024-02-01T17:29:47.232Z 23 Acked 1022 Write 8 Done Done Sent false
50822024-02-01T17:29:47.232Z 24 Acked 1023 Flush 0 Done Done Sent false
50832024-02-01T17:29:47.232Z 25 Acked 1024 Write 10 Done Done Sent false
50842024-02-01T17:29:47.232Z 26 Acked 1025 Write 2 Done Done Sent false
50852024-02-01T17:29:47.232Z 27 Acked 1026 Write 2 Done Done Sent false
50862024-02-01T17:29:47.232Z 28 Acked 1027 Read 4 Done Done Sent false
50872024-02-01T17:29:47.232Z 29 Acked 1028 Read 2 Done Done Sent false
50882024-02-01T17:29:47.232Z 30 Acked 1029 Read 9 Sent Done Sent false
50892024-02-01T17:29:47.232Z STATES DS:0 DS:1 DS:2 TOTAL
50902024-02-01T17:29:47.232Z New 0 0 0 0
50912024-02-01T17:29:47.232Z Sent 1 0 26 27
50922024-02-01T17:29:47.232Z Done 29 30 4 63
50932024-02-01T17:29:47.232Z Skipped 0 0 0 0
50942024-02-01T17:29:47.232Z Error 0 0 0 0
50952024-02-01T17:29:47.233ZLast Flush: 1023 1023 0
50962024-02-01T17:29:47.233ZDownstairs last five completed:
50972024-02-01T17:29:47.234ZUpstairs last five completed: 30 29 28 27 26
50982024-02-01T17:29:47.234Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:46.112020709Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1313}
50992024-02-01T17:29:47.234ZWrote out file "/tmp/test_repair_verify.data"
51002024-02-01T17:29:47.234Z
51012024-02-01T17:29:47.234ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51022024-02-01T17:29:47.235ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51032024-02-01T17:29:47.235Z 0 000-019 39 39 39 53 53 53 F F F
51042024-02-01T17:29:47.235Z 1 020-039 44 44 44 61 61 61 F F F
51052024-02-01T17:29:47.235Z 2 040-059 44 44 44 61 61 61 F F F
51062024-02-01T17:29:47.235Z 3 060-079 44 44 44 61 61 61 F F F
51072024-02-01T17:29:47.235Z 4 080-099 44 44 44 61 61 61 F F F
51082024-02-01T17:29:47.235Z 5 100-119 43 43 43 60 60 60 F F F
51092024-02-01T17:29:47.235Z 6 120-139 44 44 44 61 61 61 F F F
51102024-02-01T17:29:47.235Z 7 140-159 44 44 44 61 61 61 F F F
51112024-02-01T17:29:47.235Z 8 160-179 40 40 40 56 56 56 F F F
51122024-02-01T17:29:47.235Z 9 180-199 44 44 44 61 61 61 F F F
51132024-02-01T17:29:47.235Z 10 200-219 46 46 42 64 64 58 T T F <---
51142024-02-01T17:29:47.235Z 11 220-239 39 39 39 53 53 53 F F F
51152024-02-01T17:29:47.235Z 12 240-259 46 46 43 62 62 60 F F T <---
51162024-02-01T17:29:47.236Z 13 260-279 46 46 37 62 62 51 F F T <---
51172024-02-01T17:29:47.236Z 14 280-299 38 38 38 52 52 52 T T F <---
51182024-02-01T17:29:47.236Z 15 300-319 39 39 39 53 53 53 F F F
51192024-02-01T17:29:47.236Z 16 320-339 46 46 44 62 62 61 F F F <---
51202024-02-01T17:29:47.236Z 17 340-359 36 36 36 50 50 50 F F F
51212024-02-01T17:29:47.237Z 18 360-379 44 44 44 61 61 61 F F F
51222024-02-01T17:29:47.237Z 19 380-399 44 44 44 61 61 61 F F F
51232024-02-01T17:29:47.237Z 20 400-419 44 44 44 61 61 61 F F F
51242024-02-01T17:29:47.237Z 21 420-439 41 41 41 57 57 57 F F F
51252024-02-01T17:29:47.237Z 22 440-459 46 46 34 65 65 48 F F F <---
51262024-02-01T17:29:47.243Z 23 460-479 46 46 37 65 65 51 F F F <---
51272024-02-01T17:29:47.244Z 24 480-499 46 46 40 65 65 55 F F F <---
51282024-02-01T17:29:47.244Z 25 500-519 40 40 40 56 56 56 T T F <---
51292024-02-01T17:29:47.244Z 26 520-539 34 34 34 48 48 48 T T F <---
51302024-02-01T17:29:47.244Z 27 540-559 46 46 44 65 65 61 F F F <---
51312024-02-01T17:29:47.244Z 28 560-579 43 43 43 60 60 60 F F F
51322024-02-01T17:29:47.244Z 29 580-599 46 46 40 63 63 56 F F T <---
51332024-02-01T17:29:47.244ZMax gen: 46, Max flush: 65
51342024-02-01T17:29:47.244ZError: Difference in extent metadata found!
51352024-02-01T17:29:47.244ZA Difference in extent metadata is expected here
51362024-02-01T17:29:47.245ZOn loop 23
51372024-02-01T17:29:47.245Z
51382024-02-01T17:29:47.245ZVerifying data now
51392024-02-01T17:29:49.030ZNo change after verify
51402024-02-01T17:29:49.030ZLoop: 23 Downstairs dump after verify (and repair):
51412024-02-01T17:29:49.037ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51422024-02-01T17:29:49.037Z 0 000-019 39 39 39 53 53 53 F F F
51432024-02-01T17:29:49.037Z 1 020-039 44 44 44 61 61 61 F F F
51442024-02-01T17:29:49.037Z 2 040-059 44 44 44 61 61 61 F F F
51452024-02-01T17:29:49.037Z 3 060-079 44 44 44 61 61 61 F F F
51462024-02-01T17:29:49.037Z 4 080-099 44 44 44 61 61 61 F F F
51472024-02-01T17:29:49.038Z 5 100-119 43 43 43 60 60 60 F F F
51482024-02-01T17:29:49.038Z 6 120-139 44 44 44 61 61 61 F F F
51492024-02-01T17:29:49.038Z 7 140-159 44 44 44 61 61 61 F F F
51502024-02-01T17:29:49.038Z 8 160-179 40 40 40 56 56 56 F F F
51512024-02-01T17:29:49.038Z 9 180-199 44 44 44 61 61 61 F F F
51522024-02-01T17:29:49.038Z 10 200-219 47 47 47 66 66 66 F F F
51532024-02-01T17:29:49.038Z 11 220-239 39 39 39 53 53 53 F F F
51542024-02-01T17:29:49.038Z 12 240-259 46 46 46 62 62 62 F F F
51552024-02-01T17:29:49.038Z 13 260-279 46 46 46 62 62 62 F F F
51562024-02-01T17:29:49.038Z 14 280-299 47 47 47 66 66 66 F F F
51572024-02-01T17:29:49.038Z 15 300-319 39 39 39 53 53 53 F F F
51582024-02-01T17:29:49.038Z 16 320-339 46 46 46 62 62 62 F F F
51592024-02-01T17:29:49.038Z 17 340-359 36 36 36 50 50 50 F F F
51602024-02-01T17:29:49.038Z 18 360-379 44 44 44 61 61 61 F F F
51612024-02-01T17:29:49.038Z 19 380-399 44 44 44 61 61 61 F F F
51622024-02-01T17:29:49.038Z 20 400-419 44 44 44 61 61 61 F F F
51632024-02-01T17:29:49.038Z 21 420-439 41 41 41 57 57 57 F F F
51642024-02-01T17:29:49.038Z 22 440-459 46 46 46 65 65 65 F F F
51652024-02-01T17:29:49.038Z 23 460-479 46 46 46 65 65 65 F F F
51662024-02-01T17:29:49.038Z 24 480-499 46 46 46 65 65 65 F F F
51672024-02-01T17:29:49.038Z 25 500-519 47 47 47 66 66 66 F F F
51682024-02-01T17:29:49.038Z 26 520-539 47 47 47 66 66 66 F F F
51692024-02-01T17:29:49.038Z 27 540-559 46 46 46 65 65 65 F F F
51702024-02-01T17:29:49.038Z 28 560-579 43 43 43 60 60 60 F F F
51712024-02-01T17:29:49.039Z 29 580-599 46 46 46 63 63 63 F F F
51722024-02-01T17:29:49.039ZMax gen: 47, Max flush: 66
51732024-02-01T17:29:49.039Z
51742024-02-01T17:29:49.039ZBegin loop 24
51752024-02-01T17:29:49.039ZDownstairs to restart: 0
51762024-02-01T17:29:49.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.260662955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
51772024-02-01T17:29:49.039Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261145557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
51782024-02-01T17:29:49.039Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261211381Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
51792024-02-01T17:29:49.039ZCrucible runtime is spawned
51802024-02-01T17:29:49.039Z{"msg":"Crucible 3f464580-9151-48bd-9682-14db31802f18 has session id: 4299ddbd-432e-4eaa-aeae-4af4accc39f8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261294247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51812024-02-01T17:29:49.039Z{"msg":"Upstairs opts: Upstairs UUID: 3f464580-9151-48bd-9682-14db31802f18, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261398096Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51822024-02-01T17:29:49.039Z{"msg":"Crucible stats registered with UUID: 3f464580-9151-48bd-9682-14db31802f18","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261473417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51832024-02-01T17:29:49.039Z{"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261544769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
51842024-02-01T17:29:49.039Z{"msg":"3f464580-9151-48bd-9682-14db31802f18 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261701204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51852024-02-01T17:29:49.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261788772Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51862024-02-01T17:29:49.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26190466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51872024-02-01T17:29:49.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.261958574Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51882024-02-01T17:29:49.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262150216Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51892024-02-01T17:29:49.039Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262199025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51902024-02-01T17:29:49.039Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262291773Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51912024-02-01T17:29:49.039Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.2623378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51922024-02-01T17:29:49.039Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262408304Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51932024-02-01T17:29:49.039Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262455082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51942024-02-01T17:29:49.039Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262503Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51952024-02-01T17:29:49.040Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262545049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51962024-02-01T17:29:49.040Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.262586921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"io task","client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51972024-02-01T17:29:49.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.263607696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51982024-02-01T17:29:49.040Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26367341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
51992024-02-01T17:29:49.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.263700816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52002024-02-01T17:29:49.040Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.263723833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52012024-02-01T17:29:49.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.264442939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52022024-02-01T17:29:49.040Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.264534781Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52032024-02-01T17:29:49.040Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26477489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52042024-02-01T17:29:49.040Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.264836312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52052024-02-01T17:29:49.041Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26559431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52062024-02-01T17:29:49.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.265763273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52072024-02-01T17:29:49.041Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.265863163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52082024-02-01T17:29:49.041Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26590738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52092024-02-01T17:29:49.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.265936995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52102024-02-01T17:29:49.041Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.265987261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52112024-02-01T17:29:49.041Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266045407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52122024-02-01T17:29:49.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266301175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52132024-02-01T17:29:49.041Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266373961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52142024-02-01T17:29:49.041Z{"msg":"[0]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 66, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266404062Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52152024-02-01T17:29:49.041Z{"msg":"[0]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 47, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26642758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52162024-02-01T17:29:49.041Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266451341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52172024-02-01T17:29:49.041Z{"msg":"[1]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 66, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266502588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52182024-02-01T17:29:49.041Z{"msg":"[1]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 47, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266530165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52192024-02-01T17:29:49.041Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266553797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52202024-02-01T17:29:49.041Z{"msg":"[2]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 61, 66, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266595581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52212024-02-01T17:29:49.042Z{"msg":"[2]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 44, 47, 39]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266620128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52222024-02-01T17:29:49.042Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266643802Z","hostname":"Wait for a query_work_queue command to finish before sending IO
52232024-02-01T17:29:49.042Zip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52242024-02-01T17:29:49.042Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266730913Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52252024-02-01T17:29:49.042Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266778115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52262024-02-01T17:29:49.042Z{"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266802382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52272024-02-01T17:29:49.042Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266824719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"":"downstairs","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52282024-02-01T17:29:49.042Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266864967Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52292024-02-01T17:29:49.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26688704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"0","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52302024-02-01T17:29:49.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266909212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"1","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52312024-02-01T17:29:49.042Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266962009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"client":"2","":"downstairs","session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52322024-02-01T17:29:49.042Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.266986064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52332024-02-01T17:29:49.042Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.267068901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52342024-02-01T17:29:49.042Z{"msg":"3f464580-9151-48bd-9682-14db31802f18 is now active with session: 4299ddbd-432e-4eaa-aeae-4af4accc39f8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.267095058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52352024-02-01T17:29:49.042Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.267152218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321,"session_id":"4299ddbd-432e-4eaa-aeae-4af4accc39f8"}
52362024-02-01T17:29:49.042Z{"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:48.26717491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
52372024-02-01T17:29:49.042ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52382024-02-01T17:29:49.042ZLoading write count information from file "/tmp/test_repair_verify.data"
52392024-02-01T17:29:49.042ZRun Repair workload
52402024-02-01T17:29:49.042Z01/30 Write block 420 len 36864 data: 5 5 6 6 5 6 6 4 4
52412024-02-01T17:29:49.046Z02/30 Read block 152 len 28672
52422024-02-01T17:29:49.046Z03/30 Read block 568 len 40960
52432024-02-01T17:29:49.046Z04/30 Read block 564 len 36864
52442024-02-01T17:29:49.046Z05/30 Read block 402 len 4096
52452024-02-01T17:29:49.046Z06/30 Read block 367 len 36864
52462024-02-01T17:29:49.046Z07/30 Write block 276 len 12288 data: 6 4 4
52472024-02-01T17:29:49.046Z08/30 Write block 255 len 16384 data: 5 4 5 6
52482024-02-01T17:29:49.046Z09/30 Read block 404 len 36864
52492024-02-01T17:29:49.046Z10/30 Read block 268 len 40960
52502024-02-01T17:29:49.047Z11/30 Read block 230 len 4096
52512024-02-01T17:29:49.047Z12/30 Read block 587 len 36864
52522024-02-01T17:29:49.047Z13/30 Write block 447 len 8192 data: 4 4
52532024-02-01T17:29:49.047Z14/30 Read block 572 len 8192
52542024-02-01T17:29:49.047Z15/30 Flush
52552024-02-01T17:29:49.047Z16/30 Write block 327 len 40960 data: 4 4 4 5 6 7 8 10 10 9
52562024-02-01T17:29:49.047Z17/30 Read block 196 len 4096
52572024-02-01T17:29:49.047Z18/30 Read block 4 len 40960
52582024-02-01T17:29:49.047Z19/30 Read block 152 len 28672
52592024-02-01T17:29:49.047Z20/30 Write block 360 len 36864 data: 6 7 7 7 6 5 5 4 3
52602024-02-01T17:29:49.047Z21/30 Write block 270 len 8192 data: 6 6
52612024-02-01T17:29:49.048Z22/30 Read block 419 len 8192
52622024-02-01T17:29:49.048Z23/30 Read block 502 len 24576
52632024-02-01T17:29:49.048Z24/30 Read block 24 len 8192
52642024-02-01T17:29:49.048Z25/30 Write block 234 len 28672 data: 3 2 2 2 2 2 2
52652024-02-01T17:29:49.048Z26/30 Write block 523 len 28672 data: 5 5 4 4 3 3 3
52662024-02-01T17:29:49.048Z27/30 Read block 292 len 32768
52672024-02-01T17:29:49.048Z28/30 Read block 454 len 32768
52682024-02-01T17:29:49.048Z29/30 Write block 182 len 20480 data: 7 9 8 9 7
52692024-02-01T17:29:49.048Z30/30 Read block 23 len 32768
52702024-02-01T17:29:49.051Z----------------------------------------------------------------
52712024-02-01T17:29:49.051Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
52722024-02-01T17:29:49.052ZStates: Active Active Active
52732024-02-01T17:29:49.052ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52742024-02-01T17:29:49.052Z 1 Acked 1000 Write 9 Done Done Done false
52752024-02-01T17:29:49.052Z 2 Acked 1001 Read 7 Done Done Done false
52762024-02-01T17:29:49.052Z 3 Acked 1002 Read 10 Sent Done Done false
52772024-02-01T17:29:49.052Z 4 Acked 1003 Read 9 Sent Done Done false
52782024-02-01T17:29:49.052Z 5 Acked 1004 Read 1 Sent Done Done false
52792024-02-01T17:29:49.052Z 6 Acked 1005 Read 9 Sent Done Done false
52802024-02-01T17:29:49.052Z 7 Acked 1006 Write 3 Sent Done Done false
52812024-02-01T17:29:49.052Z 8 Acked 1007 Write 4 Sent Done Done false
52822024-02-01T17:29:49.052Z 9 Acked 1008 Read 9 Sent Done Done false
52832024-02-01T17:29:49.052Z 10 Acked 1009 Read 10 Sent Done Done false
52842024-02-01T17:29:49.052Z 11 Acked 1010 Read 1 Sent Done Done false
52852024-02-01T17:29:49.052Z 12 Acked 1011 Read 9 Sent Done Done false
52862024-02-01T17:29:49.052Z 13 Acked 1012 Write 2 Sent Done Done false
52872024-02-01T17:29:49.052Z 14 Acked 1013 Read 2 Sent Done Done false
52882024-02-01T17:29:49.052Z 15 Acked 1014 Flush 0 Sent Done Done false
52892024-02-01T17:29:49.052Z 16 Acked 1015 Write 10 Sent Done Done false
52902024-02-01T17:29:49.052Z 17 Acked 1016 Read 1 Sent Done Done false
52912024-02-01T17:29:49.052Z 18 Acked 1017 Read 10 Sent Done Done false
52922024-02-01T17:29:49.052Z 19 Acked 1018 Read 7 Sent Done Done false
52932024-02-01T17:29:49.052Z 20 Acked 1019 Write 9 Sent Done Done false
52942024-02-01T17:29:49.053Z 21 Acked 1020 Write 2 Sent Done Done false
52952024-02-01T17:29:49.053Z 22 Acked 1021 Read 2 Sent Done Done false
52962024-02-01T17:29:49.053Z 23 Acked 1022 Read 6 Sent Done Done false
52972024-02-01T17:29:49.053Z 24 Acked 1023 Read 2 Sent Done Done false
52982024-02-01T17:29:49.053Z 25 Acked 1024 Write 7 Sent Done Done false
52992024-02-01T17:29:49.053Z 26 Acked 1025 Write 7 Sent Done Done false
53002024-02-01T17:29:49.053Z 27 Acked 1026 Read 8 Sent Done Done false
53012024-02-01T17:29:49.053Z 28 Acked 1027 Read 8 Sent Done Done false
53022024-02-01T17:29:49.053Z 29 Acked 1028 Write 5 Sent Done Done false
53032024-02-01T17:29:49.053Z 30 Acked 1029 Read 8 Sent Sent Done false
53042024-02-01T17:29:49.053Z STATES DS:0 DS:1 DS:2 TOTAL
53052024-02-01T17:29:49.053Z New 0 0 0 0
53062024-02-01T17:29:49.053Z Sent 28 1 0 29
53072024-02-01T17:29:49.053Z Done 2 29 30 61
53082024-02-01T17:29:49.053Z Skipped 0 0 0 0
53092024-02-01T17:29:49.053Z Error 0 0 0 0
53102024-02-01T17:29:49.053ZLast Flush: 0 1014 1014
53112024-02-01T17:29:49.053ZDownstairs last five completed:
53122024-02-01T17:29:49.053ZUpstairs last five completed: 30 29 28 27 26
53132024-02-01T17:29:49.053Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:48.314540387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1321}
53142024-02-01T17:29:49.053ZWrote out file "/tmp/test_repair_verify.data"
53152024-02-01T17:29:49.057Z
53162024-02-01T17:29:49.062ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53172024-02-01T17:29:49.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53182024-02-01T17:29:49.091Z 0 000-019 39 39 39 53 53 53 F F F
53192024-02-01T17:29:49.091Z 1 020-039 44 44 44 61 61 61 F F F
53202024-02-01T17:29:49.091Z 2 040-059 44 44 44 61 61 61 F F F
53212024-02-01T17:29:49.091Z 3 060-079 44 44 44 61 61 61 F F F
53222024-02-01T17:29:49.095Z 4 080-099 44 44 44 61 61 61 F F F
53232024-02-01T17:29:49.095Z 5 100-119 43 43 43 60 60 60 F F F
53242024-02-01T17:29:49.095Z 6 120-139 44 44 44 61 61 61 F F F
53252024-02-01T17:29:49.095Z 7 140-159 44 44 44 61 61 61 F F F
53262024-02-01T17:29:49.095Z 8 160-179 40 40 40 56 56 56 F F F
53272024-02-01T17:29:49.095Z 9 180-199 44 44 44 61 61 61 F T T <---
53282024-02-01T17:29:49.095Z 10 200-219 47 47 47 66 66 66 F F F
53292024-02-01T17:29:49.096Z 11 220-239 39 39 39 53 53 53 F T T <---
53302024-02-01T17:29:49.096Z 12 240-259 46 48 48 62 67 67 F T T <---
53312024-02-01T17:29:49.096Z 13 260-279 46 48 48 62 67 67 F T T <---
53322024-02-01T17:29:49.096Z 14 280-299 47 47 47 66 66 66 F F F
53332024-02-01T17:29:49.096Z 15 300-319 39 39 39 53 53 53 F F F
53342024-02-01T17:29:49.096Z 16 320-339 46 46 46 62 62 62 F T T <---
53352024-02-01T17:29:49.096Z 17 340-359 36 36 36 50 50 50 F F F
53362024-02-01T17:29:49.096Z 18 360-379 44 44 44 61 61 61 F T T <---
53372024-02-01T17:29:49.096Z 19 380-399 44 44 44 61 61 61 F F F
53382024-02-01T17:29:49.096Z 20 400-419 44 44 44 61 61 61 F F F
53392024-02-01T17:29:49.096Z 21 420-439 41 48 48 57 67 67 T F F <---
53402024-02-01T17:29:49.096Z 22 440-459 46 48 48 65 67 67 F F F <---
53412024-02-01T17:29:49.097Z 23 460-479 46 46 46 65 65 65 F F F
53422024-02-01T17:29:49.097Z 24 480-499 46 46 46 65 65 65 F F F
53432024-02-01T17:29:49.097Z 25 500-519 47 47 47 66 66 66 F F F
53442024-02-01T17:29:49.097Z 26 520-539 47 47 47 66 66 66 F T T <---
53452024-02-01T17:29:49.097Z 27 540-559 46 46 46 65 65 65 F F F
53462024-02-01T17:29:49.097Z 28 560-579 43 43 43 60 60 60 F F F
53472024-02-01T17:29:49.097Z 29 580-599 46 46 46 63 63 63 F F F
53482024-02-01T17:29:49.097ZMax gen: 48, Max flush: 67
53492024-02-01T17:29:49.097ZError: Difference in extent metadata found!
53502024-02-01T17:29:49.100ZA Difference in extent metadata is expected here
53512024-02-01T17:29:49.100ZOn loop 24
53522024-02-01T17:29:49.101Z
53532024-02-01T17:29:49.101ZVerifying data now
53542024-02-01T17:29:54.341ZNo change after verify
53552024-02-01T17:29:54.341ZLoop: 24 Downstairs dump after verify (and repair):
53562024-02-01T17:29:54.764ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53572024-02-01T17:29:54.764Z 0 000-019 39 39 39 53 53 53 F F F
53582024-02-01T17:29:54.764Z 1 020-039 44 44 44 61 61 61 F F F
53592024-02-01T17:29:54.764Z 2 040-059 44 44 44 61 61 61 F F F
53602024-02-01T17:29:54.764Z 3 060-079 44 44 44 61 61 61 F F F
53612024-02-01T17:29:54.764Z 4 080-099 44 44 44 61 61 61 F F F
53622024-02-01T17:29:54.764Z 5 100-119 43 43 43 60 60 60 F F F
53632024-02-01T17:29:54.764Z 6 120-139 44 44 44 61 61 61 F F F
53642024-02-01T17:29:54.764Z 7 140-159 44 44 44 61 61 61 F F F
53652024-02-01T17:29:54.764Z 8 160-179 40 40 40 56 56 56 F F F
53662024-02-01T17:29:54.765Z 9 180-199 49 49 49 68 68 68 F F F
53672024-02-01T17:29:54.765Z 10 200-219 47 47 47 66 66 66 F F F
53682024-02-01T17:29:54.765Z 11 220-239 49 49 49 68 68 68 F F F
53692024-02-01T17:29:54.765Z 12 240-259 49 49 49 68 68 68 F F F
53702024-02-01T17:29:54.765Z 13 260-279 49 49 49 68 68 68 F F F
53712024-02-01T17:29:54.765Z 14 280-299 47 47 47 66 66 66 F F F
53722024-02-01T17:29:54.765Z 15 300-319 39 39 39 53 53 53 F F F
53732024-02-01T17:29:54.765Z 16 320-339 49 49 49 68 68 68 F F F
53742024-02-01T17:29:54.765Z 17 340-359 36 36 36 50 50 50 F F F
53752024-02-01T17:29:54.765Z 18 360-379 49 49 49 68 68 68 F F F
53762024-02-01T17:29:54.765Z 19 380-399 44 44 44 61 61 61 F F F
53772024-02-01T17:29:54.765Z 20 400-419 44 44 44 61 61 61 F F F
53782024-02-01T17:29:54.765Z 21 420-439 48 48 48 67 67 67 F F F
53792024-02-01T17:29:54.766Z 22 440-459 48 48 48 67 67 67 F F F
53802024-02-01T17:29:54.766Z 23 460-479 46 46 46 65 65 65 F F F
53812024-02-01T17:29:54.766Z 24 480-499 46 46 46 65 65 65 F F F
53822024-02-01T17:29:54.766Z 25 500-519 47 47 47 66 66 66 F F F
53832024-02-01T17:29:54.766Z 26 520-539 49 49 49 68 68 68 F F F
53842024-02-01T17:29:54.766Z 27 540-559 46 46 46 65 65 65 F F F
53852024-02-01T17:29:54.766Z 28 560-579 43 43 43 60 60 60 F F F
53862024-02-01T17:29:54.766Z 29 580-599 46 46 46 63 63 63 F F F
53872024-02-01T17:29:54.766ZMax gen: 49, Max flush: 68
53882024-02-01T17:29:54.766Z
53892024-02-01T17:29:54.766ZBegin loop 25
53902024-02-01T17:29:54.766ZDownstairs to restart: 1
53912024-02-01T17:29:54.767Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.258539732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
53922024-02-01T17:29:54.767Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259035273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
53932024-02-01T17:29:54.767Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259178022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
53942024-02-01T17:29:54.767Z{"msg":"Crucible 3d782865-fdc5-437a-8072-c49f67eeffee has session id: 6cddbbba-5b62-4af2-836d-ef7ee1dd3275","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.25927444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
53952024-02-01T17:29:54.767Z{"msg":"Upstairs opts: Upstairs UUID: 3d782865-fdc5-437a-8072-c49f67eeffee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259338364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
53962024-02-01T17:29:54.767Z{"msg":"Crucible stats registered with UUID: 3d782865-fdc5-437a-8072-c49f67eeffee","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259381605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
53972024-02-01T17:29:54.767ZCrucible runtime is spawned
53982024-02-01T17:29:54.767Z{"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259704901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
53992024-02-01T17:29:54.767Z{"msg":"3d782865-fdc5-437a-8072-c49f67eeffee active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259836851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54002024-02-01T17:29:54.767Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.259922669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54012024-02-01T17:29:54.767Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260167376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54022024-02-01T17:29:54.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260228689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54032024-02-01T17:29:54.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260270284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54042024-02-01T17:29:54.774Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260296678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54052024-02-01T17:29:54.774Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260337986Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54062024-02-01T17:29:54.774Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260437329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54072024-02-01T17:29:54.774Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260482285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54082024-02-01T17:29:54.774Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260612842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54092024-02-01T17:29:54.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260709885Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54102024-02-01T17:29:54.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.260757837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54112024-02-01T17:29:54.774Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26080197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"io task","client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54122024-02-01T17:29:54.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.261274762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54132024-02-01T17:29:54.774Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.261360843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54142024-02-01T17:29:54.774Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.261406836Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54152024-02-01T17:29:54.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.261450094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54162024-02-01T17:29:54.775Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.262256854Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54172024-02-01T17:29:54.775Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.262321137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54182024-02-01T17:29:54.775Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.262482259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54192024-02-01T17:29:54.775Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.262548026Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54202024-02-01T17:29:54.775Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26315705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54212024-02-01T17:29:54.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.263611306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54222024-02-01T17:29:54.775Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26379564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54232024-02-01T17:29:54.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.263885937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54242024-02-01T17:29:54.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.263932605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54252024-02-01T17:29:54.775Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264002537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54262024-02-01T17:29:54.775Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264033337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54272024-02-01T17:29:54.775Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264053912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54282024-02-01T17:29:54.775Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26409603Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54292024-02-01T17:29:54.775Z{"msg":"[0]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 68, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264121799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54302024-02-01T17:29:54.775Z{"msg":"[0]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 49, 47, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264145024Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54312024-02-01T17:29:54.777Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264187352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54322024-02-01T17:29:54.777Z{"msg":"[1]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 68, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264212421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54332024-02-01T17:29:54.777Z{"msg":"[1]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 49, 47, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264235309Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54342024-02-01T17:29:54.777Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264258598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54352024-02-01T17:29:54.777Z{"msg":"[2]R flush_numbers[0..12]: [53, 61, 61, 61, 61, 60, 61, 61, 56, 68, 66, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264282481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54362024-02-01T17:29:54.778ZWait for a query_work_queue command to finish before sending IO
54372024-02-01T17:29:54.778Z{"msg":"[2]R generation[0..12]: [39, 44, 44, 44, 44, 43, 44, 44, 40, 49, 47, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264325476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54382024-02-01T17:29:54.778Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264351152Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54392024-02-01T17:29:54.778Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264372816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54402024-02-01T17:29:54.778Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264421033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54412024-02-01T17:29:54.778Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264444288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54422024-02-01T17:29:54.778Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264466284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"":"downstairs","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54432024-02-01T17:29:54.778Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264527652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54442024-02-01T17:29:54.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26454979Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"0","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54452024-02-01T17:29:54.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264571894Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"1","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54462024-02-01T17:29:54.778Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264593312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"client":"2","":"downstairs","session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54472024-02-01T17:29:54.778Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264615087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54482024-02-01T17:29:54.778Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264635846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54492024-02-01T17:29:54.779Z{"msg":"3d782865-fdc5-437a-8072-c49f67eeffee is now active with session: 6cddbbba-5b62-4af2-836d-ef7ee1dd3275","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264700762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54502024-02-01T17:29:54.779Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.26472247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329,"session_id":"6cddbbba-5b62-4af2-836d-ef7ee1dd3275"}
54512024-02-01T17:29:54.779Z{"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:53.264743428Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
54522024-02-01T17:29:54.779ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54532024-02-01T17:29:54.779ZLoading write count information from file "/tmp/test_repair_verify.data"
54542024-02-01T17:29:54.779ZRun Repair workload
54552024-02-01T17:29:54.780Z01/30 Write block 534 len 16384 data: 5 4 4 5
54562024-02-01T17:29:54.785Z02/30 Read block 227 len 16384
54572024-02-01T17:29:54.785Z03/30 Flush
54582024-02-01T17:29:54.785Z04/30 Write block 4 len 16384 data: 4 4 4 3
54592024-02-01T17:29:54.785Z05/30 Read block 228 len 36864
54602024-02-01T17:29:54.785Z06/30 Read block 264 len 24576
54612024-02-01T17:29:54.785Z07/30 Read block 129 len 40960
54622024-02-01T17:29:54.785Z08/30 Write block 137 len 8192 data: 6 5
54632024-02-01T17:29:54.786Z09/30 Read block 595 len 20480
54642024-02-01T17:29:54.786Z10/30 Write block 105 len 12288 data: 5 5 5
54652024-02-01T17:29:54.786Z11/30 Read block 100 len 24576
54662024-02-01T17:29:54.786Z12/30 Write block 278 len 36864 data: 5 3 3 3 2 3 3 5 6
54672024-02-01T17:29:54.786Z13/30 Read block 503 len 8192
54682024-02-01T17:29:54.786Z14/30 Write block 108 len 8192 data: 5 5
54692024-02-01T17:29:54.786Z15/30 Flush
54702024-02-01T17:29:54.786Z16/30 Write block 106 len 12288 data: 6 6 6
54712024-02-01T17:29:54.786Z17/30 Write block 313 len 40960 data: 6 5 7 7 7 5 5 5 4 4
54722024-02-01T17:29:54.786Z18/30 Read block 298 len 36864
54732024-02-01T17:29:54.786Z19/30 Read block 1 len 4096
54742024-02-01T17:29:54.786Z20/30 Read block 545 len 24576
54752024-02-01T17:29:54.786Z21/30 Flush
54762024-02-01T17:29:54.786Z22/30 Flush
54772024-02-01T17:29:54.786Z23/30 Write block 72 len 20480 data: 6 7 7 7 3
54782024-02-01T17:29:54.786Z24/30 Write block 403 len 24576 data: 5 4 4 4 4 4
54792024-02-01T17:29:54.786Z25/30 Write block 168 len 24576 data: 4 5 6 7 7 7
54802024-02-01T17:29:54.786Z26/30 Read block 501 len 4096
54812024-02-01T17:29:54.786Z27/30 Read block 424 len 36864
54822024-02-01T17:29:54.786Z28/30 Write block 285 len 20480 data: 6 7 5 5 5
54832024-02-01T17:29:54.786Z29/30 Write block 207 len 20480 data: 10 10 8 5 6
54842024-02-01T17:29:54.786Z30/30 Write block 6 len 20480 data: 5 4 4 2 3
54852024-02-01T17:29:54.787Z----------------------------------------------------------------
54862024-02-01T17:29:54.787Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:27
54872024-02-01T17:29:54.787ZStates: Active Active Active
54882024-02-01T17:29:54.787ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
54892024-02-01T17:29:54.787Z 4 Acked 1003 Write 4 Done Sent Done false
54902024-02-01T17:29:54.787Z 5 Acked 1004 Read 9 Done Sent Done false
54912024-02-01T17:29:54.787Z 6 Acked 1005 Read 6 Done Sent Done false
54922024-02-01T17:29:54.787Z 7 Acked 1006 Read 10 Done Sent Done false
54932024-02-01T17:29:54.787Z 8 Acked 1007 Write 2 Done Sent Done false
54942024-02-01T17:29:54.787Z 9 Acked 1008 Read 5 Done Sent Done false
54952024-02-01T17:29:54.787Z 10 Acked 1009 Write 3 Done Sent Done false
54962024-02-01T17:29:54.787Z 11 Acked 1010 Read 6 Done Sent Done false
54972024-02-01T17:29:54.787Z 12 Acked 1011 Write 9 Done Sent Done false
54982024-02-01T17:29:54.788Z 13 Acked 1012 Read 2 Done Sent Done false
54992024-02-01T17:29:54.788Z 14 Acked 1013 Write 2 Done Sent Done false
55002024-02-01T17:29:54.788Z 15 Acked 1014 Flush 0 Done Sent Done false
55012024-02-01T17:29:54.788Z 16 Acked 1015 Write 3 Done Sent Done false
55022024-02-01T17:29:54.788Z 17 Acked 1016 Write 10 Done Sent Done false
55032024-02-01T17:29:54.788Z 18 Acked 1017 Read 9 Done Sent Done false
55042024-02-01T17:29:54.788Z 19 Acked 1018 Read 1 Done Sent Done false
55052024-02-01T17:29:54.788Z 20 Acked 1019 Read 6 Done Sent Done false
55062024-02-01T17:29:54.788Z 21 Acked 1020 Flush 0 Done Sent Done false
55072024-02-01T17:29:54.788Z 22 Acked 1021 Flush 0 Done Sent Done false
55082024-02-01T17:29:54.788Z 23 Acked 1022 Write 5 Done Sent Done false
55092024-02-01T17:29:54.788Z 24 Acked 1023 Write 6 Done Sent Done false
55102024-02-01T17:29:54.788Z 25 Acked 1024 Write 6 Done Sent Done false
55112024-02-01T17:29:54.788Z 26 Acked 1025 Read 1 Done Sent Done false
55122024-02-01T17:29:54.788Z 27 Acked 1026 Read 9 Done Sent Done false
55132024-02-01T17:29:54.788Z 28 Acked 1027 Write 5 Done Sent Done false
55142024-02-01T17:29:54.788Z 29 Acked 1028 Write 5 Done Sent Done false
55152024-02-01T17:29:54.788Z 30 Acked 1029 Write 5 Sent Sent Sent false
55162024-02-01T17:29:54.788Z STATES DS:0 DS:1 DS:2 TOTAL
55172024-02-01T17:29:54.789Z New 0 0 0 0
55182024-02-01T17:29:54.789Z Sent 1 27 1 29
55192024-02-01T17:29:54.789Z Done 26 0 26 52
55202024-02-01T17:29:54.789Z Skipped 0 0 0 0
55212024-02-01T17:29:54.789Z Error 0 0 0 0
55222024-02-01T17:29:54.789ZLast Flush: 1021 1002 1021
55232024-02-01T17:29:54.789ZDownstairs last five completed: 1002 1001 1000
55242024-02-01T17:29:54.790ZUpstairs last five completed: 30 29 28 27 26
55252024-02-01T17:29:54.790Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:53.312185486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1329}
55262024-02-01T17:29:54.790ZWrote out file "/tmp/test_repair_verify.data"
55272024-02-01T17:29:54.790Z
55282024-02-01T17:29:54.790ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55292024-02-01T17:29:54.790ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55302024-02-01T17:29:54.790Z 0 000-019 50 39 50 70 53 70 T F T <---
55312024-02-01T17:29:54.790Z 1 020-039 44 44 44 61 61 61 F F F
55322024-02-01T17:29:54.790Z 2 040-059 44 44 44 61 61 61 F F F
55332024-02-01T17:29:54.790Z 3 060-079 44 44 44 61 61 61 T F T <---
55342024-02-01T17:29:54.790Z 4 080-099 44 44 44 61 61 61 F F F
55352024-02-01T17:29:54.790Z 5 100-119 50 43 50 71 60 71 F F F <---
55362024-02-01T17:29:54.790Z 6 120-139 50 44 50 70 61 70 F F F <---
55372024-02-01T17:29:54.790Z 7 140-159 44 44 44 61 61 61 F F F
55382024-02-01T17:29:54.790Z 8 160-179 40 40 40 56 56 56 T F T <---
55392024-02-01T17:29:54.790Z 9 180-199 49 49 49 68 68 68 F F F
55402024-02-01T17:29:54.790Z 10 200-219 47 47 47 66 66 66 T F T <---
55412024-02-01T17:29:54.790Z 11 220-239 49 49 49 68 68 68 F F F
55422024-02-01T17:29:54.790Z 12 240-259 49 49 49 68 68 68 F F F
55432024-02-01T17:29:54.790Z 13 260-279 50 49 50 70 68 70 F F F <---
55442024-02-01T17:29:54.790Z 14 280-299 50 47 50 70 66 70 T F T <---
55452024-02-01T17:29:54.790Z 15 300-319 50 39 50 71 53 71 F F F <---
55462024-02-01T17:29:54.790Z 16 320-339 50 49 50 71 68 71 F F F <---
55472024-02-01T17:29:54.790Z 17 340-359 36 36 36 50 50 50 F F F
55482024-02-01T17:29:54.790Z 18 360-379 49 49 49 68 68 68 F F F
55492024-02-01T17:29:54.791Z 19 380-399 44 44 44 61 61 61 F F F
55502024-02-01T17:29:54.791Z 20 400-419 44 44 44 61 61 61 T F T <---
55512024-02-01T17:29:54.791Z 21 420-439 48 48 48 67 67 67 F F F
55522024-02-01T17:29:54.791Z 22 440-459 48 48 48 67 67 67 F F F
55532024-02-01T17:29:54.791Z 23 460-479 46 46 46 65 65 65 F F F
55542024-02-01T17:29:54.791Z 24 480-499 46 46 46 65 65 65 F F F
55552024-02-01T17:29:54.791Z 25 500-519 47 47 47 66 66 66 F F F
55562024-02-01T17:29:54.794Z 26 520-539 50 50 50 69 69 69 F F F
55572024-02-01T17:29:54.794Z 27 540-559 46 46 46 65 65 65 F F F
55582024-02-01T17:29:54.794Z 28 560-579 43 43 43 60 60 60 F F F
55592024-02-01T17:29:54.794Z 29 580-599 46 46 46 63 63 63 F F F
55602024-02-01T17:29:54.794ZMax gen: 50, Max flush: 71
55612024-02-01T17:29:54.794ZError: Difference in extent metadata found!
55622024-02-01T17:29:54.795ZA Difference in extent metadata is expected here
55632024-02-01T17:29:54.795ZOn loop 25
55642024-02-01T17:29:54.795Z
55652024-02-01T17:29:54.795ZVerifying data now
55662024-02-01T17:29:59.068ZNo change after verify
55672024-02-01T17:29:59.068ZLoop: 25 Downstairs dump after verify (and repair):
55682024-02-01T17:29:59.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55692024-02-01T17:29:59.091Z 0 000-019 51 51 51 72 72 72 F F F
55702024-02-01T17:29:59.091Z 1 020-039 44 44 44 61 61 61 F F F
55712024-02-01T17:29:59.091Z 2 040-059 44 44 44 61 61 61 F F F
55722024-02-01T17:29:59.091Z 3 060-079 51 51 51 72 72 72 F F F
55732024-02-01T17:29:59.091Z 4 080-099 44 44 44 61 61 61 F F F
55742024-02-01T17:29:59.091Z 5 100-119 50 50 50 71 71 71 F F F
55752024-02-01T17:29:59.091Z 6 120-139 50 50 50 70 70 70 F F F
55762024-02-01T17:29:59.091Z 7 140-159 44 44 44 61 61 61 F F F
55772024-02-01T17:29:59.099Z 8 160-179 51 51 51 72 72 72 F F F
55782024-02-01T17:29:59.100Z 9 180-199 49 49 49 68 68 68 F F F
55792024-02-01T17:29:59.100Z 10 200-219 51 51 51 72 72 72 F F F
55802024-02-01T17:29:59.100Z 11 220-239 49 49 49 68 68 68 F F F
55812024-02-01T17:29:59.100Z 12 240-259 49 49 49 68 68 68 F F F
55822024-02-01T17:29:59.100Z 13 260-279 50 50 50 70 70 70 F F F
55832024-02-01T17:29:59.100Z 14 280-299 51 51 51 72 72 72 F F F
55842024-02-01T17:29:59.100Z 15 300-319 50 50 50 71 71 71 F F F
55852024-02-01T17:29:59.100Z 16 320-339 50 50 50 71 71 71 F F F
55862024-02-01T17:29:59.100Z 17 340-359 36 36 36 50 50 50 F F F
55872024-02-01T17:29:59.101Z 18 360-379 49 49 49 68 68 68 F F F
55882024-02-01T17:29:59.101Z 19 380-399 44 44 44 61 61 61 F F F
55892024-02-01T17:29:59.101Z 20 400-419 51 51 51 72 72 72 F F F
55902024-02-01T17:29:59.101Z 21 420-439 48 48 48 67 67 67 F F F
55912024-02-01T17:29:59.101Z 22 440-459 48 48 48 67 67 67 F F F
55922024-02-01T17:29:59.101Z 23 460-479 46 46 46 65 65 65 F F F
55932024-02-01T17:29:59.101Z 24 480-499 46 46 46 65 65 65 F F F
55942024-02-01T17:29:59.103Z 25 500-519 47 47 47 66 66 66 F F F
55952024-02-01T17:29:59.103Z 26 520-539 50 50 50 69 69 69 F F F
55962024-02-01T17:29:59.103Z 27 540-559 46 46 46 65 65 65 F F F
55972024-02-01T17:29:59.103Z 28 560-579 43 43 43 60 60 60 F F F
55982024-02-01T17:29:59.103Z 29 580-599 46 46 46 63 63 63 F F F
55992024-02-01T17:29:59.103ZMax gen: 51, Max flush: 72
56002024-02-01T17:29:59.103Z
56012024-02-01T17:29:59.103ZBegin loop 26
56022024-02-01T17:29:59.103ZDownstairs to restart: 1
56032024-02-01T17:29:59.123Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.389147837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
56042024-02-01T17:29:59.123Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.389579247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
56052024-02-01T17:29:59.127Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.389686066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
56062024-02-01T17:29:59.127Z{"msg":"Crucible 6d95dad8-a222-47e1-b3bb-b44ab56a5362 has session id: a3b5f5a2-c1db-49b6-8cc2-bea543766902","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.389879645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56072024-02-01T17:29:59.127Z{"msg":"Crucible runtime is spawned
56082024-02-01T17:29:59.127ZUpstairs opts: Upstairs UUID: 6d95dad8-a222-47e1-b3bb-b44ab56a5362, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390091975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56092024-02-01T17:29:59.127Z{"msg":"Crucible stats registered with UUID: 6d95dad8-a222-47e1-b3bb-b44ab56a5362","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390182768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56102024-02-01T17:29:59.127Z{"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39025515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
56112024-02-01T17:29:59.127Z{"msg":"6d95dad8-a222-47e1-b3bb-b44ab56a5362 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390332031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56122024-02-01T17:29:59.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390408908Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56132024-02-01T17:29:59.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390491625Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56142024-02-01T17:29:59.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390590929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56152024-02-01T17:29:59.127Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390680714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56162024-02-01T17:29:59.127Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390731705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56172024-02-01T17:29:59.128Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.390920717Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56182024-02-01T17:29:59.129Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391060556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56192024-02-01T17:29:59.129Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391155014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56202024-02-01T17:29:59.129Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391202036Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56212024-02-01T17:29:59.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391257635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56222024-02-01T17:29:59.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391283929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56232024-02-01T17:29:59.129Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.391306485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"io task","client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56242024-02-01T17:29:59.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39248571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56252024-02-01T17:29:59.129Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39263555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56262024-02-01T17:29:59.129Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.392683156Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56272024-02-01T17:29:59.129Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.392726138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56282024-02-01T17:29:59.134Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.393008415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56292024-02-01T17:29:59.134Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.393081501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56302024-02-01T17:29:59.134Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.393601835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56312024-02-01T17:29:59.134Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.393678072Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56322024-02-01T17:29:59.134Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394287857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56332024-02-01T17:29:59.134Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394571731Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56342024-02-01T17:29:59.134Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394636579Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56352024-02-01T17:29:59.134Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394681071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56362024-02-01T17:29:59.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394741227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56372024-02-01T17:29:59.135Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39478545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56382024-02-01T17:29:59.135Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.394837938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56392024-02-01T17:29:59.135Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395189385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56402024-02-01T17:29:59.136Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.3952901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56412024-02-01T17:29:59.136Z{"msg":"[0]R flush_numbers[0..12]: [72, 61, 61, 72, 61, 71, 70, 61, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395382225Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56422024-02-01T17:29:59.136Z{"msg":"[0]R generation[0..12]: [51, 44, 44, 51, 44, 50, 50, 44, 51, 49, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395411443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56432024-02-01T17:29:59.136Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395455893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56442024-02-01T17:29:59.136Z{"msg":"[1]R flush_numbers[0..12]: [72, 61, 61, 72, 61, 71, 70, 61, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395481468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56452024-02-01T17:29:59.136Z{"msg":"[1]R generation[0..12]: [51, 44, 44, 51, 44, 50, 50, 44, 51, 49, 51, 49]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
56462024-02-01T17:29:59.136Z,"time":"2024-02-01T17:29:58.395541042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56472024-02-01T17:29:59.136Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395654803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56482024-02-01T17:29:59.136Z{"msg":"[2]R flush_numbers[0..12]: [72, 61, 61, 72, 61, 71, 70, 61, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395684974Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56492024-02-01T17:29:59.136Z{"msg":"[2]R generation[0..12]: [51, 44, 44, 51, 44, 50, 50, 44, 51, 49, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395740762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56502024-02-01T17:29:59.136Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395766686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56512024-02-01T17:29:59.136Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395788714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56522024-02-01T17:29:59.136Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395835016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56532024-02-01T17:29:59.136Z{"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395867054Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56542024-02-01T17:29:59.136Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395890191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"":"downstairs","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56552024-02-01T17:29:59.136Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395913679Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56562024-02-01T17:29:59.136Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395934025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"0","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56572024-02-01T17:29:59.137Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395955823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"1","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56582024-02-01T17:29:59.137Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39597738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"client":"2","":"downstairs","session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56592024-02-01T17:29:59.137Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.395999458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56602024-02-01T17:29:59.137Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.396019823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56612024-02-01T17:29:59.137Z{"msg":"6d95dad8-a222-47e1-b3bb-b44ab56a5362 is now active with session: a3b5f5a2-c1db-49b6-8cc2-bea543766902","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.396041423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56622024-02-01T17:29:59.137Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.39606127Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337,"session_id":"a3b5f5a2-c1db-49b6-8cc2-bea543766902"}
56632024-02-01T17:29:59.137Z{"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:29:58.396081424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
56642024-02-01T17:29:59.137ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56652024-02-01T17:29:59.137ZLoading write count information from file "/tmp/test_repair_verify.data"
56662024-02-01T17:29:59.137ZRun Repair workload
56672024-02-01T17:29:59.137Z01/30 Write block 523 len 12288 data: 6 6 5
56682024-02-01T17:29:59.141Z02/30 Read block 220 len 24576
56692024-02-01T17:29:59.141Z03/30 Write block 32 len 28672 data: 3 2 3 4 5 6 6
56702024-02-01T17:29:59.141Z04/30 Write block 50 len 8192 data: 6 6
56712024-02-01T17:29:59.141Z05/30 Write block 566 len 20480 data: 4 3 3 5 4
56722024-02-01T17:29:59.141Z06/30 Write block 251 len 36864 data: 5 4 3 6 6 5 6 7 6
56732024-02-01T17:29:59.142Z07/30 Read block 264 len 40960
56742024-02-01T17:29:59.147Z08/30 Write block 317 len 32768 data: 8 6 6 6 5 5 3 4
56752024-02-01T17:29:59.147Z09/30 Read block 140 len 32768
56762024-02-01T17:29:59.147Z10/30 Write block 551 len 12288 data: 6 6 5
56772024-02-01T17:29:59.150Z11/30 Write block 494 len 20480 data: 5 4 4 2 2
56782024-02-01T17:29:59.150Z12/30 Write block 266 len 12288 data: 5 5 5
56792024-02-01T17:29:59.150Z13/30 Write block 59 len 12288 data: 5 6 5
56802024-02-01T17:29:59.151Z14/30 Read block 255 len 36864
56812024-02-01T17:29:59.162Z15/30 Read block 589 len 8192
56822024-02-01T17:29:59.162Z16/30 Write block 533 len 40960 data: 4 6 5 5 6 4 4 6 9 9
56832024-02-01T17:29:59.188Z17/30 Read block 262 len 40960
56842024-02-01T17:29:59.189Z18/30 Write block 367 len 24576 data: 5 4 3 4 4 4
56852024-02-01T17:29:59.189Z19/30 Read block 571 len 28672
56862024-02-01T17:29:59.189Z20/30 Read block 302 len 40960
56872024-02-01T17:29:59.189Z21/30 Read block 515 len 16384
56882024-02-01T17:29:59.189Z22/30 Read block 482 len 40960
56892024-02-01T17:29:59.189Z23/30 Read block 41 len 40960
56902024-02-01T17:29:59.190Z24/30 Write block 555 len 12288 data: 5 4 6
56912024-02-01T17:29:59.190Z25/30 Write block 48 len 28672 data: 5 4 7 7 4 5 7
56922024-02-01T17:29:59.190Z26/30 Write block 339 len 36864 data: 4 2 2 2 2 2 2 2 2
56932024-02-01T17:29:59.190Z27/30 Write block 374 len 12288 data: 6 8 10
56942024-02-01T17:29:59.190Z28/30 Read block 345 len 8192
56952024-02-01T17:29:59.191Z29/30 Read block 210 len 16384
56962024-02-01T17:29:59.191Z30/30 Write block 148 len 32768 data: 4 4 3 3 7 8 6 7
56972024-02-01T17:29:59.191Z----------------------------------------------------------------
56982024-02-01T17:29:59.191Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
56992024-02-01T17:29:59.191ZStates: Active Active Active
57002024-02-01T17:29:59.191ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57012024-02-01T17:29:59.191Z 1 Acked 1000 Write 3 Done Sent Done false
57022024-02-01T17:29:59.191Z 2 Acked 1001 Read 6 Done Sent Done false
57032024-02-01T17:29:59.191Z 3 Acked 1002 Write 7 Done Sent Done false
57042024-02-01T17:29:59.191Z 4 Acked 1003 Write 2 Done Sent Done false
57052024-02-01T17:29:59.191Z 5 Acked 1004 Write 5 Done Sent Done false
57062024-02-01T17:29:59.192Z 6 Acked 1005 Write 9 Done Sent Done false
57072024-02-01T17:29:59.192Z 7 Acked 1006 Read 10 Done Sent Done false
57082024-02-01T17:29:59.192Z 8 Acked 1007 Write 8 Done Sent Done false
57092024-02-01T17:29:59.192Z 9 Acked 1008 Read 8 Done Sent Done false
57102024-02-01T17:29:59.192Z 10 Acked 1009 Write 3 Done Sent Done false
57112024-02-01T17:29:59.192Z 11 Acked 1010 Write 5 Done Sent Done false
57122024-02-01T17:29:59.192Z 12 Acked 1011 Write 3 Done Sent Done false
57132024-02-01T17:29:59.192Z 13 Acked 1012 Write 3 Done Sent Done false
57142024-02-01T17:29:59.192Z 14 Acked 1013 Read 9 Done Sent Done false
57152024-02-01T17:29:59.192Z 15 Acked 1014 Read 2 Done Sent Done false
57162024-02-01T17:29:59.193Z 16 Acked 1015 Write 10 Done Sent Done false
57172024-02-01T17:29:59.193Z 17 Acked 1016 Read 10 Done Sent Done false
57182024-02-01T17:29:59.193Z 18 Acked 1017 Write 6 Done Sent Done false
57192024-02-01T17:29:59.193Z 19 Acked 1018 Read 7 Done Sent Done false
57202024-02-01T17:29:59.194Z 20 Acked 1019 Read 10 Done Sent Done false
57212024-02-01T17:29:59.194Z 21 Acked 1020 Read 4 Done Sent Done false
57222024-02-01T17:29:59.194Z 22 Acked 1021 Read 10 Done Sent Done false
57232024-02-01T17:29:59.194Z 23 Acked 1022 Read 10 Done Sent Done false
57242024-02-01T17:29:59.194Z 24 Acked 1023 Write 3 Done Sent Done false
57252024-02-01T17:29:59.194Z 25 Acked 1024 Write 7 Done Sent Done false
57262024-02-01T17:29:59.194Z 26 Acked 1025 Write 9 Done Sent Done false
57272024-02-01T17:29:59.194Z 27 Acked 1026 Write 3 Done Sent Done false
57282024-02-01T17:29:59.194Z 28 Acked 1027 Read 2 Done Sent Done false
57292024-02-01T17:29:59.195Z 29 Acked 1028 Read 4 Done Sent Done false
57302024-02-01T17:29:59.195Z 30 Acked 1029 Write 8 Sent Sent Sent false
57312024-02-01T17:29:59.195Z STATES DS:0 DS:1 DS:2 TOTAL
57322024-02-01T17:29:59.195Z New 0 0 0 0
57332024-02-01T17:29:59.195Z Sent 1 30 1 32
57342024-02-01T17:29:59.195Z Done 29 0 29 58
57352024-02-01T17:29:59.195Z Skipped 0 0 0 0
57362024-02-01T17:29:59.196Z Error 0 0 0 0
57372024-02-01T17:29:59.196ZLast Flush: 0 0 0
57382024-02-01T17:29:59.196ZDownstairs last five completed:
57392024-02-01T17:29:59.196ZUpstairs last five completed: 30 29 28 27 26
57402024-02-01T17:29:59.196Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:29:58.441927675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1337}
57412024-02-01T17:29:59.196ZWrote out file "/tmp/test_repair_verify.data"
57422024-02-01T17:29:59.206Z
57432024-02-01T17:29:59.209ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
57442024-02-01T17:29:59.219ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57452024-02-01T17:29:59.219Z 0 000-019 51 51 51 72 72 72 F F F
57462024-02-01T17:29:59.219Z 1 020-039 44 44 44 61 61 61 T F T <---
57472024-02-01T17:29:59.219Z 2 040-059 44 44 44 61 61 61 T F T <---
57482024-02-01T17:29:59.219Z 3 060-079 51 51 51 72 72 72 T F T <---
57492024-02-01T17:29:59.220Z 4 080-099 44 44 44 61 61 61 F F F
57502024-02-01T17:29:59.224Z 5 100-119 50 50 50 71 71 71 F F F
57512024-02-01T17:29:59.224Z 6 120-139 50 50 50 70 70 70 F F F
57522024-02-01T17:29:59.224Z 7 140-159 44 44 44 61 61 61 T F T <---
57532024-02-01T17:29:59.224Z 8 160-179 51 51 51 72 72 72 F F F
57542024-02-01T17:29:59.225Z 9 180-199 49 49 49 68 68 68 F F F
57552024-02-01T17:29:59.225Z 10 200-219 51 51 51 72 72 72 F F F
57562024-02-01T17:29:59.225Z 11 220-239 49 49 49 68 68 68 F F F
57572024-02-01T17:29:59.225Z 12 240-259 49 49 49 68 68 68 T F T <---
57582024-02-01T17:29:59.225Z 13 260-279 50 50 50 70 70 70 T F T <---
57592024-02-01T17:29:59.225Z 14 280-299 51 51 51 72 72 72 F F F
57602024-02-01T17:29:59.225Z 15 300-319 50 50 50 71 71 71 T F T <---
57612024-02-01T17:29:59.225Z 16 320-339 50 50 50 71 71 71 T F T <---
57622024-02-01T17:29:59.225Z 17 340-359 36 36 36 50 50 50 T F T <---
57632024-02-01T17:29:59.225Z 18 360-379 49 49 49 68 68 68 T F T <---
57642024-02-01T17:29:59.225Z 19 380-399 44 44 44 61 61 61 F F F
57652024-02-01T17:29:59.225Z 20 400-419 51 51 51 72 72 72 F F F
57662024-02-01T17:29:59.225Z 21 420-439 48 48 48 67 67 67 F F F
57672024-02-01T17:29:59.226Z 22 440-459 48 48 48 67 67 67 F F F
57682024-02-01T17:29:59.226Z 23 460-479 46 46 46 65 65 65 F F F
57692024-02-01T17:29:59.226Z 24 480-499 46 46 46 65 65 65 T F T <---
57702024-02-01T17:29:59.226Z 25 500-519 47 47 47 66 66 66 F F F
57712024-02-01T17:29:59.226Z 26 520-539 50 50 50 69 69 69 T F T <---
57722024-02-01T17:29:59.226Z 27 540-559 46 46 46 65 65 65 T F T <---
57732024-02-01T17:29:59.226Z 28 560-579 43 43 43 60 60 60 T F T <---
57742024-02-01T17:29:59.226Z 29 580-599 46 46 46 63 63 63 F F F
57752024-02-01T17:29:59.226ZMax gen: 51, Max flush: 72
57762024-02-01T17:29:59.226ZError: Difference in extent metadata found!
57772024-02-01T17:29:59.234ZA Difference in extent metadata is expected here
57782024-02-01T17:29:59.234ZOn loop 26
57792024-02-01T17:29:59.234Z
57802024-02-01T17:29:59.234ZVerifying data now
57812024-02-01T17:30:04.426ZNo change after verify
57822024-02-01T17:30:04.426ZLoop: 26 Downstairs dump after verify (and repair):
57832024-02-01T17:30:04.458ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57842024-02-01T17:30:04.458Z 0 000-019 51 51 51 72 72 72 F F F
57852024-02-01T17:30:04.460Z 1 020-039 52 52 52 73 73 73 F F F
57862024-02-01T17:30:04.460Z 2 040-059 52 52 52 73 73 73 F F F
57872024-02-01T17:30:04.460Z 3 060-079 52 52 52 73 73 73 F F F
57882024-02-01T17:30:04.460Z 4 080-099 44 44 44 61 61 61 F F F
57892024-02-01T17:30:04.460Z 5 100-119 50 50 50 71 71 71 F F F
57902024-02-01T17:30:04.460Z 6 120-139 50 50 50 70 70 70 F F F
57912024-02-01T17:30:04.460Z 7 140-159 52 52 52 73 73 73 F F F
57922024-02-01T17:30:04.461Z 8 160-179 51 51 51 72 72 72 F F F
57932024-02-01T17:30:04.461Z 9 180-199 49 49 49 68 68 68 F F F
57942024-02-01T17:30:04.461Z 10 200-219 51 51 51 72 72 72 F F F
57952024-02-01T17:30:04.461Z 11 220-239 49 49 49 68 68 68 F F F
57962024-02-01T17:30:04.461Z 12 240-259 52 52 52 73 73 73 F F F
57972024-02-01T17:30:04.487Z 13 260-279 52 52 52 73 73 73 F F F
57982024-02-01T17:30:04.487Z 14 280-299 51 51 51 72 72 72 F F F
57992024-02-01T17:30:04.487Z 15 300-319 52 52 52 73 73 73 F F F
58002024-02-01T17:30:04.487Z 16 320-339 52 52 52 73 73 73 F F F
58012024-02-01T17:30:04.487Z 17 340-359 52 52 52 73 73 73 F F F
58022024-02-01T17:30:04.487Z 18 360-379 52 52 52 73 73 73 F F F
58032024-02-01T17:30:04.487Z 19 380-399 44 44 44 61 61 61 F F F
58042024-02-01T17:30:04.487Z 20 400-419 51 51 51 72 72 72 F F F
58052024-02-01T17:30:04.487Z 21 420-439 48 48 48 67 67 67 F F F
58062024-02-01T17:30:04.487Z 22 440-459 48 48 48 67 67 67 F F F
58072024-02-01T17:30:04.487Z 23 460-479 46 46 46 65 65 65 F F F
58082024-02-01T17:30:04.487Z 24 480-499 52 52 52 73 73 73 F F F
58092024-02-01T17:30:04.487Z 25 500-519 47 47 47 66 66 66 F F F
58102024-02-01T17:30:04.487Z 26 520-539 52 52 52 73 73 73 F F F
58112024-02-01T17:30:04.487Z 27 540-559 52 52 52 73 73 73 F F F
58122024-02-01T17:30:04.487Z 28 560-579 52 52 52 73 73 73 F F F
58132024-02-01T17:30:04.487Z 29 580-599 46 46 46 63 63 63 F F F
58142024-02-01T17:30:04.487ZMax gen: 52, Max flush: 73
58152024-02-01T17:30:04.487Z
58162024-02-01T17:30:04.488ZBegin loop 27
58172024-02-01T17:30:04.488ZDownstairs to restart: 2
58182024-02-01T17:30:04.494Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752040644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
58192024-02-01T17:30:04.494Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752533465Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
58202024-02-01T17:30:04.494Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752608847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
58212024-02-01T17:30:04.494ZCrucible runtime is spawned
58222024-02-01T17:30:04.494Z{"msg":"Crucible 61ca3d65-644a-496d-8b77-5154f8483dd4 has session id: a44e5f06-e21d-4449-a9af-b029537ee182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752693544Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58232024-02-01T17:30:04.494Z{"msg":"Upstairs opts: Upstairs UUID: 61ca3d65-644a-496d-8b77-5154f8483dd4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752831592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58242024-02-01T17:30:04.494Z{"msg":"Crucible stats registered with UUID: 61ca3d65-644a-496d-8b77-5154f8483dd4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752917876Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58252024-02-01T17:30:04.494Z{"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.752962261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
58262024-02-01T17:30:04.494Z{"msg":"61ca3d65-644a-496d-8b77-5154f8483dd4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753035863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58272024-02-01T17:30:04.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753077941Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58282024-02-01T17:30:04.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753157732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58292024-02-01T17:30:04.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753202092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58302024-02-01T17:30:04.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753244894Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58312024-02-01T17:30:04.494Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.75331197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58322024-02-01T17:30:04.494Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753346057Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58332024-02-01T17:30:04.494Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753377645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58342024-02-01T17:30:04.494Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753423373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58352024-02-01T17:30:04.494Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753463059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58362024-02-01T17:30:04.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753733738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58372024-02-01T17:30:04.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753840377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58382024-02-01T17:30:04.495Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.753946035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"io task","client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58392024-02-01T17:30:04.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.755168181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58402024-02-01T17:30:04.495Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.755228172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58412024-02-01T17:30:04.495Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.755341737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58422024-02-01T17:30:04.495Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.755430852Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58432024-02-01T17:30:04.495Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.756324206Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58442024-02-01T17:30:04.495Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.756612372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58452024-02-01T17:30:04.496Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757092765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58462024-02-01T17:30:04.496Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.7571685Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58472024-02-01T17:30:04.496Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757196658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58482024-02-01T17:30:04.497Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757250869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58492024-02-01T17:30:04.497Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757282181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58502024-02-01T17:30:04.497Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757324332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58512024-02-01T17:30:04.497Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757352747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58522024-02-01T17:30:04.497Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.757374998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58532024-02-01T17:30:04.497Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.758065954Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58542024-02-01T17:30:04.503Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.758955925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58552024-02-01T17:30:04.503Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759013824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58562024-02-01T17:30:04.503Z{"msg":"[0]R flush_numbers[0..12]: [72, 73, 73, 73, 61, 71, 70, 73, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759040799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58572024-02-01T17:30:04.503Z{"msg":"[0]R generation[0..12]: [51, 52, 52, 52, 44, 50, 50, 52, 51, 49, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.75906398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58582024-02-01T17:30:04.503Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759106957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58592024-02-01T17:30:04.503Z{"msg":"[1]R flush_numbers[0..12]: [72, 73, 73, 73, 61, 71, 70, 73, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759132266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58602024-02-01T17:30:04.503Z{"msg":"[1]R generation[0..12]: [51, 52, 52, 52, 44, 50, 50, 52, 51, 49, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759155212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58612024-02-01T17:30:04.503Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759196559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58622024-02-01T17:30:04.503Z{"msg":"[2]R flush_numbers[0..12]: [72, 73, 73, 73, 61, 71, 70, 73, 72, 68, 72, 68]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759221142Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58632024-02-01T17:30:04.503Z{"msg":"[2]R generation[0..12]: [51, 52, 52, 52, 44, 50, 50, 52, 51, 49, 51, 49]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.7592445Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58642024-02-01T17:30:04.503Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759291446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58652024-02-01T17:30:04.503Z{"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759315325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58662024-02-01T17:30:04.503Z{"msg":"Generation requested: 54 >= found:53","v":0,"nameWait for a query_work_queue command to finish before sending IO
58672024-02-01T17:30:04.503Z":"crucible","level":30,"time":"2024-02-01T17:30:03.759342283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58682024-02-01T17:30:04.503Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.75936407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58692024-02-01T17:30:04.503Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759387613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"":"downstairs","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58702024-02-01T17:30:04.503Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759409664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58712024-02-01T17:30:04.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759429294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"0","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58722024-02-01T17:30:04.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759452167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"1","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58732024-02-01T17:30:04.503Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759474024Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"client":"2","":"downstairs","session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58742024-02-01T17:30:04.503Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759495813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58752024-02-01T17:30:04.504Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759541383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58762024-02-01T17:30:04.504Z{"msg":"61ca3d65-644a-496d-8b77-5154f8483dd4 is now active with session: a44e5f06-e21d-4449-a9af-b029537ee182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759564568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58772024-02-01T17:30:04.504Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759584187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345,"session_id":"a44e5f06-e21d-4449-a9af-b029537ee182"}
58782024-02-01T17:30:04.504Z{"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:03.759626745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
58792024-02-01T17:30:04.504ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58802024-02-01T17:30:04.504ZLoading write count information from file "/tmp/test_repair_verify.data"
58812024-02-01T17:30:04.504ZRun Repair workload
58822024-02-01T17:30:04.504Z01/30 Write block 48 len 40960 data: 6 5 8 8 5 6 8 8 7 6
58832024-02-01T17:30:04.504Z02/30 Read block 75 len 24576
58842024-02-01T17:30:04.504Z03/30 Read block 554 len 16384
58852024-02-01T17:30:04.508Z04/30 Read block 249 len 24576
58862024-02-01T17:30:04.508Z05/30 Read block 207 len 12288
58872024-02-01T17:30:04.508Z06/30 Read block 355 len 24576
58882024-02-01T17:30:04.508Z07/30 Write block 320 len 16384 data: 7 6 6 4
58892024-02-01T17:30:04.508Z08/30 Write block 422 len 20480 data: 7 7 6 7 7
58902024-02-01T17:30:04.508Z09/30 Read block 328 len 28672
58912024-02-01T17:30:04.511Z10/30 Read block 234 len 40960
58922024-02-01T17:30:04.514Z11/30 Write block 498 len 16384 data: 3 2 2 2
58932024-02-01T17:30:04.514Z12/30 Write block 558 len 20480 data: 7 7 7 7 7
58942024-02-01T17:30:04.521Z13/30 Write block 236 len 8192 data: 3 3
58952024-02-01T17:30:04.525Z14/30 Read block 347 len 4096
58962024-02-01T17:30:04.525Z15/30 Read block 555 len 40960
58972024-02-01T17:30:04.525Z16/30 Read block 380 len 40960
58982024-02-01T17:30:04.525Z17/30 Read block 336 len 28672
58992024-02-01T17:30:04.525Z18/30 Read block 554 len 32768
59002024-02-01T17:30:04.525Z19/30 Read block 565 len 40960
59012024-02-01T17:30:04.525Z20/30 Read block 447 len 40960
59022024-02-01T17:30:04.528Z21/30 Write block 169 len 32768 data: 6 7 8 8 8 9 8 9
59032024-02-01T17:30:04.528Z22/30 Read block 150 len 12288
59042024-02-01T17:30:04.532Z23/30 Write block 443 len 28672 data: 5 5 5 5 5 5 6
59052024-02-01T17:30:04.532Z24/30 Read block 154 len 24576
59062024-02-01T17:30:04.535Z25/30 Read block 194 len 16384
59072024-02-01T17:30:04.535Z26/30 Write block 585 len 16384 data: 5 6 5 6
59082024-02-01T17:30:04.535Z27/30 Read block 131 len 40960
59092024-02-01T17:30:04.538Z28/30 Write block 322 len 4096 data: 7
59102024-02-01T17:30:04.538Z29/30 Read block 199 len 40960
59112024-02-01T17:30:04.541Z30/30 Read block 111 len 28672
59122024-02-01T17:30:04.541Z----------------------------------------------------------------
59132024-02-01T17:30:04.541Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
59142024-02-01T17:30:04.541ZStates: Active Active Active
59152024-02-01T17:30:04.542ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59162024-02-01T17:30:04.542Z 1 Acked 1000 Write 10 Done Done Done false
59172024-02-01T17:30:04.542Z 2 Acked 1001 Read 6 Done Done Done false
59182024-02-01T17:30:04.543Z 3 Acked 1002 Read 4 Done Done Sent false
59192024-02-01T17:30:04.543Z 4 Acked 1003 Read 6 Done Done Sent false
59202024-02-01T17:30:04.543Z 5 Acked 1004 Read 3 Done Done Sent false
59212024-02-01T17:30:04.543Z 6 Acked 1005 Read 6 Done Done Sent false
59222024-02-01T17:30:04.543Z 7 Acked 1006 Write 4 Done Done Sent false
59232024-02-01T17:30:04.543Z 8 Acked 1007 Write 5 Done Done Sent false
59242024-02-01T17:30:04.543Z 9 Acked 1008 Read 7 Done Done Sent false
59252024-02-01T17:30:04.543Z 10 Acked 1009 Read 10 Done Done Sent false
59262024-02-01T17:30:04.543Z 11 Acked 1010 Write 4 Done Done Sent false
59272024-02-01T17:30:04.544Z 12 Acked 1011 Write 5 Done Done Sent false
59282024-02-01T17:30:04.544Z 13 Acked 1012 Write 2 Done Done Sent false
59292024-02-01T17:30:04.544Z 14 Acked 1013 Read 1 Done Done Sent false
59302024-02-01T17:30:04.544Z 15 Acked 1014 Read 10 Done Done Sent false
59312024-02-01T17:30:04.544Z 16 Acked 1015 Read 10 Done Done Sent false
59322024-02-01T17:30:04.544Z 17 Acked 1016 Read 7 Done Done Sent false
59332024-02-01T17:30:04.544Z 18 Acked 1017 Read 8 Done Done Sent false
59342024-02-01T17:30:04.544Z 19 Acked 1018 Read 10 Done Done Sent false
59352024-02-01T17:30:04.544Z 20 Acked 1019 Read 10 Done Done Sent false
59362024-02-01T17:30:04.544Z 21 Acked 1020 Write 8 Done Done Sent false
59372024-02-01T17:30:04.544Z 22 Acked 1021 Read 3 Done Done Sent false
59382024-02-01T17:30:04.544Z 23 Acked 1022 Write 7 Done Done Sent false
59392024-02-01T17:30:04.544Z 24 Acked 1023 Read 6 Done Done Sent false
59402024-02-01T17:30:04.544Z 25 Acked 1024 Read 4 Done Done Sent false
59412024-02-01T17:30:04.548Z 26 Acked 1025 Write 4 Done Done Sent false
59422024-02-01T17:30:04.548Z 27 Acked 1026 Read 10 Done Done Sent false
59432024-02-01T17:30:04.548Z 28 Acked 1027 Write 1 Done Done Sent false
59442024-02-01T17:30:04.548Z 29 Acked 1028 Read 10 Done Done Sent false
59452024-02-01T17:30:04.549Z 30 Acked 1029 Read 7 Done Sent Sent false
59462024-02-01T17:30:04.549Z STATES DS:0 DS:1 DS:2 TOTAL
59472024-02-01T17:30:04.549Z New 0 0 0 0
59482024-02-01T17:30:04.549Z Sent 0 1 28 29
59492024-02-01T17:30:04.549Z Done 30 29 2 61
59502024-02-01T17:30:04.549Z Skipped 0 0 0 0
59512024-02-01T17:30:04.549Z Error 0 0 0 0
59522024-02-01T17:30:04.549ZLast Flush: 0 0 0
59532024-02-01T17:30:04.549ZDownstairs last five completed:
59542024-02-01T17:30:04.549ZUpstairs last five completed: 30 29 28 27 26
59552024-02-01T17:30:04.549Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:03.80702881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1345}
59562024-02-01T17:30:04.549ZWrote out file "/tmp/test_repair_verify.data"
59572024-02-01T17:30:04.552Z
59582024-02-01T17:30:04.557ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59592024-02-01T17:30:04.585ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59602024-02-01T17:30:04.586Z 0 000-019 51 51 51 72 72 72 F F F
59612024-02-01T17:30:04.586Z 1 020-039 52 52 52 73 73 73 F F F
59622024-02-01T17:30:04.586Z 2 040-059 52 52 52 73 73 73 T T T
59632024-02-01T17:30:04.586Z 3 060-079 52 52 52 73 73 73 F F F
59642024-02-01T17:30:04.591Z 4 080-099 44 44 44 61 61 61 F F F
59652024-02-01T17:30:04.592Z 5 100-119 50 50 50 71 71 71 F F F
59662024-02-01T17:30:04.592Z 6 120-139 50 50 50 70 70 70 F F F
59672024-02-01T17:30:04.592Z 7 140-159 52 52 52 73 73 73 F F F
59682024-02-01T17:30:04.592Z 8 160-179 51 51 51 72 72 72 T T F <---
59692024-02-01T17:30:04.592Z 9 180-199 49 49 49 68 68 68 F F F
59702024-02-01T17:30:04.592Z 10 200-219 51 51 51 72 72 72 F F F
59712024-02-01T17:30:04.592Z 11 220-239 49 49 49 68 68 68 T T F <---
59722024-02-01T17:30:04.592Z 12 240-259 52 52 52 73 73 73 F F F
59732024-02-01T17:30:04.593Z 13 260-279 52 52 52 73 73 73 F F F
59742024-02-01T17:30:04.593Z 14 280-299 51 51 51 72 72 72 F F F
59752024-02-01T17:30:04.593Z 15 300-319 52 52 52 73 73 73 F F F
59762024-02-01T17:30:04.593Z 16 320-339 52 52 52 73 73 73 T T F <---
59772024-02-01T17:30:04.593Z 17 340-359 52 52 52 73 73 73 F F F
59782024-02-01T17:30:04.593Z 18 360-379 52 52 52 73 73 73 F F F
59792024-02-01T17:30:04.593Z 19 380-399 44 44 44 61 61 61 F F F
59802024-02-01T17:30:04.593Z 20 400-419 51 51 51 72 72 72 F F F
59812024-02-01T17:30:04.593Z 21 420-439 48 48 48 67 67 67 T T F <---
59822024-02-01T17:30:04.593Z 22 440-459 48 48 48 67 67 67 T T F <---
59832024-02-01T17:30:04.594Z 23 460-479 46 46 46 65 65 65 F F F
59842024-02-01T17:30:04.594Z 24 480-499 52 52 52 73 73 73 T T F <---
59852024-02-01T17:30:04.594Z 25 500-519 47 47 47 66 66 66 T T F <---
59862024-02-01T17:30:04.594Z 26 520-539 52 52 52 73 73 73 F F F
59872024-02-01T17:30:04.594Z 27 540-559 52 52 52 73 73 73 T T F <---
59882024-02-01T17:30:04.594Z 28 560-579 52 52 52 73 73 73 T T F <---
59892024-02-01T17:30:04.594Z 29 580-599 46 46 46 63 63 63 T T F <---
59902024-02-01T17:30:04.596ZMax gen: 52, Max flush: 73
59912024-02-01T17:30:04.596ZError: Difference in extent metadata found!
59922024-02-01T17:30:04.596ZA Difference in extent metadata is expected here
59932024-02-01T17:30:04.596ZOn loop 27
59942024-02-01T17:30:04.596Z
59952024-02-01T17:30:04.603ZVerifying data now
59962024-02-01T17:30:09.582ZNo change after verify
59972024-02-01T17:30:09.582ZLoop: 27 Downstairs dump after verify (and repair):
59982024-02-01T17:30:09.605ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59992024-02-01T17:30:09.605Z 0 000-019 51 51 51 72 72 72 F F F
60002024-02-01T17:30:09.606Z 1 020-039 52 52 52 73 73 73 F F F
60012024-02-01T17:30:09.606Z 2 040-059 53 53 53 74 74 74 F F F
60022024-02-01T17:30:09.606Z 3 060-079 52 52 52 73 73 73 F F F
60032024-02-01T17:30:09.606Z 4 080-099 44 44 44 61 61 61 F F F
60042024-02-01T17:30:09.606Z 5 100-119 50 50 50 71 71 71 F F F
60052024-02-01T17:30:09.607Z 6 120-139 50 50 50 70 70 70 F F F
60062024-02-01T17:30:09.607Z 7 140-159 52 52 52 73 73 73 F F F
60072024-02-01T17:30:09.607Z 8 160-179 53 53 53 74 74 74 F F F
60082024-02-01T17:30:09.607Z 9 180-199 49 49 49 68 68 68 F F F
60092024-02-01T17:30:09.607Z 10 200-219 51 51 51 72 72 72 F F F
60102024-02-01T17:30:09.608Z 11 220-239 53 53 53 74 74 74 F F F
60112024-02-01T17:30:09.608Z 12 240-259 52 52 52 73 73 73 F F F
60122024-02-01T17:30:09.608Z 13 260-279 52 52 52 73 73 73 F F F
60132024-02-01T17:30:09.608Z 14 280-299 51 51 51 72 72 72 F F F
60142024-02-01T17:30:09.608Z 15 300-319 52 52 52 73 73 73 F F F
60152024-02-01T17:30:09.616Z 16 320-339 53 53 53 74 74 74 F F F
60162024-02-01T17:30:09.616Z 17 340-359 52 52 52 73 73 73 F F F
60172024-02-01T17:30:09.616Z 18 360-379 52 52 52 73 73 73 F F F
60182024-02-01T17:30:09.616Z 19 380-399 44 44 44 61 61 61 F F F
60192024-02-01T17:30:09.616Z 20 400-419 51 51 51 72 72 72 F F F
60202024-02-01T17:30:09.616Z 21 420-439 53 53 53 74 74 74 F F F
60212024-02-01T17:30:09.616Z 22 440-459 53 53 53 74 74 74 F F F
60222024-02-01T17:30:09.616Z 23 460-479 46 46 46 65 65 65 F F F
60232024-02-01T17:30:09.616Z 24 480-499 53 53 53 74 74 74 F F F
60242024-02-01T17:30:09.616Z 25 500-519 53 53 53 74 74 74 F F F
60252024-02-01T17:30:09.616Z 26 520-539 52 52 52 73 73 73 F F F
60262024-02-01T17:30:09.616Z 27 540-559 53 53 53 74 74 74 F F F
60272024-02-01T17:30:09.617Z 28 560-579 53 53 53 74 74 74 F F F
60282024-02-01T17:30:09.617Z 29 580-599 53 53 53 74 74 74 F F F
60292024-02-01T17:30:09.617ZMax gen: 53, Max flush: 74
60302024-02-01T17:30:09.617Z
60312024-02-01T17:30:09.617ZBegin loop 28
60322024-02-01T17:30:09.617ZDownstairs to restart: 2
60332024-02-01T17:30:09.641Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
60342024-02-01T17:30:09.641Z,"time":"2024-02-01T17:30:08.905234711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
60352024-02-01T17:30:09.658Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.905808223Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
60362024-02-01T17:30:09.658Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.905884415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
60372024-02-01T17:30:09.658Z{"msg":"Crucible 92d21da8-6969-4b94-a205-55a54ff12c8b has session id: 10c30d9b-6f59-458e-aff1-0b99d60a6f5e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.905928549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60382024-02-01T17:30:09.658Z{"msg":"Upstairs opts: Upstairs UUID: 92d21da8-6969-4b94-a205-55a54ff12c8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.905996583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60392024-02-01T17:30:09.658Z{"msg":"Crucible stats registered with UUID: 92d21da8-6969-4b94-a205-55a54ff12c8b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906042527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60402024-02-01T17:30:09.658Z{"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906098122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
60412024-02-01T17:30:09.658Z{"msg":"92d21da8-6969-4b94-a205-55a54ff12c8b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906135486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60422024-02-01T17:30:09.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906173861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60432024-02-01T17:30:09.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906220884Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60442024-02-01T17:30:09.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906297213Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60452024-02-01T17:30:09.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906343615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60462024-02-01T17:30:09.658Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906412613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60472024-02-01T17:30:09.658Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906456127Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60482024-02-01T17:30:09.658Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906499087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60492024-02-01T17:30:09.659Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906542378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60502024-02-01T17:30:09.659Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906584117Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60512024-02-01T17:30:09.659Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906610042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60522024-02-01T17:30:09.659Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906632576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60532024-02-01T17:30:09.659Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.906678885Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"io task","client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60542024-02-01T17:30:09.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.907995364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60552024-02-01T17:30:09.659Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.908181937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60562024-02-01T17:30:09.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.908231111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60572024-02-01T17:30:09.659Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.908365055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60582024-02-01T17:30:09.659Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.908948393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60592024-02-01T17:30:09.659Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.909167091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60602024-02-01T17:30:09.659Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.909354758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60612024-02-01T17:30:09.659Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.90944602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60622024-02-01T17:30:09.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910109625Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60632024-02-01T17:30:09.661Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910175837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60642024-02-01T17:30:09.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.91020398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60652024-02-01T17:30:09.661Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910247551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60662024-02-01T17:30:09.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910272592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60672024-02-01T17:30:09.661Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.91029524Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60682024-02-01T17:30:09.661Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910339486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60692024-02-01T17:30:09.661Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.910920582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60702024-02-01T17:30:09.661Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911016488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60712024-02-01T17:30:09.661Z{"msg":"[0]R flush_numbers[0..12]: [72, 73, 74, 73, 61, 71, 70, 73, 74, 68, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911047537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60722024-02-01T17:30:09.662Z{"msg":"[0]R generation[0..12]: [51, 52, 53, 52, 44, 50, 50, 52, 53, 49, 51, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911072025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60732024-02-01T17:30:09.662Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911096592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60742024-02-01T17:30:09.664Z{"msg":"[1]R flush_numbers[0..12]: [72, 73, 74, 73, 61, 71, 70, 73, 74, 68, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911140124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60752024-02-01T17:30:09.664Z{"msg":"[1]R generation[0..12]: [51, 52, 53, 52, 44, 50, 50, 52, 53, 49, 51, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911165873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60762024-02-01T17:30:09.664Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911208203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60772024-02-01T17:30:09.664Z{"msg":"[2]R flush_numbers[0..12]: [72, 73, 74, 73, 61, 71, 70, 73, 74, 68, 72, 74]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911233537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60782024-02-01T17:30:09.664Z{"msg":"[2]R generation[0..12]: [51, 52, 53, 52, 44, 50, 50, 52, 53, 49, 51, 53]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.91125701Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60792024-02-01T17:30:09.664Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
60802024-02-01T17:30:09.664Z,"level":30,"time":"2024-02-01T17:30:08.911375383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60812024-02-01T17:30:09.664Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911429037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60822024-02-01T17:30:09.664Z{"msg":"Generation requested: 56 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911454799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60832024-02-01T17:30:09.664Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911495636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60842024-02-01T17:30:09.664Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911519591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"":"downstairs","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60852024-02-01T17:30:09.664Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911560555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60862024-02-01T17:30:09.664Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911604773Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"0","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60872024-02-01T17:30:09.665Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911629071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"1","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60882024-02-01T17:30:09.665Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911682884Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"client":"2","":"downstairs","session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60892024-02-01T17:30:09.665Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911707317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60902024-02-01T17:30:09.665Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911728327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60912024-02-01T17:30:09.665Z{"msg":"92d21da8-6969-4b94-a205-55a54ff12c8b is now active with session: 10c30d9b-6f59-458e-aff1-0b99d60a6f5e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911750102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60922024-02-01T17:30:09.666Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911789313Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353,"session_id":"10c30d9b-6f59-458e-aff1-0b99d60a6f5e"}
60932024-02-01T17:30:09.666Z{"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:08.911812324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
60942024-02-01T17:30:09.666ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
60952024-02-01T17:30:09.666ZLoading write count information from file "/tmp/test_repair_verify.data"
60962024-02-01T17:30:09.666ZRun Repair workload
60972024-02-01T17:30:09.673Z01/30 Write block 444 len 8192 data: 6 6
60982024-02-01T17:30:09.673Z02/30 Read block 498 len 20480
60992024-02-01T17:30:09.673Z03/30 Read block 437 len 28672
61002024-02-01T17:30:09.673Z04/30 Write block 342 len 32768 data: 3 3 3 3 3 3 3 3
61012024-02-01T17:30:09.673Z05/30 Read block 184 len 12288
61022024-02-01T17:30:09.673Z06/30 Read block 66 len 20480
61032024-02-01T17:30:09.673Z07/30 Read block 177 len 32768
61042024-02-01T17:30:09.674Z08/30 Write block 493 len 4096 data: 7
61052024-02-01T17:30:09.674Z09/30 Read block 112 len 36864
61062024-02-01T17:30:09.674Z10/30 Read block 283 len 4096
61072024-02-01T17:30:09.674Z11/30 Flush
61082024-02-01T17:30:09.674Z12/30 Flush
61092024-02-01T17:30:09.674Z13/30 Write block 176 len 8192 data: 10 9
61102024-02-01T17:30:09.674Z14/30 Write block 339 len 40960 data: 5 3 3 4 4 4 4 4 4 4
61112024-02-01T17:30:09.674Z15/30 Write block 347 len 8192 data: 5 5
61122024-02-01T17:30:09.674Z16/30 Read block 324 len 36864
61132024-02-01T17:30:09.674Z17/30 Read block 355 len 40960
61142024-02-01T17:30:09.722Z18/30 Write block 555 len 24576 data: 6 5 7 8 8 8
61152024-02-01T17:30:09.722Z19/30 Write block 426 len 4096 data: 8
61162024-02-01T17:30:09.722Z20/30 Write block 485 len 32768 data: 6 6 6 7 7 7 7 8
61172024-02-01T17:30:09.722Z21/30 Write block 27 len 28672 data: 5 4 4 4 4 4 3
61182024-02-01T17:30:09.722Z22/30 Read block 134 len 16384
61192024-02-01T17:30:09.722Z23/30 Flush
61202024-02-01T17:30:09.732Z24/30 Write block 218 len 12288 data: 4 4 3
61212024-02-01T17:30:09.733Z25/30 Read block 118 len 12288
61222024-02-01T17:30:09.733Z26/30 Read block 93 len 40960
61232024-02-01T17:30:09.733Z27/30 Write block 565 len 4096 data: 6
61242024-02-01T17:30:09.733Z28/30 Read block 534 len 40960
61252024-02-01T17:30:09.733Z29/30 Read block 419 len 20480
61262024-02-01T17:30:09.734Z30/30 Read block 97 len 40960
61272024-02-01T17:30:09.734Z----------------------------------------------------------------
61282024-02-01T17:30:09.734Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
61292024-02-01T17:30:09.734ZStates: Active Active Active
61302024-02-01T17:30:09.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61312024-02-01T17:30:09.734Z 1 Acked 1000 Write 2 Done Done Done false
61322024-02-01T17:30:09.734Z 2 Acked 1001 Read 5 Done Done Done false
61332024-02-01T17:30:09.734Z 3 Acked 1002 Read 7 Done Done Done false
61342024-02-01T17:30:09.734Z 4 Acked 1003 Write 8 Done Done Done false
61352024-02-01T17:30:09.735Z 5 Acked 1004 Read 3 Done Done Done false
61362024-02-01T17:30:09.735Z 6 Acked 1005 Read 5 Done Done Done false
61372024-02-01T17:30:09.735Z 7 Acked 1006 Read 8 Done Done Done false
61382024-02-01T17:30:09.735Z 8 Acked 1007 Write 1 Done Done Done false
61392024-02-01T17:30:09.735Z 9 Acked 1008 Read 9 Done Done Done false
61402024-02-01T17:30:09.736Z 10 Acked 1009 Read 1 Done Done Sent false
61412024-02-01T17:30:09.736Z 11 Acked 1010 Flush 0 Done Done Sent false
61422024-02-01T17:30:09.736Z 12 Acked 1011 Flush 0 Done Done Sent false
61432024-02-01T17:30:09.736Z 13 Acked 1012 Write 2 Done Done Sent false
61442024-02-01T17:30:09.736Z 14 Acked 1013 Write 10 Done Done Sent false
61452024-02-01T17:30:09.737Z 15 Acked 1014 Write 2 Done Done Sent false
61462024-02-01T17:30:09.737Z 16 Acked 1015 Read 9 Done Done Sent false
61472024-02-01T17:30:09.737Z 17 Acked 1016 Read 10 Done Done Sent false
61482024-02-01T17:30:09.737Z 18 Acked 1017 Write 6 Done Done Sent false
61492024-02-01T17:30:09.737Z 19 Acked 1018 Write 1 Done Done Sent false
61502024-02-01T17:30:09.737Z 20 Acked 1019 Write 8 Done Done Sent false
61512024-02-01T17:30:09.737Z 21 Acked 1020 Write 7 Done Done Sent false
61522024-02-01T17:30:09.737Z 22 Acked 1021 Read 4 Done Done Sent false
61532024-02-01T17:30:09.737Z 23 Acked 1022 Flush 0 Done Done Sent false
61542024-02-01T17:30:09.737Z 24 Acked 1023 Write 3 Done Done Sent false
61552024-02-01T17:30:09.737Z 25 Acked 1024 Read 3 Done Done Sent false
61562024-02-01T17:30:09.738Z 26 Acked 1025 Read 10 Done Done Sent false
61572024-02-01T17:30:09.738Z 27 Acked 1026 Write 1 Done Done Sent false
61582024-02-01T17:30:09.738Z 28 Acked 1027 Read 10 Done Done Sent false
61592024-02-01T17:30:09.738Z 29 Acked 1028 Read 5 Done Done Sent false
61602024-02-01T17:30:09.738Z 30 Acked 1029 Read 10 Done Done Sent false
61612024-02-01T17:30:09.738Z STATES DS:0 DS:1 DS:2 TOTAL
61622024-02-01T17:30:09.739Z New 0 0 0 0
61632024-02-01T17:30:09.739Z Sent 0 0 21 21
61642024-02-01T17:30:09.740Z Done 30 30 9 69
61652024-02-01T17:30:09.740Z Skipped 0 0 0 0
61662024-02-01T17:30:09.740Z Error 0 0 0 0
61672024-02-01T17:30:09.740ZLast Flush: 1022 1022 0
61682024-02-01T17:30:09.740ZDownstairs last five completed:
61692024-02-01T17:30:09.740ZUpstairs last five completed: 30 29 28 27 26
61702024-02-01T17:30:09.741Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:08.95734664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1353}
61712024-02-01T17:30:09.741ZWrote out file "/tmp/test_repair_verify.data"
61722024-02-01T17:30:09.742Z
61732024-02-01T17:30:09.742ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
61742024-02-01T17:30:09.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
61752024-02-01T17:30:09.753Z 0 000-019 51 51 51 72 72 72 F F F
61762024-02-01T17:30:09.753Z 1 020-039 56 56 52 77 77 73 F F F <---
61772024-02-01T17:30:09.753Z 2 040-059 53 53 53 74 74 74 F F F
61782024-02-01T17:30:09.753Z 3 060-079 52 52 52 73 73 73 F F F
61792024-02-01T17:30:09.753Z 4 080-099 44 44 44 61 61 61 F F F
61802024-02-01T17:30:09.753Z 5 100-119 50 50 50 71 71 71 F F F
61812024-02-01T17:30:09.754Z 6 120-139 50 50 50 70 70 70 F F F
61822024-02-01T17:30:09.754Z 7 140-159 52 52 52 73 73 73 F F F
61832024-02-01T17:30:09.754Z 8 160-179 56 56 53 77 77 74 F F F <---
61842024-02-01T17:30:09.754Z 9 180-199 49 49 49 68 68 68 F F F
61852024-02-01T17:30:09.754Z 10 200-219 51 51 51 72 72 72 T T F <---
61862024-02-01T17:30:09.755Z 11 220-239 53 53 53 74 74 74 T T F <---
61872024-02-01T17:30:09.755Z 12 240-259 52 52 52 73 73 73 F F F
61882024-02-01T17:30:09.755Z 13 260-279 52 52 52 73 73 73 F F F
61892024-02-01T17:30:09.756Z 14 280-299 51 51 51 72 72 72 F F F
61902024-02-01T17:30:09.756Z 15 300-319 52 52 52 73 73 73 F F F
61912024-02-01T17:30:09.756Z 16 320-339 56 56 53 77 77 74 F F F <---
61922024-02-01T17:30:09.756Z 17 340-359 56 56 52 77 77 73 F F T <---
61932024-02-01T17:30:09.757Z 18 360-379 52 52 52 73 73 73 F F F
61942024-02-01T17:30:09.757Z 19 380-399 44 44 44 61 61 61 F F F
61952024-02-01T17:30:09.757Z 20 400-419 51 51 51 72 72 72 F F F
61962024-02-01T17:30:09.757Z 21 420-439 56 56 53 77 77 74 F F F <---
61972024-02-01T17:30:09.757Z 22 440-459 56 56 53 75 75 74 F F T <---
61982024-02-01T17:30:09.757Z 23 460-479 46 46 46 65 65 65 F F F
61992024-02-01T17:30:09.757Z 24 480-499 56 56 53 77 77 74 F F T <---
62002024-02-01T17:30:09.757Z 25 500-519 53 53 53 74 74 74 F F F
62012024-02-01T17:30:09.758Z 26 520-539 52 52 52 73 73 73 F F F
62022024-02-01T17:30:09.758Z 27 540-559 56 56 53 77 77 74 F F F <---
62032024-02-01T17:30:09.758Z 28 560-579 56 56 53 77 77 74 T T F <---
62042024-02-01T17:30:09.758Z 29 580-599 53 53 53 74 74 74 F F F
62052024-02-01T17:30:09.758ZMax gen: 56, Max flush: 77
62062024-02-01T17:30:09.758ZError: Difference in extent metadata found!
62072024-02-01T17:30:09.758ZA Difference in extent metadata is expected here
62082024-02-01T17:30:09.758ZOn loop 28
62092024-02-01T17:30:09.758Z
62102024-02-01T17:30:09.759ZVerifying data now
62112024-02-01T17:30:15.711ZNo change after verify
62122024-02-01T17:30:15.711ZLoop: 28 Downstairs dump after verify (and repair):
62132024-02-01T17:30:15.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62142024-02-01T17:30:15.731Z 0 000-019 51 51 51 72 72 72 F F F
62152024-02-01T17:30:15.735Z 1 020-039 56 56 56 77 77 77 F F F
62162024-02-01T17:30:15.735Z 2 040-059 53 53 53 74 74 74 F F F
62172024-02-01T17:30:15.735Z 3 060-079 52 52 52 73 73 73 F F F
62182024-02-01T17:30:15.735Z 4 080-099 44 44 44 61 61 61 F F F
62192024-02-01T17:30:15.735Z 5 100-119 50 50 50 71 71 71 F F F
62202024-02-01T17:30:15.735Z 6 120-139 50 50 50 70 70 70 F F F
62212024-02-01T17:30:15.735Z 7 140-159 52 52 52 73 73 73 F F F
62222024-02-01T17:30:15.735Z 8 160-179 56 56 56 77 77 77 F F F
62232024-02-01T17:30:15.735Z 9 180-199 49 49 49 68 68 68 F F F
62242024-02-01T17:30:15.735Z 10 200-219 57 57 57 78 78 78 F F F
62252024-02-01T17:30:15.735Z 11 220-239 57 57 57 78 78 78 F F F
62262024-02-01T17:30:15.735Z 12 240-259 52 52 52 73 73 73 F F F
62272024-02-01T17:30:15.735Z 13 260-279 52 52 52 73 73 73 F F F
62282024-02-01T17:30:15.735Z 14 280-299 51 51 51 72 72 72 F F F
62292024-02-01T17:30:15.735Z 15 300-319 52 52 52 73 73 73 F F F
62302024-02-01T17:30:15.735Z 16 320-339 56 56 56 77 77 77 F F F
62312024-02-01T17:30:15.735Z 17 340-359 56 56 56 77 77 77 F F F
62322024-02-01T17:30:15.736Z 18 360-379 52 52 52 73 73 73 F F F
62332024-02-01T17:30:15.736Z 19 380-399 44 44 44 61 61 61 F F F
62342024-02-01T17:30:15.736Z 20 400-419 51 51 51 72 72 72 F F F
62352024-02-01T17:30:15.736Z 21 420-439 56 56 56 77 77 77 F F F
62362024-02-01T17:30:15.736Z 22 440-459 56 56 56 75 75 75 F F F
62372024-02-01T17:30:15.736Z 23 460-479 46 46 46 65 65 65 F F F
62382024-02-01T17:30:15.736Z 24 480-499 56 56 56 77 77 77 F F F
62392024-02-01T17:30:15.736Z 25 500-519 53 53 53 74 74 74 F F F
62402024-02-01T17:30:15.736Z 26 520-539 52 52 52 73 73 73 F F F
62412024-02-01T17:30:15.736Z 27 540-559 56 56 56 77 77 77 F F F
62422024-02-01T17:30:15.736Z 28 560-579 57 57 57 78 78 78 F F F
62432024-02-01T17:30:15.736Z 29 580-599 53 53 53 74 74 74 F F F
62442024-02-01T17:30:15.736ZMax gen: 57, Max flush: 78
62452024-02-01T17:30:15.740Z
62462024-02-01T17:30:15.740ZBegin loop 29
62472024-02-01T17:30:15.740ZDownstairs to restart: 1
62482024-02-01T17:30:15.770Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.032421137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
62492024-02-01T17:30:15.770Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.032994241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
62502024-02-01T17:30:15.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033034243Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
62512024-02-01T17:30:15.770Z{Crucible runtime is spawned
62522024-02-01T17:30:15.779Z"msg":"Crucible 1360ab16-c1b2-4a8d-a78e-6f2abf41852c has session id: 1d8a1245-2cc5-4a86-9f5d-c6d02b68997d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033110104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62532024-02-01T17:30:15.779Z{"msg":"Upstairs opts: Upstairs UUID: 1360ab16-c1b2-4a8d-a78e-6f2abf41852c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033242095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62542024-02-01T17:30:15.779Z{"msg":"Crucible stats registered with UUID: 1360ab16-c1b2-4a8d-a78e-6f2abf41852c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033292712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62552024-02-01T17:30:15.779Z{"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033347757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
62562024-02-01T17:30:15.779Z{"msg":"1360ab16-c1b2-4a8d-a78e-6f2abf41852c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033381041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62572024-02-01T17:30:15.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033421736Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62582024-02-01T17:30:15.779Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.0334558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62592024-02-01T17:30:15.779Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033516281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62602024-02-01T17:30:15.779Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033623318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62612024-02-01T17:30:15.780Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033671869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62622024-02-01T17:30:15.780Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033716452Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62632024-02-01T17:30:15.780Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033793372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62642024-02-01T17:30:15.780Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033861981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62652024-02-01T17:30:15.781Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.033958083Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62662024-02-01T17:30:15.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.034023555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62672024-02-01T17:30:15.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.03410433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62682024-02-01T17:30:15.781Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.03431554Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"io task","client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62692024-02-01T17:30:15.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035552548Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62702024-02-01T17:30:15.781Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035617311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62712024-02-01T17:30:15.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035697538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62722024-02-01T17:30:15.781Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035720713Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62732024-02-01T17:30:15.781Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035778577Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62742024-02-01T17:30:15.781Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.035931917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62752024-02-01T17:30:15.781Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.036639879Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62762024-02-01T17:30:15.782Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.036800544Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62772024-02-01T17:30:15.782Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037025252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62782024-02-01T17:30:15.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037543549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62792024-02-01T17:30:15.782Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037627082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62802024-02-01T17:30:15.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037659468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62812024-02-01T17:30:15.782Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037702384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62822024-02-01T17:30:15.782Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037727162Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62832024-02-01T17:30:15.782Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037749249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62842024-02-01T17:30:15.783Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037875604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62852024-02-01T17:30:15.783Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.037964343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62862024-02-01T17:30:15.783Z{"msg":"[0]R flush_numbers[0..12]: [72, 77, 74, 73, 61, 71, 70, 73, 77, 68, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038105743Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62872024-02-01T17:30:15.783Z{"msg":"[0]R generation[0..12]: [51, 56, 53, 52, 44, 50, 50, 52, 56, 49, 57, 57]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
62882024-02-01T17:30:15.783Z,"time":"2024-02-01T17:30:15.038185948Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62892024-02-01T17:30:15.784Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038266687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62902024-02-01T17:30:15.784Z{"msg":"[1]R flush_numbers[0..12]: [72, 77, 74, 73, 61, 71, 70, 73, 77, 68, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038340238Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62912024-02-01T17:30:15.784Z{"msg":"[1]R generation[0..12]: [51, 56, 53, 52, 44, 50, 50, 52, 56, 49, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038414628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62922024-02-01T17:30:15.784Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038510541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62932024-02-01T17:30:15.784Z{"msg":"[2]R flush_numbers[0..12]: [72, 77, 74, 73, 61, 71, 70, 73, 77, 68, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038552059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62942024-02-01T17:30:15.785Z{"msg":"[2]R generation[0..12]: [51, 56, 53, 52, 44, 50, 50, 52, 56, 49, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038619797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62952024-02-01T17:30:15.785Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038696952Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62962024-02-01T17:30:15.785Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038767231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62972024-02-01T17:30:15.785Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038838043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62982024-02-01T17:30:15.785Z{"msg":"Next flush: 79","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038906435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
62992024-02-01T17:30:15.785ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63002024-02-01T17:30:15.785Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.038979575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"":"downstairs","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63012024-02-01T17:30:15.786Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039015611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63022024-02-01T17:30:15.786Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039059213Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"0","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63032024-02-01T17:30:15.786Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039084006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"1","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63042024-02-01T17:30:15.786Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039124119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"client":"2","":"downstairs","session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63052024-02-01T17:30:15.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039148542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63062024-02-01T17:30:15.787Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039187673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63072024-02-01T17:30:15.787Z{"msg":"1360ab16-c1b2-4a8d-a78e-6f2abf41852c is now active with session: 1d8a1245-2cc5-4a86-9f5d-c6d02b68997d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039211732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63082024-02-01T17:30:15.787Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039232298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361,"session_id":"1d8a1245-2cc5-4a86-9f5d-c6d02b68997d"}
63092024-02-01T17:30:15.788Z{"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:15.039271066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
63102024-02-01T17:30:15.788ZLoading write count information from file "/tmp/test_repair_verify.data"
63112024-02-01T17:30:15.788ZRun Repair workload
63122024-02-01T17:30:15.788Z01/30 Write block 440 len 12288 data: 6 6 6
63132024-02-01T17:30:15.788Z02/30 Read block 474 len 12288
63142024-02-01T17:30:15.792Z03/30 Read block 595 len 4096
63152024-02-01T17:30:15.792Z04/30 Read block 374 len 24576
63162024-02-01T17:30:15.792Z05/30 Write block 540 len 36864 data: 7 10 10 8 9 8 9 8 6
63172024-02-01T17:30:15.792Z06/30 Write block 50 len 24576 data: 9 9 6 7 9 9
63182024-02-01T17:30:15.792Z07/30 Write block 445 len 4096 data: 7
63192024-02-01T17:30:15.793Z08/30 Read block 482 len 36864
63202024-02-01T17:30:15.793Z09/30 Read block 582 len 36864
63212024-02-01T17:30:15.793Z10/30 Write block 587 len 8192 data: 6 7
63222024-02-01T17:30:15.793Z11/30 Write block 393 len 24576 data: 7 5 4 3 3 3
63232024-02-01T17:30:15.794Z12/30 Flush
63242024-02-01T17:30:15.797Z13/30 Write block 71 len 40960 data: 6 7 8 8 8 4 3 2 2 3
63252024-02-01T17:30:15.797Z14/30 Write block 481 len 12288 data: 6 6 6
63262024-02-01T17:30:15.797Z15/30 Read block 527 len 8192
63272024-02-01T17:30:15.798Z16/30 Read block 40 len 28672
63282024-02-01T17:30:15.801Z17/30 Read block 140 len 28672
63292024-02-01T17:30:15.801Z18/30 Read block 452 len 36864
63302024-02-01T17:30:15.801Z19/30 Read block 171 len 4096
63312024-02-01T17:30:15.805Z20/30 Flush
63322024-02-01T17:30:15.805Z21/30 Write block 159 len 16384 data: 5 6 7 6
63332024-02-01T17:30:15.809Z22/30 Write block 104 len 16384 data: 3 6 7 7
63342024-02-01T17:30:15.809Z23/30 Flush
63352024-02-01T17:30:15.809Z24/30 Write block 475 len 8192 data: 3 3
63362024-02-01T17:30:15.813Z25/30 Read block 253 len 36864
63372024-02-01T17:30:15.813Z26/30 Read block 503 len 40960
63382024-02-01T17:30:15.816Z27/30 Write block 89 len 32768 data: 4 4 3 4 3 3 3 4
63392024-02-01T17:30:15.816Z28/30 Write block 392 len 24576 data: 7 8 6 5 4 4
63402024-02-01T17:30:15.816Z29/30 Read block 88 len 16384
63412024-02-01T17:30:15.820Z30/30 Write block 284 len 12288 data: 4 7 8
63422024-02-01T17:30:15.820Z----------------------------------------------------------------
63432024-02-01T17:30:15.820Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:18
63442024-02-01T17:30:15.820ZStates: Active Active Active
63452024-02-01T17:30:15.820ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
63462024-02-01T17:30:15.820Z 13 Acked 1012 Write 10 Done Done Done false
63472024-02-01T17:30:15.820Z 14 Acked 1013 Write 3 Done Sent Done false
63482024-02-01T17:30:15.820Z 15 Acked 1014 Read 2 Done Sent Done false
63492024-02-01T17:30:15.820Z 16 Acked 1015 Read 7 Done Sent Done false
63502024-02-01T17:30:15.820Z 17 Acked 1016 Read 7 Done Sent Done false
63512024-02-01T17:30:15.820Z 18 Acked 1017 Read 9 Done Sent Done false
63522024-02-01T17:30:15.820Z 19 Acked 1018 Read 1 Done Sent Done false
63532024-02-01T17:30:15.821Z 20 Acked 1019 Flush 0 Done Sent Done false
63542024-02-01T17:30:15.821Z 21 Acked 1020 Write 4 Done Sent Done false
63552024-02-01T17:30:15.821Z 22 Acked 1021 Write 4 Done Sent Done false
63562024-02-01T17:30:15.821Z 23 Acked 1022 Flush 0 Done Sent Done false
63572024-02-01T17:30:15.821Z 24 Acked 1023 Write 2 Done Sent Done false
63582024-02-01T17:30:15.821Z 25 Acked 1024 Read 9 Done Sent Done false
63592024-02-01T17:30:15.821Z 26 Acked 1025 Read 10 Done Sent Done false
63602024-02-01T17:30:15.821Z 27 Acked 1026 Write 8 Done Sent Done false
63612024-02-01T17:30:15.821Z 28 Acked 1027 Write 6 Done Sent Done false
63622024-02-01T17:30:15.821Z 29 Acked 1028 Read 4 Done Sent Done false
63632024-02-01T17:30:15.821Z 30 Acked 1029 Write 3 Sent Sent Sent false
63642024-02-01T17:30:15.821Z STATES DS:0 DS:1 DS:2 TOTAL
63652024-02-01T17:30:15.821Z New 0 0 0 0
63662024-02-01T17:30:15.821Z Sent 1 17 1 19
63672024-02-01T17:30:15.821Z Done 17 1 17 35
63682024-02-01T17:30:15.821Z Skipped 0 0 0 0
63692024-02-01T17:30:15.821Z Error 0 0 0 0
63702024-02-01T17:30:15.821ZLast Flush: 1022 1011 1022
63712024-02-01T17:30:15.821ZDownstairs last five completed: 1011 1010 1009 1008 1007
63722024-02-01T17:30:15.821ZUpstairs last five completed: 30 29 28 27 26
63732024-02-01T17:30:15.822Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:15.085985853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1361}
63742024-02-01T17:30:15.822ZWrote out file "/tmp/test_repair_verify.data"
63752024-02-01T17:30:15.826Z
63762024-02-01T17:30:15.830ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63772024-02-01T17:30:15.857ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63782024-02-01T17:30:15.857Z 0 000-019 51 51 51 72 72 72 F F F
63792024-02-01T17:30:15.865Z 1 020-039 56 56 56 77 77 77 F F F
63802024-02-01T17:30:15.865Z 2 040-059 58 58 58 79 79 79 F F F
63812024-02-01T17:30:15.865Z 3 060-079 58 52 58 80 73 80 F T F <---
63822024-02-01T17:30:15.865Z 4 080-099 58 44 58 80 61 80 T T T <---
63832024-02-01T17:30:15.865Z 5 100-119 58 50 58 81 71 81 F F F <---
63842024-02-01T17:30:15.865Z 6 120-139 50 50 50 70 70 70 F F F
63852024-02-01T17:30:15.865Z 7 140-159 58 52 58 81 73 81 F F F <---
63862024-02-01T17:30:15.865Z 8 160-179 58 56 58 81 77 81 F F F <---
63872024-02-01T17:30:15.865Z 9 180-199 49 49 49 68 68 68 F F F
63882024-02-01T17:30:15.865Z 10 200-219 57 57 57 78 78 78 F F F
63892024-02-01T17:30:15.866Z 11 220-239 57 57 57 78 78 78 F F F
63902024-02-01T17:30:15.866Z 12 240-259 52 52 52 73 73 73 F F F
63912024-02-01T17:30:15.866Z 13 260-279 52 52 52 73 73 73 F F F
63922024-02-01T17:30:15.866Z 14 280-299 51 51 51 72 72 72 T F T <---
63932024-02-01T17:30:15.866Z 15 300-319 52 52 52 73 73 73 F F F
63942024-02-01T17:30:15.866Z 16 320-339 56 56 56 77 77 77 F F F
63952024-02-01T17:30:15.867Z 17 340-359 56 56 56 77 77 77 F F F
63962024-02-01T17:30:15.867Z 18 360-379 52 52 52 73 73 73 F F F
63972024-02-01T17:30:15.867Z 19 380-399 58 58 58 79 79 79 T F T <---
63982024-02-01T17:30:15.867Z 20 400-419 51 51 51 72 72 72 F F F
63992024-02-01T17:30:15.867Z 21 420-439 56 56 56 77 77 77 F F F
64002024-02-01T17:30:15.867Z 22 440-459 58 58 58 79 79 79 F F F
64012024-02-01T17:30:15.867Z 23 460-479 46 46 46 65 65 65 T F T <---
64022024-02-01T17:30:15.867Z 24 480-499 58 56 58 80 77 80 F F F <---
64032024-02-01T17:30:15.867Z 25 500-519 53 53 53 74 74 74 F F F
64042024-02-01T17:30:15.867Z 26 520-539 52 52 52 73 73 73 F F F
64052024-02-01T17:30:15.867Z 27 540-559 58 58 58 79 79 79 F F F
64062024-02-01T17:30:15.867Z 28 560-579 57 57 57 78 78 78 F F F
64072024-02-01T17:30:15.867Z 29 580-599 58 58 58 79 79 79 F F F
64082024-02-01T17:30:15.867ZMax gen: 58, Max flush: 81
64092024-02-01T17:30:15.868ZError: Difference in extent metadata found!
64102024-02-01T17:30:15.868ZA Difference in extent metadata is expected here
64112024-02-01T17:30:15.868ZOn loop 29
64122024-02-01T17:30:15.868Z
64132024-02-01T17:30:15.881ZVerifying data now
64142024-02-01T17:30:20.714ZNo change after verify
64152024-02-01T17:30:20.714ZLoop: 29 Downstairs dump after verify (and repair):
64162024-02-01T17:30:20.740ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64172024-02-01T17:30:20.740Z 0 000-019 51 51 51 72 72 72 F F F
64182024-02-01T17:30:20.740Z 1 020-039 56 56 56 77 77 77 F F F
64192024-02-01T17:30:20.741Z 2 040-059 58 58 58 79 79 79 F F F
64202024-02-01T17:30:20.741Z 3 060-079 58 58 58 80 80 80 F F F
64212024-02-01T17:30:20.741Z 4 080-099 59 59 59 82 82 82 F F F
64222024-02-01T17:30:20.741Z 5 100-119 58 58 58 81 81 81 F F F
64232024-02-01T17:30:20.741Z 6 120-139 50 50 50 70 70 70 F F F
64242024-02-01T17:30:20.741Z 7 140-159 58 58 58 81 81 81 F F F
64252024-02-01T17:30:20.742Z 8 160-179 58 58 58 81 81 81 F F F
64262024-02-01T17:30:20.742Z 9 180-199 49 49 49 68 68 68 F F F
64272024-02-01T17:30:20.742Z 10 200-219 57 57 57 78 78 78 F F F
64282024-02-01T17:30:20.742Z 11 220-239 57 57 57 78 78 78 F F F
64292024-02-01T17:30:20.742Z 12 240-259 52 52 52 73 73 73 F F F
64302024-02-01T17:30:20.743Z 13 260-279 52 52 52 73 73 73 F F F
64312024-02-01T17:30:20.743Z 14 280-299 59 59 59 82 82 82 F F F
64322024-02-01T17:30:20.743Z 15 300-319 52 52 52 73 73 73 F F F
64332024-02-01T17:30:20.743Z 16 320-339 56 56 56 77 77 77 F F F
64342024-02-01T17:30:20.744Z 17 340-359 56 56 56 77 77 77 F F F
64352024-02-01T17:30:20.745Z 18 360-379 52 52 52 73 73 73 F F F
64362024-02-01T17:30:20.745Z 19 380-399 59 59 59 82 82 82 F F F
64372024-02-01T17:30:20.745Z 20 400-419 51 51 51 72 72 72 F F F
64382024-02-01T17:30:20.746Z 21 420-439 56 56 56 77 77 77 F F F
64392024-02-01T17:30:20.746Z 22 440-459 58 58 58 79 79 79 F F F
64402024-02-01T17:30:20.746Z 23 460-479 59 59 59 82 82 82 F F F
64412024-02-01T17:30:20.746Z 24 480-499 58 58 58 80 80 80 F F F
64422024-02-01T17:30:20.746Z 25 500-519 53 53 53 74 74 74 F F F
64432024-02-01T17:30:20.747Z 26 520-539 52 52 52 73 73 73 F F F
64442024-02-01T17:30:20.747Z 27 540-559 58 58 58 79 79 79 F F F
64452024-02-01T17:30:20.747Z 28 560-579 57 57 57 78 78 78 F F F
64462024-02-01T17:30:20.747Z 29 580-599 58 58 58 79 79 79 F F F
64472024-02-01T17:30:20.747ZMax gen: 59, Max flush: 82
64482024-02-01T17:30:20.748Z
64492024-02-01T17:30:20.748ZBegin loop 30
64502024-02-01T17:30:20.748ZDownstairs to restart: 1
64512024-02-01T17:30:20.758Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007170041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
64522024-02-01T17:30:20.758Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.00767344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
64532024-02-01T17:30:20.759Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007742098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
64542024-02-01T17:30:20.759Z{"msg":"Crucible 46468bfe-c54c-4bdb-b780-90f2ca6c5184 has session id: a449c3b7-1403-4743-9f2c-e1c31bbf4f5e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007782526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64552024-02-01T17:30:20.759Z{"msg":"Upstairs opts: Upstairs UUID: 46468bfe-c54c-4bdb-b780-90f2ca6c5184, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007830175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":Crucible runtime is spawned
64562024-02-01T17:30:20.759Z1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64572024-02-01T17:30:20.760Z{"msg":"Crucible stats registered with UUID: 46468bfe-c54c-4bdb-b780-90f2ca6c5184","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007944758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64582024-02-01T17:30:20.760Z{"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.007992554Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
64592024-02-01T17:30:20.760Z{"msg":"46468bfe-c54c-4bdb-b780-90f2ca6c5184 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.00833838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64602024-02-01T17:30:20.760Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008424962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64612024-02-01T17:30:20.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008497515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64622024-02-01T17:30:20.761Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008538368Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64632024-02-01T17:30:20.761Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008584509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64642024-02-01T17:30:20.777Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008627187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64652024-02-01T17:30:20.777Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008673928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64662024-02-01T17:30:20.777Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.008912663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64672024-02-01T17:30:20.777Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.00899963Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64682024-02-01T17:30:20.778Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.009045232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64692024-02-01T17:30:20.778Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.009147549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64702024-02-01T17:30:20.778Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.009194229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64712024-02-01T17:30:20.778Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.009239796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"io task","client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64722024-02-01T17:30:20.778Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.010116489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64732024-02-01T17:30:20.778Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.010171931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64742024-02-01T17:30:20.778Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.010195677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64752024-02-01T17:30:20.778Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.010240214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64762024-02-01T17:30:20.778Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.01026488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64772024-02-01T17:30:20.778Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.010352023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64782024-02-01T17:30:20.778Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.011092369Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64792024-02-01T17:30:20.778Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.011259429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64802024-02-01T17:30:20.779Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.011310109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64812024-02-01T17:30:20.779Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012172957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64822024-02-01T17:30:20.779Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012253011Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64832024-02-01T17:30:20.779Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012359589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64842024-02-01T17:30:20.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012404788Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64852024-02-01T17:30:20.780Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012466276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64862024-02-01T17:30:20.780Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012510916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64872024-02-01T17:30:20.780Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012561791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64882024-02-01T17:30:20.780Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012605062Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64892024-02-01T17:30:20.780Z{"msg":"[0]R flush_numbers[0..12]: [72, 77, 79, 80, 82, 81, 70, 81, 81, 68, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012658673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64902024-02-01T17:30:20.780Z{"msg":"[0]R generation[0..12]: [51, 56, 58, 58, 59, 58, 50, 58, 58, 49, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012703382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64912024-02-01T17:30:20.780Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012758312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64922024-02-01T17:30:20.780Z{"msg":"[1]R flush_numbers[0..12]: [72, 77, 79, 80, 82, 81, 70, 81, 81, 68, 78, 78]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
64932024-02-01T17:30:20.780Z,"time":"2024-02-01T17:30:20.012804236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64942024-02-01T17:30:20.782Z{"msg":"[1]R generation[0..12]: [51, 56, 58, 58, 59, 58, 50, 58, 58, 49, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012866089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64952024-02-01T17:30:20.782Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.012911454Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64962024-02-01T17:30:20.782Z{"msg":"[2]R flush_numbers[0..12]: [72, 77, 79, 80, 82, 81, 70, 81, 81, 68, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.01297545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64972024-02-01T17:30:20.782Z{"msg":"[2]R generation[0..12]: [51, 56, 58, 58, 59, 58, 50, 58, 58, 49, 57, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.01308637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64982024-02-01T17:30:20.782Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013116498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
64992024-02-01T17:30:20.782Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013158799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65002024-02-01T17:30:20.782Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013184663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65012024-02-01T17:30:20.782Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013206629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65022024-02-01T17:30:20.783Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013259978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"":"downstairs","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65032024-02-01T17:30:20.783Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013284385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65042024-02-01T17:30:20.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013304752Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"0","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65052024-02-01T17:30:20.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013350395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"1","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65062024-02-01T17:30:20.783Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013374753Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"client":"2","":"downstairs","session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65072024-02-01T17:30:20.783Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013397085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65082024-02-01T17:30:20.783Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013436111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65092024-02-01T17:30:20.783Z{"msg":"46468bfe-c54c-4bdb-b780-90f2ca6c5184 is now active with session: a449c3b7-1403-4743-9f2c-e1c31bbf4f5e","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
65102024-02-01T17:30:20.783Z,"time":"2024-02-01T17:30:20.01346002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65112024-02-01T17:30:20.784Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013502787Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369,"session_id":"a449c3b7-1403-4743-9f2c-e1c31bbf4f5e"}
65122024-02-01T17:30:20.784Z{"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:20.013526047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
65132024-02-01T17:30:20.784ZLoading write count information from file "/tmp/test_repair_verify.data"
65142024-02-01T17:30:20.784ZRun Repair workload
65152024-02-01T17:30:20.784Z01/30 Write block 407 len 40960 data: 5 5 4 2 2 2 2 4 4 6
65162024-02-01T17:30:20.784Z02/30 Write block 182 len 16384 data: 8 10 9 10
65172024-02-01T17:30:20.784Z03/30 Read block 242 len 32768
65182024-02-01T17:30:20.800Z04/30 Read block 252 len 8192
65192024-02-01T17:30:20.800Z05/30 Write block 59 len 32768 data: 6 7 6 5 5 4 2 3
65202024-02-01T17:30:20.800Z06/30 Write block 148 len 20480 data: 5 5 4 4 8
65212024-02-01T17:30:20.800Z07/30 Read block 208 len 8192
65222024-02-01T17:30:20.800Z08/30 Read block 67 len 36864
65232024-02-01T17:30:20.800Z09/30 Read block 589 len 36864
65242024-02-01T17:30:20.800Z10/30 Read block 198 len 4096
65252024-02-01T17:30:20.800Z11/30 Write block 361 len 8192 data: 8 8
65262024-02-01T17:30:20.800Z12/30 Write block 598 len 8192 data: 3 3
65272024-02-01T17:30:20.800Z13/30 Read block 81 len 4096
65282024-02-01T17:30:20.800Z14/30 Write block 326 len 8192 data: 4 5
65292024-02-01T17:30:20.800Z15/30 Flush
65302024-02-01T17:30:20.800Z16/30 Write block 441 len 12288 data: 7 7 6
65312024-02-01T17:30:20.801Z17/30 Flush
65322024-02-01T17:30:20.801Z18/30 Write block 526 len 36864 data: 5 4 4 4 5 4 5 5 7
65332024-02-01T17:30:20.801Z19/30 Read block 160 len 40960
65342024-02-01T17:30:20.801Z20/30 Read block 490 len 32768
65352024-02-01T17:30:20.801Z21/30 Write block 444 len 40960 data: 7 8 6 6 6 7 5 4 4 4
65362024-02-01T17:30:20.802Z22/30 Read block 430 len 12288
65372024-02-01T17:30:20.802Z23/30 Write block 265 len 36864 data: 5 6 6 6 6 7 7 5 5
65382024-02-01T17:30:20.802Z24/30 Write block 412 len 32768 data: 3 3 5 5 7 4 5 6
65392024-02-01T17:30:20.802Z25/30 Write block 155 len 40960 data: 8 6 5 6 6 7 8 7 5 5
65402024-02-01T17:30:20.802Z26/30 Write block 207 len 16384 data: 11 11 9 6
65412024-02-01T17:30:20.802Z27/30 Write block 65 len 40960 data: 3 4 4 4 6 6 7 8 9 9
65422024-02-01T17:30:20.813Z28/30 Read block 181 len 8192
65432024-02-01T17:30:20.814Z29/30 Read block 19 len 36864
65442024-02-01T17:30:20.814Z30/30 Write block 562 len 4096 data: 8
65452024-02-01T17:30:20.814Z----------------------------------------------------------------
65462024-02-01T17:30:20.814Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
65472024-02-01T17:30:20.814ZStates: Active Active Active
65482024-02-01T17:30:20.814ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65492024-02-01T17:30:20.814Z 1 Acked 1000 Write 10 Done Done Done false
65502024-02-01T17:30:20.814Z 2 Acked 1001 Write 4 Done Done Done false
65512024-02-01T17:30:20.814Z 3 Acked 1002 Read 8 Done Done Done false
65522024-02-01T17:30:20.814Z 4 Acked 1003 Read 2 Done Sent Done false
65532024-02-01T17:30:20.814Z 5 Acked 1004 Write 8 Done Sent Done false
65542024-02-01T17:30:20.816Z 6 Acked 1005 Write 5 Done Sent Done false
65552024-02-01T17:30:20.816Z 7 Acked 1006 Read 2 Done Sent Done false
65562024-02-01T17:30:20.816Z 8 Acked 1007 Read 9 Done Sent Done false
65572024-02-01T17:30:20.816Z 9 Acked 1008 Read 9 Done Sent Done false
65582024-02-01T17:30:20.816Z 10 Acked 1009 Read 1 Done Sent Done false
65592024-02-01T17:30:20.816Z 11 Acked 1010 Write 2 Done Sent Done false
65602024-02-01T17:30:20.816Z 12 Acked 1011 Write 2 Done Sent Done false
65612024-02-01T17:30:20.816Z 13 Acked 1012 Read 1 Done Sent Done false
65622024-02-01T17:30:20.816Z 14 Acked 1013 Write 2 Done Sent Done false
65632024-02-01T17:30:20.816Z 15 Acked 1014 Flush 0 Done Sent Done false
65642024-02-01T17:30:20.816Z 16 Acked 1015 Write 3 Done Sent Done false
65652024-02-01T17:30:20.816Z 17 Acked 1016 Flush 0 Done Sent Done false
65662024-02-01T17:30:20.818Z 18 Acked 1017 Write 9 Done Sent Done false
65672024-02-01T17:30:20.818Z 19 Acked 1018 Read 10 Done Sent Done false
65682024-02-01T17:30:20.818Z 20 Acked 1019 Read 8 Done Sent Done false
65692024-02-01T17:30:20.818Z 21 Acked 1020 Write 10 Done Sent Done false
65702024-02-01T17:30:20.818Z 22 Acked 1021 Read 3 Done Sent Done false
65712024-02-01T17:30:20.818Z 23 Acked 1022 Write 9 Done Sent Done false
65722024-02-01T17:30:20.818Z 24 Acked 1023 Write 8 Done Sent Done false
65732024-02-01T17:30:20.818Z 25 Acked 1024 Write 10 Done Sent Done false
65742024-02-01T17:30:20.818Z 26 Acked 1025 Write 4 Done Sent Done false
65752024-02-01T17:30:20.818Z 27 Acked 1026 Write 10 Done Sent Done false
65762024-02-01T17:30:20.818Z 28 Acked 1027 Read 2 Done Sent Done false
65772024-02-01T17:30:20.818Z 29 Acked 1028 Read 9 Done Sent Done false
65782024-02-01T17:30:20.818Z 30 Acked 1029 Write 1 Sent Sent Sent false
65792024-02-01T17:30:20.818Z STATES DS:0 DS:1 DS:2 TOTAL
65802024-02-01T17:30:20.818Z New 0 0 0 0
65812024-02-01T17:30:20.818Z Sent 1 27 1 29
65822024-02-01T17:30:20.819Z Done 29 3 29 61
65832024-02-01T17:30:20.819Z Skipped 0 0 0 0
65842024-02-01T17:30:20.819Z Error 0 0 0 0
65852024-02-01T17:30:20.819ZLast Flush: 1016 0 1016
65862024-02-01T17:30:20.819ZDownstairs last five completed:
65872024-02-01T17:30:20.819ZUpstairs last five completed: 30 29 28 27 26
65882024-02-01T17:30:20.819Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:20.060158838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1369}
65892024-02-01T17:30:20.819ZWrote out file "/tmp/test_repair_verify.data"
65902024-02-01T17:30:20.819Z
65912024-02-01T17:30:20.819ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65922024-02-01T17:30:20.833ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65932024-02-01T17:30:20.833Z 0 000-019 51 51 51 72 72 72 F F F
65942024-02-01T17:30:20.833Z 1 020-039 56 56 56 77 77 77 F F F
65952024-02-01T17:30:20.834Z 2 040-059 60 58 60 83 79 83 F F F <---
65962024-02-01T17:30:20.834Z 3 060-079 60 58 60 83 80 83 T F T <---
65972024-02-01T17:30:20.834Z 4 080-099 59 59 59 82 82 82 F F F
65982024-02-01T17:30:20.839Z 5 100-119 58 58 58 81 81 81 F F F
65992024-02-01T17:30:20.839Z 6 120-139 50 50 50 70 70 70 F F F
66002024-02-01T17:30:20.839Z 7 140-159 60 58 60 83 81 83 T F T <---
66012024-02-01T17:30:20.839Z 8 160-179 58 58 58 81 81 81 T F T <---
66022024-02-01T17:30:20.839Z 9 180-199 60 49 60 83 68 83 F T F <---
66032024-02-01T17:30:20.839Z 10 200-219 57 57 57 78 78 78 T F T <---
66042024-02-01T17:30:20.839Z 11 220-239 57 57 57 78 78 78 F F F
66052024-02-01T17:30:20.840Z 12 240-259 52 52 52 73 73 73 F F F
66062024-02-01T17:30:20.840Z 13 260-279 52 52 52 73 73 73 T F T <---
66072024-02-01T17:30:20.840Z 14 280-299 59 59 59 82 82 82 F F F
66082024-02-01T17:30:20.840Z 15 300-319 52 52 52 73 73 73 F F F
66092024-02-01T17:30:20.840Z 16 320-339 60 56 60 83 77 83 F F F <---
66102024-02-01T17:30:20.840Z 17 340-359 56 56 56 77 77 77 F F F
66112024-02-01T17:30:20.841Z 18 360-379 60 52 60 83 73 83 F F F <---
66122024-02-01T17:30:20.841Z 19 380-399 59 59 59 82 82 82 F F F
66132024-02-01T17:30:20.841Z 20 400-419 60 51 60 83 72 83 T T T <---
66142024-02-01T17:30:20.841Z 21 420-439 56 56 56 77 77 77 F F F
66152024-02-01T17:30:20.841Z 22 440-459 60 58 60 84 79 84 T F T <---
66162024-02-01T17:30:20.841Z 23 460-479 59 59 59 82 82 82 F F F
66172024-02-01T17:30:20.841Z 24 480-499 58 58 58 80 80 80 F F F
66182024-02-01T17:30:20.841Z 25 500-519 53 53 53 74 74 74 F F F
66192024-02-01T17:30:20.841Z 26 520-539 52 52 52 73 73 73 T F T <---
66202024-02-01T17:30:20.841Z 27 540-559 58 58 58 79 79 79 F F F
66212024-02-01T17:30:20.841Z 28 560-579 57 57 57 78 78 78 T F T <---
66222024-02-01T17:30:20.841Z 29 580-599 60 58 60 83 79 83 F F F <---
66232024-02-01T17:30:20.842ZMax gen: 60, Max flush: 84
66242024-02-01T17:30:20.842ZError: Difference in extent metadata found!
66252024-02-01T17:30:20.842ZA Difference in extent metadata is expected here
66262024-02-01T17:30:20.842ZOn loop 30
66272024-02-01T17:30:20.842Z
66282024-02-01T17:30:20.850ZVerifying data now
66292024-02-01T17:30:27.839ZNo change after verify
66302024-02-01T17:30:27.839ZLoop: 30 Downstairs dump after verify (and repair):
66312024-02-01T17:30:27.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
66322024-02-01T17:30:27.860Z 0 000-019 51 51 51 72 72 72 F F F
66332024-02-01T17:30:27.860Z 1 020-039 56 56 56 77 77 77 F F F
66342024-02-01T17:30:27.860Z 2 040-059 60 60 60 83 83 83 F F F
66352024-02-01T17:30:27.863Z 3 060-079 61 61 61 85 85 85 F F F
66362024-02-01T17:30:27.863Z 4 080-099 59 59 59 82 82 82 F F F
66372024-02-01T17:30:27.864Z 5 100-119 58 58 58 81 81 81 F F F
66382024-02-01T17:30:27.864Z 6 120-139 50 50 50 70 70 70 F F F
66392024-02-01T17:30:27.864Z 7 140-159 61 61 61 85 85 85 F F F
66402024-02-01T17:30:27.864Z 8 160-179 61 61 61 85 85 85 F F F
66412024-02-01T17:30:27.864Z 9 180-199 60 60 60 83 83 83 F F F
66422024-02-01T17:30:27.865Z 10 200-219 61 61 61 85 85 85 F F F
66432024-02-01T17:30:27.865Z 11 220-239 57 57 57 78 78 78 F F F
66442024-02-01T17:30:27.865Z 12 240-259 52 52 52 73 73 73 F F F
66452024-02-01T17:30:27.865Z 13 260-279 61 61 61 85 85 85 F F F
66462024-02-01T17:30:27.865Z 14 280-299 59 59 59 82 82 82 F F F
66472024-02-01T17:30:27.865Z 15 300-319 52 52 52 73 73 73 F F F
66482024-02-01T17:30:27.865Z 16 320-339 60 60 60 83 83 83 F F F
66492024-02-01T17:30:27.865Z 17 340-359 56 56 56 77 77 77 F F F
66502024-02-01T17:30:27.866Z 18 360-379 60 60 60 83 83 83 F F F
66512024-02-01T17:30:27.866Z 19 380-399 59 59 59 82 82 82 F F F
66522024-02-01T17:30:27.866Z 20 400-419 61 61 61 85 85 85 F F F
66532024-02-01T17:30:27.866Z 21 420-439 56 56 56 77 77 77 F F F
66542024-02-01T17:30:27.866Z 22 440-459 61 61 61 85 85 85 F F F
66552024-02-01T17:30:27.866Z 23 460-479 59 59 59 82 82 82 F F F
66562024-02-01T17:30:27.867Z 24 480-499 58 58 58 80 80 80 F F F
66572024-02-01T17:30:27.867Z 25 500-519 53 53 53 74 74 74 F F F
66582024-02-01T17:30:27.867Z 26 520-539 61 61 61 85 85 85 F F F
66592024-02-01T17:30:27.867Z 27 540-559 58 58 58 79 79 79 F F F
66602024-02-01T17:30:27.868Z 28 560-579 61 61 61 85 85 85 F F F
66612024-02-01T17:30:27.868Z 29 580-599 60 60 60 83 83 83 F F F
66622024-02-01T17:30:27.868ZMax gen: 61, Max flush: 85
66632024-02-01T17:30:27.868Z
66642024-02-01T17:30:27.869ZBegin loop 31
66652024-02-01T17:30:27.869ZDownstairs to restart: 2
66662024-02-01T17:30:27.869Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.354775779Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
66672024-02-01T17:30:27.869Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355250662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
66682024-02-01T17:30:27.869Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355283644Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
66692024-02-01T17:30:27.869Z{"msg":"Crucible b5a89aa0-ce47-4bd9-8144-3e569d41cda8 has session id: 79a7f028-758b-4116-856d-43f46eb3c9f8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355321439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66702024-02-01T17:30:27.869Z{"msg":"Upstairs opts: Upstairs UUID: b5a89aa0-ce47-4bd9-8144-3e569d41cda8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":Crucible runtime is spawned
66712024-02-01T17:30:27.869Z"2024-02-01T17:30:25.355365965Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66722024-02-01T17:30:27.869Z{"msg":"Crucible stats registered with UUID: b5a89aa0-ce47-4bd9-8144-3e569d41cda8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.35551936Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66732024-02-01T17:30:27.869Z{"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355592376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
66742024-02-01T17:30:27.870Z{"msg":"b5a89aa0-ce47-4bd9-8144-3e569d41cda8 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355632257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66752024-02-01T17:30:27.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355672052Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66762024-02-01T17:30:27.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355715737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66772024-02-01T17:30:27.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355837092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66782024-02-01T17:30:27.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.355898737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66792024-02-01T17:30:27.870Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356000783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66802024-02-01T17:30:27.870Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356136877Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66812024-02-01T17:30:27.870Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356186568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66822024-02-01T17:30:27.870Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356226561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66832024-02-01T17:30:27.870Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356353999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66842024-02-01T17:30:27.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356425045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66852024-02-01T17:30:27.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356455254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66862024-02-01T17:30:27.871Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.356523906Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"io task","client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66872024-02-01T17:30:27.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.35745875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66882024-02-01T17:30:27.871Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.357543268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66892024-02-01T17:30:27.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.357571567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66902024-02-01T17:30:27.871Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.357709918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66912024-02-01T17:30:27.871Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.357921748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66922024-02-01T17:30:27.871Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.357985037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66932024-02-01T17:30:27.871Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.358714687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66942024-02-01T17:30:27.872Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.358860517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66952024-02-01T17:30:27.872Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.358971874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66962024-02-01T17:30:27.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359710583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66972024-02-01T17:30:27.872Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.35977022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66982024-02-01T17:30:27.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359796814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
66992024-02-01T17:30:27.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359817983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67002024-02-01T17:30:27.872Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359858903Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67012024-02-01T17:30:27.872Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359883193Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67022024-02-01T17:30:27.872Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.35992242Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67032024-02-01T17:30:27.872Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359946698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67042024-02-01T17:30:27.873Z{"msg":"[0]R flush_numbers[0..12]: [72, 77, 83, 85, 82, 81, 70, 85, 85, 83, 85, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.359987635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67052024-02-01T17:30:27.873Z{"msg":"[0]R generation[0..12]: [51, 56, 60, 61, 59, 58, 50, 61, 61, 60, 61, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360013109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67062024-02-01T17:30:27.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360037145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67072024-02-01T17:30:27.873Z{"msg":"[1]R flush_numbers[0..12]: [72, 77, 83, 85, 82, 81, 70, 85, 85, 83, 85, 78]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.36006069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67082024-02-01T17:30:27.873Z{"msg":"[1]R generation[0..12]: [51, 56, 60, 61, 59, 58, 50, 61, 61, 60, 61, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360101946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67092024-02-01T17:30:27.874Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360127529Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67102024-02-01T17:30:27.874Z{"msg":"[2]R flush_numbers[0..12]: [72, 77, 83, 85, 82, 81, 70, 85, 85, 83, 85, 78]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
67112024-02-01T17:30:27.874Z,"time":"2024-02-01T17:30:25.360197141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67122024-02-01T17:30:27.874Z{"msg":"[2]R generation[0..12]: [51, 56, 60, 61, 59, 58, 50, 61, 61, 60, 61, 57]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360327283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67132024-02-01T17:30:27.874Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360357502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67142024-02-01T17:30:27.874Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360379847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67152024-02-01T17:30:27.874Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360427156Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67162024-02-01T17:30:27.874Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360464744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67172024-02-01T17:30:27.874Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360518459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"":"downstairs","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67182024-02-01T17:30:27.874Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360543562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67192024-02-01T17:30:27.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360563749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"0","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67202024-02-01T17:30:27.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360604531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"1","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67212024-02-01T17:30:27.874Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360628325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"client":"2","":"downstairs","session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67222024-02-01T17:30:27.874Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360668719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67232024-02-01T17:30:27.874Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360691056Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67242024-02-01T17:30:27.874Z{"msg":"b5a89aa0-ce47-4bd9-8144-3e569d41cda8 is now active with session: 79a7f028-758b-4116-856d-43f46eb3c9f8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360735174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67252024-02-01T17:30:27.874Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.3607597Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377,"session_id":"79a7f028-758b-4116-856d-43f46eb3c9f8"}
67262024-02-01T17:30:27.874Z{"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:25.360801055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
67272024-02-01T17:30:27.874ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67282024-02-01T17:30:27.874ZLoading write count information from file "/tmp/test_repair_verify.data"
67292024-02-01T17:30:27.875ZRun Repair workload
67302024-02-01T17:30:27.875Z01/30 Write block 495 len 36864 data: 5 5 3 4 3 3 3 4 4
67312024-02-01T17:30:27.896Z02/30 Write block 530 len 36864 data: 6 5 6 6 8 6 6 7 5
67322024-02-01T17:30:27.896Z03/30 Read block 564 len 12288
67332024-02-01T17:30:27.896Z04/30 Write block 7 len 16384 data: 5 5 3 4
67342024-02-01T17:30:27.896Z05/30 Read block 194 len 16384
67352024-02-01T17:30:27.896Z06/30 Read block 225 len 32768
67362024-02-01T17:30:27.896Z07/30 Flush
67372024-02-01T17:30:27.896Z08/30 Write block 544 len 36864 data: 10 9 10 9 7 6 6 7 7
67382024-02-01T17:30:27.896Z09/30 Write block 378 len 40960 data: 9 10 10 11 7 7 6 4 3 3
67392024-02-01T17:30:27.897Z10/30 Read block 322 len 8192
67402024-02-01T17:30:27.897Z11/30 Flush
67412024-02-01T17:30:27.897Z12/30 Write block 176 len 28672 data: 11 10 6 6 5 7 9
67422024-02-01T17:30:27.897Z13/30 Flush
67432024-02-01T17:30:27.897Z14/30 Write block 312 len 20480 data: 5 7 6 8 8
67442024-02-01T17:30:27.897Z15/30 Read block 490 len 12288
67452024-02-01T17:30:27.897Z16/30 Read block 450 len 4096
67462024-02-01T17:30:27.897Z17/30 Write block 278 len 28672 data: 6 4 4 4 3 4 5
67472024-02-01T17:30:27.897Z18/30 Flush
67482024-02-01T17:30:27.899Z19/30 Flush
67492024-02-01T17:30:27.899Z20/30 Write block 140 len 12288 data: 4 4 3
67502024-02-01T17:30:27.899Z21/30 Read block 53 len 40960
67512024-02-01T17:30:27.899Z22/30 Flush
67522024-02-01T17:30:27.900Z23/30 Write block 435 len 4096 data: 8
67532024-02-01T17:30:27.900Z24/30 Write block 238 len 28672 data: 3 3 3 2 4 4 4
67542024-02-01T17:30:27.900Z25/30 Write block 467 len 40960 data: 7 6 4 5 5 5 3 2 4 4
67552024-02-01T17:30:27.900Z26/30 Write block 468 len 24576 data: 7 5 6 6 6 4
67562024-02-01T17:30:27.900Z27/30 Read block 116 len 16384
67572024-02-01T17:30:27.900Z28/30 Read block 149 len 40960
67582024-02-01T17:30:27.900Z29/30 Read block 180 len 4096
67592024-02-01T17:30:27.900Z30/30 Write block 189 len 36864 data: 6 6 5 4 4 5 5 6 7
67602024-02-01T17:30:27.900Z----------------------------------------------------------------
67612024-02-01T17:30:27.900Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
67622024-02-01T17:30:27.900ZStates: Active Active Active
67632024-02-01T17:30:27.900ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67642024-02-01T17:30:27.900Z 1 Acked 1000 Write 9 Done Done Sent false
67652024-02-01T17:30:27.900Z 2 Acked 1001 Write 9 Done Done Sent false
67662024-02-01T17:30:27.900Z 3 Acked 1002 Read 3 Done Done Sent false
67672024-02-01T17:30:27.900Z 4 Acked 1003 Write 4 Done Done Sent false
67682024-02-01T17:30:27.900Z 5 Acked 1004 Read 4 Done Done Sent false
67692024-02-01T17:30:27.900Z 6 Acked 1005 Read 8 Done Done Sent false
67702024-02-01T17:30:27.900Z 7 Acked 1006 Flush 0 Done Done Sent false
67712024-02-01T17:30:27.900Z 8 Acked 1007 Write 9 Done Done Sent false
67722024-02-01T17:30:27.900Z 9 Acked 1008 Write 10 Done Done Sent false
67732024-02-01T17:30:27.900Z 10 Acked 1009 Read 2 Done Done Sent false
67742024-02-01T17:30:27.900Z 11 Acked 1010 Flush 0 Done Done Sent false
67752024-02-01T17:30:27.901Z 12 Acked 1011 Write 7 Done Done Sent false
67762024-02-01T17:30:27.901Z 13 Acked 1012 Flush 0 Done Done Sent false
67772024-02-01T17:30:27.901Z 14 Acked 1013 Write 5 Done Done Sent false
67782024-02-01T17:30:27.901Z 15 Acked 1014 Read 3 Done Done Sent false
67792024-02-01T17:30:27.901Z 16 Acked 1015 Read 1 Done Done Sent false
67802024-02-01T17:30:27.901Z 17 Acked 1016 Write 7 Done Done Sent false
67812024-02-01T17:30:27.901Z 18 Acked 1017 Flush 0 Done Done Sent false
67822024-02-01T17:30:27.901Z 19 Acked 1018 Flush 0 Done Done Sent false
67832024-02-01T17:30:27.901Z 20 Acked 1019 Write 3 Done Done Sent false
67842024-02-01T17:30:27.901Z 21 Acked 1020 Read 10 Done Done Sent false
67852024-02-01T17:30:27.901Z 22 Acked 1021 Flush 0 Done Done Sent false
67862024-02-01T17:30:27.901Z 23 Acked 1022 Write 1 Done Done Sent false
67872024-02-01T17:30:27.901Z 24 Acked 1023 Write 7 Done Done Sent false
67882024-02-01T17:30:27.901Z 25 Acked 1024 Write 10 Done Done Sent false
67892024-02-01T17:30:27.901Z 26 Acked 1025 Write 6 Done Done Sent false
67902024-02-01T17:30:27.901Z 27 Acked 1026 Read 4 Done Done Sent false
67912024-02-01T17:30:27.901Z 28 Acked 1027 Read 10 Done Done Sent false
67922024-02-01T17:30:27.901Z 29 Acked 1028 Read 1 Done Done Sent false
67932024-02-01T17:30:27.901Z 30 Acked 1029 Write 9 Sent Sent Sent false
67942024-02-01T17:30:27.901Z STATES DS:0 DS:1 DS:2 TOTAL
67952024-02-01T17:30:27.902Z New 0 0 0 0
67962024-02-01T17:30:27.902Z Sent 1 1 30 32
67972024-02-01T17:30:27.902Z Done 29 29 0 58
67982024-02-01T17:30:27.902Z Skipped 0 0 0 0
67992024-02-01T17:30:27.902Z Error 0 0 0 0
68002024-02-01T17:30:27.902ZLast Flush: 1021 1021 0
68012024-02-01T17:30:27.902ZDownstairs last five completed:
68022024-02-01T17:30:27.902ZUpstairs last five completed: 30 29 28 27 26
68032024-02-01T17:30:27.902Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:25.412391808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1377}
68042024-02-01T17:30:27.902ZWrote out file "/tmp/test_repair_verify.data"
68052024-02-01T17:30:27.902Z
68062024-02-01T17:30:27.902ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
68072024-02-01T17:30:27.902ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68082024-02-01T17:30:27.902Z 0 000-019 62 62 51 86 86 72 F F F <---
68092024-02-01T17:30:27.902Z 1 020-039 56 56 56 77 77 77 F F F
68102024-02-01T17:30:27.903Z 2 040-059 60 60 60 83 83 83 F F F
68112024-02-01T17:30:27.903Z 3 060-079 61 61 61 85 85 85 F F F
68122024-02-01T17:30:27.903Z 4 080-099 59 59 59 82 82 82 F F F
68132024-02-01T17:30:27.903Z 5 100-119 58 58 58 81 81 81 F F F
68142024-02-01T17:30:27.903Z 6 120-139 50 50 50 70 70 70 F F F
68152024-02-01T17:30:27.903Z 7 140-159 62 62 61 91 91 85 F F F <---
68162024-02-01T17:30:27.903Z 8 160-179 62 62 61 88 88 85 F F F <---
68172024-02-01T17:30:27.903Z 9 180-199 62 62 60 88 88 83 T T F <---
68182024-02-01T17:30:27.903Z 10 200-219 61 61 61 85 85 85 F F F
68192024-02-01T17:30:27.903Z 11 220-239 57 57 57 78 78 78 T T F <---
68202024-02-01T17:30:27.903Z 12 240-259 52 52 52 73 73 73 T T F <---
68212024-02-01T17:30:27.903Z 13 260-279 62 62 61 89 89 85 F F F <---
68222024-02-01T17:30:27.903Z 14 280-299 62 62 59 89 89 82 F F F <---
68232024-02-01T17:30:27.903Z 15 300-319 62 62 52 89 89 73 F F F <---
68242024-02-01T17:30:27.903Z 16 320-339 60 60 60 83 83 83 F F F
68252024-02-01T17:30:27.903Z 17 340-359 56 56 56 77 77 77 F F F
68262024-02-01T17:30:27.903Z 18 360-379 62 62 60 87 87 83 F F F <---
68272024-02-01T17:30:27.903Z 19 380-399 62 62 59 87 87 82 F F F <---
68282024-02-01T17:30:27.903Z 20 400-419 61 61 61 85 85 85 F F F
68292024-02-01T17:30:27.903Z 21 420-439 56 56 56 77 77 77 T T F <---
68302024-02-01T17:30:27.903Z 22 440-459 61 61 61 85 85 85 F F F
68312024-02-01T17:30:27.912Z 23 460-479 59 59 59 82 82 82 T T F <---
68322024-02-01T17:30:27.912Z 24 480-499 62 62 58 86 86 80 F F F <---
68332024-02-01T17:30:27.912Z 25 500-519 62 62 53 86 86 74 F F F <---
68342024-02-01T17:30:27.912Z 26 520-539 62 62 61 86 86 85 F F F <---
68352024-02-01T17:30:27.912Z 27 540-559 62 62 58 87 87 79 F F F <---
68362024-02-01T17:30:27.912Z 28 560-579 61 61 61 85 85 85 F F F
68372024-02-01T17:30:27.912Z 29 580-599 60 60 60 83 83 83 F F F
68382024-02-01T17:30:27.912ZMax gen: 62, Max flush: 91
68392024-02-01T17:30:27.912ZError: Difference in extent metadata found!
68402024-02-01T17:30:27.912ZA Difference in extent metadata is expected here
68412024-02-01T17:30:27.912ZOn loop 31
68422024-02-01T17:30:27.912Z
68432024-02-01T17:30:27.913ZVerifying data now
68442024-02-01T17:30:31.571ZNo change after verify
68452024-02-01T17:30:31.571ZLoop: 31 Downstairs dump after verify (and repair):
68462024-02-01T17:30:31.594ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68472024-02-01T17:30:31.594Z 0 000-019 62 62 62 86 86 86 F F F
68482024-02-01T17:30:31.594Z 1 020-039 56 56 56 77 77 77 F F F
68492024-02-01T17:30:31.594Z 2 040-059 60 60 60 83 83 83 F F F
68502024-02-01T17:30:31.594Z 3 060-079 61 61 61 85 85 85 F F F
68512024-02-01T17:30:31.597Z 4 080-099 59 59 59 82 82 82 F F F
68522024-02-01T17:30:31.597Z 5 100-119 58 58 58 81 81 81 F F F
68532024-02-01T17:30:31.597Z 6 120-139 50 50 50 70 70 70 F F F
68542024-02-01T17:30:31.597Z 7 140-159 62 62 62 91 91 91 F F F
68552024-02-01T17:30:31.597Z 8 160-179 62 62 62 88 88 88 F F F
68562024-02-01T17:30:31.597Z 9 180-199 63 63 63 92 92 92 F F F
68572024-02-01T17:30:31.597Z 10 200-219 61 61 61 85 85 85 F F F
68582024-02-01T17:30:31.597Z 11 220-239 63 63 63 92 92 92 F F F
68592024-02-01T17:30:31.598Z 12 240-259 63 63 63 92 92 92 F F F
68602024-02-01T17:30:31.598Z 13 260-279 62 62 62 89 89 89 F F F
68612024-02-01T17:30:31.598Z 14 280-299 62 62 62 89 89 89 F F F
68622024-02-01T17:30:31.600Z 15 300-319 62 62 62 89 89 89 F F F
68632024-02-01T17:30:31.600Z 16 320-339 60 60 60 83 83 83 F F F
68642024-02-01T17:30:31.600Z 17 340-359 56 56 56 77 77 77 F F F
68652024-02-01T17:30:31.600Z 18 360-379 62 62 62 87 87 87 F F F
68662024-02-01T17:30:31.600Z 19 380-399 62 62 62 87 87 87 F F F
68672024-02-01T17:30:31.601Z 20 400-419 61 61 61 85 85 85 F F F
68682024-02-01T17:30:31.601Z 21 420-439 63 63 63 92 92 92 F F F
68692024-02-01T17:30:31.601Z 22 440-459 61 61 61 85 85 85 F F F
68702024-02-01T17:30:31.601Z 23 460-479 63 63 63 92 92 92 F F F
68712024-02-01T17:30:31.601Z 24 480-499 62 62 62 86 86 86 F F F
68722024-02-01T17:30:31.601Z 25 500-519 62 62 62 86 86 86 F F F
68732024-02-01T17:30:31.601Z 26 520-539 62 62 62 86 86 86 F F F
68742024-02-01T17:30:31.601Z 27 540-559 62 62 62 87 87 87 F F F
68752024-02-01T17:30:31.601Z 28 560-579 61 61 61 85 85 85 F F F
68762024-02-01T17:30:31.601Z 29 580-599 60 60 60 83 83 83 F F F
68772024-02-01T17:30:31.601ZMax gen: 63, Max flush: 92
68782024-02-01T17:30:31.605Z
68792024-02-01T17:30:31.605ZBegin loop 32
68802024-02-01T17:30:31.605ZDownstairs to restart: 2
68812024-02-01T17:30:31.629Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.896383635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
68822024-02-01T17:30:31.629Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.89685729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
68832024-02-01T17:30:31.630Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.896902318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
68842024-02-01T17:30:31.630Z{"msg":"Crucible f5d9b6de-9e7c-48c1-a5fd-b8f7590d9dfe has session id: 4fe20333-ac16-48fa-83c9-27bfab50efdc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.89693749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68852024-02-01T17:30:31.634Z{"msg":"Upstairs opts: Upstairs UUID: f5d9b6de-9e7c-48c1-a5fd-b8f7590d9dfe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
68862024-02-01T17:30:31.634Z,"time":"2024-02-01T17:30:30.896987178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68872024-02-01T17:30:31.634Z{"msg":"Crucible stats registered with UUID: f5d9b6de-9e7c-48c1-a5fd-b8f7590d9dfe","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897104093Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68882024-02-01T17:30:31.634Z{"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.8971705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
68892024-02-01T17:30:31.634Z{"msg":"f5d9b6de-9e7c-48c1-a5fd-b8f7590d9dfe active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897339616Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68902024-02-01T17:30:31.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897448514Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68912024-02-01T17:30:31.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897508286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68922024-02-01T17:30:31.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897596259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68932024-02-01T17:30:31.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897640323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68942024-02-01T17:30:31.636Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897787766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68952024-02-01T17:30:31.636Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897877519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68962024-02-01T17:30:31.636Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.897954231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68972024-02-01T17:30:31.636Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.898030502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68982024-02-01T17:30:31.636Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.898079737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
68992024-02-01T17:30:31.636Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.898161243Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69002024-02-01T17:30:31.637Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.898210166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69012024-02-01T17:30:31.637Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.89828845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"io task","client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69022024-02-01T17:30:31.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.899100998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69032024-02-01T17:30:31.637Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.899180626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69042024-02-01T17:30:31.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.899447621Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69052024-02-01T17:30:31.637Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.899636669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69062024-02-01T17:30:31.637Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.900259257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69072024-02-01T17:30:31.637Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.900455748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69082024-02-01T17:30:31.637Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.900631747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69092024-02-01T17:30:31.637Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.900668778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69102024-02-01T17:30:31.637Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.900964443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69112024-02-01T17:30:31.638Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901093455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69122024-02-01T17:30:31.643Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901352551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69132024-02-01T17:30:31.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901590372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69142024-02-01T17:30:31.643Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901654218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69152024-02-01T17:30:31.643Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901681504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69162024-02-01T17:30:31.643Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.901869373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69172024-02-01T17:30:31.643Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902627568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69182024-02-01T17:30:31.643Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902690091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69192024-02-01T17:30:31.643Z{"msg":"[0]R flush_numbers[0..12]: [86, 77, 83, 85, 82, 81, 70, 91, 88, 92, 85, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902760343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69202024-02-01T17:30:31.643Z{"msg":"[0]R generation[0..12]: [62, 56, 60, 61, 59, 58, 50, 62, 62, 63, 61, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902787872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69212024-02-01T17:30:31.643Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90281205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69222024-02-01T17:30:31.643Z{"msg":"[1]R flush_numbers[0..12]: [86, 77, 83, 85, 82, 81, 70, 91, 88, 92, 85, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902854423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69232024-02-01T17:30:31.643Z{"msg":"[1]R generation[0..12]: [62, 56, 60, 61, 59, 58, 50, 62, 62, 63, 61, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902879795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69242024-02-01T17:30:31.643Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902903754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69252024-02-01T17:30:31.643Z{"msg":"[2]R flush_numbers[0..12]: [86, 77, 83, 85, 82, 81, 70, 91, 88, 92, 85, 92]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90295843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69262024-02-01T17:30:31.643ZWait for a query_work_queue command to finish before sending IO
69272024-02-01T17:30:31.643Z{"msg":"[2]R generation[0..12]: [62, 56, 60, 61, 59, 58, 50, 62, 62, 63, 61, 63]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.902991372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69282024-02-01T17:30:31.643Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903036632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69292024-02-01T17:30:31.645Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903060383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69302024-02-01T17:30:31.645Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90309782Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69312024-02-01T17:30:31.645Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903138566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69322024-02-01T17:30:31.645Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90316815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"":"downstairs","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69332024-02-01T17:30:31.645Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903247532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69342024-02-01T17:30:31.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903272349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"0","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69352024-02-01T17:30:31.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90333015Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"1","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69362024-02-01T17:30:31.645Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.90335708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"client":"2","":"downstairs","session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69372024-02-01T17:30:31.645Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903418586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69382024-02-01T17:30:31.646Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903444566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69392024-02-01T17:30:31.647Z{"msg":"f5d9b6de-9e7c-48c1-a5fd-b8f7590d9dfe is now active with session: 4fe20333-ac16-48fa-83c9-27bfab50efdc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903482132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69402024-02-01T17:30:31.647Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903513171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385,"session_id":"4fe20333-ac16-48fa-83c9-27bfab50efdc"}
69412024-02-01T17:30:31.647Z{"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:30.903572473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
69422024-02-01T17:30:31.647ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69432024-02-01T17:30:31.647ZLoading write count information from file "/tmp/test_repair_verify.data"
69442024-02-01T17:30:31.647ZRun Repair workload
69452024-02-01T17:30:31.647Z01/30 Write block 226 len 28672 data: 3 3 3 3 3 3 3
69462024-02-01T17:30:31.647Z02/30 Write block 124 len 28672 data: 5 5 4 3 4 4 4
69472024-02-01T17:30:31.647Z03/30 Flush
69482024-02-01T17:30:31.651Z04/30 Write block 439 len 40960 data: 7 7 8 8 7 8 9 7 7 7
69492024-02-01T17:30:31.651Z05/30 Read block 423 len 4096
69502024-02-01T17:30:31.651Z06/30 Write block 532 len 16384 data: 7 7 9 7
69512024-02-01T17:30:31.651Z07/30 Write block 582 len 4096 data: 5
69522024-02-01T17:30:31.651Z08/30 Write block 71 len 8192 data: 8 9
69532024-02-01T17:30:31.651Z09/30 Read block 154 len 12288
69542024-02-01T17:30:31.654Z10/30 Write block 37 len 40960 data: 7 7 6 3 5 5 5 5 5 5
69552024-02-01T17:30:31.655Z11/30 Write block 509 len 16384 data: 4 3 4 4
69562024-02-01T17:30:31.656Z12/30 Read block 333 len 16384
69572024-02-01T17:30:31.660Z13/30 Write block 133 len 16384 data: 4 4 5 5
69582024-02-01T17:30:31.660Z14/30 Read block 69 len 12288
69592024-02-01T17:30:31.660Z15/30 Write block 287 len 28672 data: 6 6 6 4 5 5 6
69602024-02-01T17:30:31.660Z16/30 Read block 195 len 8192
69612024-02-01T17:30:31.663Z17/30 Read block 197 len 16384
69622024-02-01T17:30:31.663Z18/30 Read block 555 len 24576
69632024-02-01T17:30:31.666Z19/30 Write block 495 len 28672 data: 6 6 4 5 4 4 4
69642024-02-01T17:30:31.666Z20/30 Write block 518 len 16384 data: 3 3 4 4
69652024-02-01T17:30:31.666Z21/30 Write block 218 len 20480 data: 5 5 4 3 3
69662024-02-01T17:30:31.666Z22/30 Read block 238 len 40960
69672024-02-01T17:30:31.670Z23/30 Read block 160 len 16384
69682024-02-01T17:30:31.670Z24/30 Read block 503 len 20480
69692024-02-01T17:30:31.673Z25/30 Read block 294 len 40960
69702024-02-01T17:30:31.678Z26/30 Read block 468 len 36864
69712024-02-01T17:30:31.678Z27/30 Write block 210 len 8192 data: 7 7
69722024-02-01T17:30:31.678Z28/30 Read block 197 len 16384
69732024-02-01T17:30:31.678Z29/30 Write block 43 len 28672 data: 6 6 6 6 4 7 6
69742024-02-01T17:30:31.686Z30/30 Read block 205 len 4096
69752024-02-01T17:30:31.686Z----------------------------------------------------------------
69762024-02-01T17:30:31.686Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
69772024-02-01T17:30:31.686ZStates: Active Active Active
69782024-02-01T17:30:31.686ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69792024-02-01T17:30:31.686Z 1 Acked 1000 Write 7 Done Done Sent false
69802024-02-01T17:30:31.687Z 2 Acked 1001 Write 7 Done Done Sent false
69812024-02-01T17:30:31.687Z 3 Acked 1002 Flush 0 Done Done Sent false
69822024-02-01T17:30:31.687Z 4 Acked 1003 Write 10 Done Done Sent false
69832024-02-01T17:30:31.687Z 5 Acked 1004 Read 1 Done Done Sent false
69842024-02-01T17:30:31.688Z 6 Acked 1005 Write 4 Done Done Sent false
69852024-02-01T17:30:31.688Z 7 Acked 1006 Write 1 Done Done Sent false
69862024-02-01T17:30:31.688Z 8 Acked 1007 Write 2 Done Done Sent false
69872024-02-01T17:30:31.688Z 9 Acked 1008 Read 3 Done Done Sent false
69882024-02-01T17:30:31.688Z 10 Acked 1009 Write 10 Done Done Sent false
69892024-02-01T17:30:31.688Z 11 Acked 1010 Write 4 Done Done Sent false
69902024-02-01T17:30:31.688Z 12 Acked 1011 Read 4 Done Done Sent false
69912024-02-01T17:30:31.688Z 13 Acked 1012 Write 4 Done Done Sent false
69922024-02-01T17:30:31.688Z 14 Acked 1013 Read 3 Done Done Sent false
69932024-02-01T17:30:31.688Z 15 Acked 1014 Write 7 Done Done Sent false
69942024-02-01T17:30:31.688Z 16 Acked 1015 Read 2 Done Done Sent false
69952024-02-01T17:30:31.688Z 17 Acked 1016 Read 4 Done Done Sent false
69962024-02-01T17:30:31.688Z 18 Acked 1017 Read 6 Done Done Sent false
69972024-02-01T17:30:31.688Z 19 Acked 1018 Write 7 Done Done Sent false
69982024-02-01T17:30:31.688Z 20 Acked 1019 Write 4 Done Done Sent false
69992024-02-01T17:30:31.688Z 21 Acked 1020 Write 5 Done Done Sent false
70002024-02-01T17:30:31.688Z 22 Acked 1021 Read 10 Done Done Sent false
70012024-02-01T17:30:31.688Z 23 Acked 1022 Read 4 Done Done Sent false
70022024-02-01T17:30:31.688Z 24 Acked 1023 Read 5 Done Done Sent false
70032024-02-01T17:30:31.689Z 25 Acked 1024 Read 10 Done Done Sent false
70042024-02-01T17:30:31.689Z 26 Acked 1025 Read 9 Done Done Sent false
70052024-02-01T17:30:31.689Z 27 Acked 1026 Write 2 Done Done Sent false
70062024-02-01T17:30:31.689Z 28 Acked 1027 Read 4 Done Done Sent false
70072024-02-01T17:30:31.689Z 29 Acked 1028 Write 7 Done Done Sent false
70082024-02-01T17:30:31.689Z 30 Acked 1029 Read 1 Done Done Sent false
70092024-02-01T17:30:31.689Z STATES DS:0 DS:1 DS:2 TOTAL
70102024-02-01T17:30:31.689Z New 0 0 0 0
70112024-02-01T17:30:31.689Z Sent 0 0 30 30
70122024-02-01T17:30:31.689Z Done 30 30 0 60
70132024-02-01T17:30:31.689Z Skipped 0 0 0 0
70142024-02-01T17:30:31.690Z Error 0 0 0 0
70152024-02-01T17:30:31.690ZLast Flush: 1002 1002 0
70162024-02-01T17:30:31.690ZDownstairs last five completed:
70172024-02-01T17:30:31.690ZUpstairs last five completed: 30 29 28 27 26
70182024-02-01T17:30:31.690Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:30.94820032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1385}
70192024-02-01T17:30:31.690ZWrote out file "/tmp/test_repair_verify.data"
70202024-02-01T17:30:31.695Z
70212024-02-01T17:30:31.695ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70222024-02-01T17:30:31.721ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70232024-02-01T17:30:31.721Z 0 000-019 62 62 62 86 86 86 F F F
70242024-02-01T17:30:31.728Z 1 020-039 56 56 56 77 77 77 T T F <---
70252024-02-01T17:30:31.728Z 2 040-059 60 60 60 83 83 83 T T F <---
70262024-02-01T17:30:31.728Z 3 060-079 61 61 61 85 85 85 T T F <---
70272024-02-01T17:30:31.728Z 4 080-099 59 59 59 82 82 82 F F F
70282024-02-01T17:30:31.728Z 5 100-119 58 58 58 81 81 81 F F F
70292024-02-01T17:30:31.728Z 6 120-139 64 64 50 93 93 70 T T F <---
70302024-02-01T17:30:31.728Z 7 140-159 62 62 62 91 91 91 F F F
70312024-02-01T17:30:31.728Z 8 160-179 62 62 62 88 88 88 F F F
70322024-02-01T17:30:31.728Z 9 180-199 63 63 63 92 92 92 F F F
70332024-02-01T17:30:31.728Z 10 200-219 61 61 61 85 85 85 T T F <---
70342024-02-01T17:30:31.728Z 11 220-239 64 64 63 93 93 92 T T F <---
70352024-02-01T17:30:31.729Z 12 240-259 63 63 63 92 92 92 F F F
70362024-02-01T17:30:31.729Z 13 260-279 62 62 62 89 89 89 F F F
70372024-02-01T17:30:31.729Z 14 280-299 62 62 62 89 89 89 T T F <---
70382024-02-01T17:30:31.729Z 15 300-319 62 62 62 89 89 89 F F F
70392024-02-01T17:30:31.738Z 16 320-339 60 60 60 83 83 83 F F F
70402024-02-01T17:30:31.738Z 17 340-359 56 56 56 77 77 77 F F F
70412024-02-01T17:30:31.738Z 18 360-379 62 62 62 87 87 87 F F F
70422024-02-01T17:30:31.739Z 19 380-399 62 62 62 87 87 87 F F F
70432024-02-01T17:30:31.739Z 20 400-419 61 61 61 85 85 85 F F F
70442024-02-01T17:30:31.739Z 21 420-439 63 63 63 92 92 92 T T F <---
70452024-02-01T17:30:31.739Z 22 440-459 61 61 61 85 85 85 T T F <---
70462024-02-01T17:30:31.739Z 23 460-479 63 63 63 92 92 92 F F F
70472024-02-01T17:30:31.739Z 24 480-499 62 62 62 86 86 86 T T F <---
70482024-02-01T17:30:31.739Z 25 500-519 62 62 62 86 86 86 T T F <---
70492024-02-01T17:30:31.739Z 26 520-539 62 62 62 86 86 86 T T F <---
70502024-02-01T17:30:31.739Z 27 540-559 62 62 62 87 87 87 F F F
70512024-02-01T17:30:31.739Z 28 560-579 61 61 61 85 85 85 F F F
70522024-02-01T17:30:31.739Z 29 580-599 60 60 60 83 83 83 T T F <---
70532024-02-01T17:30:31.739ZMax gen: 64, Max flush: 93
70542024-02-01T17:30:31.739ZError: Difference in extent metadata found!
70552024-02-01T17:30:31.743ZA Difference in extent metadata is expected here
70562024-02-01T17:30:31.743ZOn loop 32
70572024-02-01T17:30:31.743Z
70582024-02-01T17:30:31.743ZVerifying data now
70592024-02-01T17:30:36.871ZNo change after verify
70602024-02-01T17:30:36.871ZLoop: 32 Downstairs dump after verify (and repair):
70612024-02-01T17:30:36.894ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70622024-02-01T17:30:36.894Z 0 000-019 62 62 62 86 86 86 F F F
70632024-02-01T17:30:36.895Z 1 020-039 65 65 65 94 94 94 F F F
70642024-02-01T17:30:36.895Z 2 040-059 65 65 65 94 94 94 F F F
70652024-02-01T17:30:36.895Z 3 060-079 65 65 65 94 94 94 F F F
70662024-02-01T17:30:36.913Z 4 080-099 59 59 59 82 82 82 F F F
70672024-02-01T17:30:36.913Z 5 100-119 58 58 58 81 81 81 F F F
70682024-02-01T17:30:36.913Z 6 120-139 65 65 65 94 94 94 F F F
70692024-02-01T17:30:36.913Z 7 140-159 62 62 62 91 91 91 F F F
70702024-02-01T17:30:36.913Z 8 160-179 62 62 62 88 88 88 F F F
70712024-02-01T17:30:36.914Z 9 180-199 63 63 63 92 92 92 F F F
70722024-02-01T17:30:36.914Z 10 200-219 65 65 65 94 94 94 F F F
70732024-02-01T17:30:36.914Z 11 220-239 65 65 65 94 94 94 F F F
70742024-02-01T17:30:36.914Z 12 240-259 63 63 63 92 92 92 F F F
70752024-02-01T17:30:36.914Z 13 260-279 62 62 62 89 89 89 F F F
70762024-02-01T17:30:36.914Z 14 280-299 65 65 65 94 94 94 F F F
70772024-02-01T17:30:36.914Z 15 300-319 62 62 62 89 89 89 F F F
70782024-02-01T17:30:36.914Z 16 320-339 60 60 60 83 83 83 F F F
70792024-02-01T17:30:36.914Z 17 340-359 56 56 56 77 77 77 F F F
70802024-02-01T17:30:36.914Z 18 360-379 62 62 62 87 87 87 F F F
70812024-02-01T17:30:36.914Z 19 380-399 62 62 62 87 87 87 F F F
70822024-02-01T17:30:36.914Z 20 400-419 61 61 61 85 85 85 F F F
70832024-02-01T17:30:36.915Z 21 420-439 65 65 65 94 94 94 F F F
70842024-02-01T17:30:36.915Z 22 440-459 65 65 65 94 94 94 F F F
70852024-02-01T17:30:36.915Z 23 460-479 63 63 63 92 92 92 F F F
70862024-02-01T17:30:36.915Z 24 480-499 65 65 65 94 94 94 F F F
70872024-02-01T17:30:36.915Z 25 500-519 65 65 65 94 94 94 F F F
70882024-02-01T17:30:36.915Z 26 520-539 65 65 65 94 94 94 F F F
70892024-02-01T17:30:36.915Z 27 540-559 62 62 62 87 87 87 F F F
70902024-02-01T17:30:36.915Z 28 560-579 61 61 61 85 85 85 F F F
70912024-02-01T17:30:36.915Z 29 580-599 65 65 65 94 94 94 F F F
70922024-02-01T17:30:36.915ZMax gen: 65, Max flush: 94
70932024-02-01T17:30:36.915Z
70942024-02-01T17:30:36.915ZBegin loop 33
70952024-02-01T17:30:36.915ZDownstairs to restart: 2
70962024-02-01T17:30:36.932Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.19756971Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
70972024-02-01T17:30:36.947Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198122929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
70982024-02-01T17:30:36.947Z{"msg":Crucible runtime is spawned
70992024-02-01T17:30:36.947Z"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198295351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
71002024-02-01T17:30:36.947Z{"msg":"Crucible 4f27ee56-5fc1-49cb-8f1e-388070357007 has session id: b695887e-af44-495d-aecc-f11a3bb4e2e1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198351086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71012024-02-01T17:30:36.947Z{"msg":"Upstairs opts: Upstairs UUID: 4f27ee56-5fc1-49cb-8f1e-388070357007, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.19838536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71022024-02-01T17:30:36.947Z{"msg":"Crucible stats registered with UUID: 4f27ee56-5fc1-49cb-8f1e-388070357007","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.19845813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71032024-02-01T17:30:36.947Z{"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.1984865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
71042024-02-01T17:30:36.947Z{"msg":"4f27ee56-5fc1-49cb-8f1e-388070357007 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198586531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71052024-02-01T17:30:36.947Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198642769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71062024-02-01T17:30:36.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198677066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71072024-02-01T17:30:36.948Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198717841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71082024-02-01T17:30:36.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198761122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71092024-02-01T17:30:36.948Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198801991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71102024-02-01T17:30:36.948Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198879753Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71112024-02-01T17:30:36.948Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198924826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71122024-02-01T17:30:36.948Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.198966591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71132024-02-01T17:30:36.948Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.199040215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71142024-02-01T17:30:36.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.199089937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71152024-02-01T17:30:36.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.199130152Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71162024-02-01T17:30:36.948Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.19929126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"io task","client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71172024-02-01T17:30:36.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.200323218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71182024-02-01T17:30:36.949Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.200405217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71192024-02-01T17:30:36.949Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.200450181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71202024-02-01T17:30:36.949Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.200527199Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71212024-02-01T17:30:36.949Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.201421402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71222024-02-01T17:30:36.950Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.201506612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71232024-02-01T17:30:36.950Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.201591904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71242024-02-01T17:30:36.950Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.201670155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71252024-02-01T17:30:36.950Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202218263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71262024-02-01T17:30:36.950Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202314455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71272024-02-01T17:30:36.951Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202388289Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71282024-02-01T17:30:36.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202416605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71292024-02-01T17:30:36.951Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202439337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71302024-02-01T17:30:36.951Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202541101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71312024-02-01T17:30:36.951Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.202815065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71322024-02-01T17:30:36.951Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20352739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71332024-02-01T17:30:36.952Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203590437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71342024-02-01T17:30:36.952Z{"msg":"[0]R flush_numbers[0..12]: [86, 94, 94, 94, 82, 81, 94, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20361823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71352024-02-01T17:30:36.952Z{"msg":"[0]R generation[0..12]: [62, 65, 65, 65, 59, 58, 65, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203641813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71362024-02-01T17:30:36.952Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20379818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71372024-02-01T17:30:36.953Z{"msg":"[1]R flush_numbers[0..12]: [86, 94, 94, 94, 82, 81, 94, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20385063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71382024-02-01T17:30:36.953Z{"msg":"[1]R generation[0..12]: [62, 65, 65, 65, 59, 58, 65, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203877115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71392024-02-01T17:30:36.953Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203919814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71402024-02-01T17:30:36.953Z{"msg":"[2]R flush_numbers[0..12]: [86, 94, 94, 94, 82, 81, 94, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203944797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71412024-02-01T17:30:36.953Z{"msg":"[2]R generation[0..12]: [62, 65, 65, 65, 59, 58, 65, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203969237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71422024-02-01T17:30:36.953Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.203992865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pidWait for a query_work_queue command to finish before sending IO
71432024-02-01T17:30:36.953Z":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71442024-02-01T17:30:36.953Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204099237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71452024-02-01T17:30:36.953Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20412816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71462024-02-01T17:30:36.953Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204174621Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71472024-02-01T17:30:36.954Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204202145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"":"downstairs","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71482024-02-01T17:30:36.954Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204253081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71492024-02-01T17:30:36.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204275503Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"0","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71502024-02-01T17:30:36.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204297749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"1","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71512024-02-01T17:30:36.954Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204395008Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"client":"2","":"downstairs","session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71522024-02-01T17:30:36.954Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.20444515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71532024-02-01T17:30:36.955Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204470694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71542024-02-01T17:30:36.955Z{"msg":"4f27ee56-5fc1-49cb-8f1e-388070357007 is now active with session: b695887e-af44-495d-aecc-f11a3bb4e2e1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204519349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71552024-02-01T17:30:36.955Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204542696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393,"session_id":"b695887e-af44-495d-aecc-f11a3bb4e2e1"}
71562024-02-01T17:30:36.955Z{"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:36.204590601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
71572024-02-01T17:30:36.955ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71582024-02-01T17:30:36.955ZLoading write count information from file "/tmp/test_repair_verify.data"
71592024-02-01T17:30:36.955ZRun Repair workload
71602024-02-01T17:30:36.955Z01/30 Write block 136 len 12288 data: 6 7 6
71612024-02-01T17:30:36.963Z02/30 Read block 429 len 28672
71622024-02-01T17:30:36.963Z03/30 Read block 283 len 12288
71632024-02-01T17:30:36.963Z04/30 Read block 99 len 8192
71642024-02-01T17:30:36.963Z05/30 Read block 200 len 32768
71652024-02-01T17:30:36.963Z06/30 Write block 457 len 28672 data: 5 5 7 6 5 5 4
71662024-02-01T17:30:36.963Z07/30 Write block 408 len 36864 data: 6 5 3 3 4 4 6 6 8
71672024-02-01T17:30:36.963Z08/30 Read block 426 len 12288
71682024-02-01T17:30:36.963Z09/30 Read block 289 len 28672
71692024-02-01T17:30:36.963Z10/30 Read block 588 len 4096
71702024-02-01T17:30:36.963Z11/30 Read block 223 len 8192
71712024-02-01T17:30:36.963Z12/30 Read block 349 len 16384
71722024-02-01T17:30:36.963Z13/30 Read block 269 len 24576
71732024-02-01T17:30:36.964Z14/30 Read block 479 len 32768
71742024-02-01T17:30:36.964Z15/30 Read block 361 len 36864
71752024-02-01T17:30:36.964Z16/30 Read block 379 len 8192
71762024-02-01T17:30:36.964Z17/30 Read block 226 len 4096
71772024-02-01T17:30:36.964Z18/30 Read block 589 len 16384
71782024-02-01T17:30:36.976Z19/30 Read block 92 len 12288
71792024-02-01T17:30:36.976Z20/30 Read block 465 len 24576
71802024-02-01T17:30:36.976Z21/30 Read block 499 len 16384
71812024-02-01T17:30:36.976Z22/30 Read block 413 len 20480
71822024-02-01T17:30:36.976Z23/30 Write block 588 len 36864 data: 8 7 7 6 4 4 4 4 3
71832024-02-01T17:30:36.976Z24/30 Write block 38 len 8192 data: 8 7
71842024-02-01T17:30:36.977Z25/30 Write block 531 len 20480 data: 6 8 8 10 8
71852024-02-01T17:30:36.977Z26/30 Read block 313 len 24576
71862024-02-01T17:30:36.977Z27/30 Write block 21 len 8192 data: 6 8
71872024-02-01T17:30:36.977Z28/30 Read block 497 len 20480
71882024-02-01T17:30:37.004Z29/30 Read block 386 len 28672
71892024-02-01T17:30:37.004Z30/30 Read block 504 len 8192
71902024-02-01T17:30:37.004Z----------------------------------------------------------------
71912024-02-01T17:30:37.004Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
71922024-02-01T17:30:37.004ZStates: Active Active Active
71932024-02-01T17:30:37.005ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
71942024-02-01T17:30:37.005Z 1 Acked 1000 Write 3 Done Done Done false
71952024-02-01T17:30:37.005Z 2 Acked 1001 Read 7 Done Done Sent false
71962024-02-01T17:30:37.005Z 3 Acked 1002 Read 3 Done Done Sent false
71972024-02-01T17:30:37.005Z 4 Acked 1003 Read 2 Done Done Sent false
71982024-02-01T17:30:37.005Z 5 Acked 1004 Read 8 Done Done Sent false
71992024-02-01T17:30:37.005Z 6 Acked 1005 Write 7 Done Done Sent false
72002024-02-01T17:30:37.005Z 7 Acked 1006 Write 9 Done Done Sent false
72012024-02-01T17:30:37.006Z 8 Acked 1007 Read 3 Done Done Sent false
72022024-02-01T17:30:37.006Z 9 Acked 1008 Read 7 Done Done Sent false
72032024-02-01T17:30:37.006Z 10 Acked 1009 Read 1 Done Done Sent false
72042024-02-01T17:30:37.006Z 11 Acked 1010 Read 2 Done Done Sent false
72052024-02-01T17:30:37.006Z 12 Acked 1011 Read 4 Done Done Sent false
72062024-02-01T17:30:37.006Z 13 Acked 1012 Read 6 Done Done Sent false
72072024-02-01T17:30:37.006Z 14 Acked 1013 Read 8 Done Done Sent false
72082024-02-01T17:30:37.007Z 15 Acked 1014 Read 9 Done Done Sent false
72092024-02-01T17:30:37.007Z 16 Acked 1015 Read 2 Done Done Sent false
72102024-02-01T17:30:37.007Z 17 Acked 1016 Read 1 Done Done Sent false
72112024-02-01T17:30:37.007Z 18 Acked 1017 Read 4 Done Done Sent false
72122024-02-01T17:30:37.007Z 19 Acked 1018 Read 3 Done Done Sent false
72132024-02-01T17:30:37.007Z 20 Acked 1019 Read 6 Done Done Sent false
72142024-02-01T17:30:37.007Z 21 Acked 1020 Read 4 Done Done Sent false
72152024-02-01T17:30:37.008Z 22 Acked 1021 Read 5 Done Done Sent false
72162024-02-01T17:30:37.008Z 23 Acked 1022 Write 9 Done Done Sent false
72172024-02-01T17:30:37.008Z 24 Acked 1023 Write 2 Done Done Sent false
72182024-02-01T17:30:37.008Z 25 Acked 1024 Write 5 Done Done Sent false
72192024-02-01T17:30:37.008Z 26 Acked 1025 Read 6 Done Done Sent false
72202024-02-01T17:30:37.008Z 27 Acked 1026 Write 2 Done Done Sent false
72212024-02-01T17:30:37.008Z 28 Acked 1027 Read 5 Done Done Sent false
72222024-02-01T17:30:37.008Z 29 Acked 1028 Read 7 Done Done Sent false
72232024-02-01T17:30:37.008Z 30 Acked 1029 Read 2 Done Done Sent false
72242024-02-01T17:30:37.009Z STATES DS:0 DS:1 DS:2 TOTAL
72252024-02-01T17:30:37.009Z New 0 0 0 0
72262024-02-01T17:30:37.009Z Sent 0 0 29 29
72272024-02-01T17:30:37.009Z Done 30 30 1 61
72282024-02-01T17:30:37.009Z Skipped 0 0 0 0
72292024-02-01T17:30:37.010Z Error 0 0 0 0
72302024-02-01T17:30:37.010ZLast Flush: 0 0 0
72312024-02-01T17:30:37.010ZDownstairs last five completed:
72322024-02-01T17:30:37.010ZUpstairs last five completed: 30 29 28 27 26
72332024-02-01T17:30:37.010Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:36.246412232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1393}
72342024-02-01T17:30:37.010ZWrote out file "/tmp/test_repair_verify.data"
72352024-02-01T17:30:37.025Z
72362024-02-01T17:30:37.025ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72372024-02-01T17:30:37.025ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72382024-02-01T17:30:37.025Z 0 000-019 62 62 62 86 86 86 F F F
72392024-02-01T17:30:37.025Z 1 020-039 65 65 65 94 94 94 T T F <---
72402024-02-01T17:30:37.026Z 2 040-059 65 65 65 94 94 94 F F F
72412024-02-01T17:30:37.026Z 3 060-079 65 65 65 94 94 94 F F F
72422024-02-01T17:30:37.026Z 4 080-099 59 59 59 82 82 82 F F F
72432024-02-01T17:30:37.026Z 5 100-119 58 58 58 81 81 81 F F F
72442024-02-01T17:30:37.026Z 6 120-139 65 65 65 94 94 94 T T T
72452024-02-01T17:30:37.026Z 7 140-159 62 62 62 91 91 91 F F F
72462024-02-01T17:30:37.026Z 8 160-179 62 62 62 88 88 88 F F F
72472024-02-01T17:30:37.026Z 9 180-199 63 63 63 92 92 92 F F F
72482024-02-01T17:30:37.026Z 10 200-219 65 65 65 94 94 94 F F F
72492024-02-01T17:30:37.026Z 11 220-239 65 65 65 94 94 94 F F F
72502024-02-01T17:30:37.026Z 12 240-259 63 63 63 92 92 92 F F F
72512024-02-01T17:30:37.026Z 13 260-279 62 62 62 89 89 89 F F F
72522024-02-01T17:30:37.026Z 14 280-299 65 65 65 94 94 94 F F F
72532024-02-01T17:30:37.026Z 15 300-319 62 62 62 89 89 89 F F F
72542024-02-01T17:30:37.026Z 16 320-339 60 60 60 83 83 83 F F F
72552024-02-01T17:30:37.026Z 17 340-359 56 56 56 77 77 77 F F F
72562024-02-01T17:30:37.027Z 18 360-379 62 62 62 87 87 87 F F F
72572024-02-01T17:30:37.027Z 19 380-399 62 62 62 87 87 87 F F F
72582024-02-01T17:30:37.027Z 20 400-419 61 61 61 85 85 85 T T F <---
72592024-02-01T17:30:37.027Z 21 420-439 65 65 65 94 94 94 F F F
72602024-02-01T17:30:37.027Z 22 440-459 65 65 65 94 94 94 T T F <---
72612024-02-01T17:30:37.027Z 23 460-479 63 63 63 92 92 92 T T F <---
72622024-02-01T17:30:37.027Z 24 480-499 65 65 65 94 94 94 F F F
72632024-02-01T17:30:37.027Z 25 500-519 65 65 65 94 94 94 F F F
72642024-02-01T17:30:37.027Z 26 520-539 65 65 65 94 94 94 T T F <---
72652024-02-01T17:30:37.027Z 27 540-559 62 62 62 87 87 87 F F F
72662024-02-01T17:30:37.027Z 28 560-579 61 61 61 85 85 85 F F F
72672024-02-01T17:30:37.027Z 29 580-599 65 65 65 94 94 94 T T F <---
72682024-02-01T17:30:37.027ZMax gen: 65, Max flush: 94
72692024-02-01T17:30:37.028ZError: Difference in extent metadata found!
72702024-02-01T17:30:37.055ZA Difference in extent metadata is expected here
72712024-02-01T17:30:37.055ZOn loop 33
72722024-02-01T17:30:37.055Z
72732024-02-01T17:30:37.055ZVerifying data now
72742024-02-01T17:30:41.720ZNo change after verify
72752024-02-01T17:30:41.720ZLoop: 33 Downstairs dump after verify (and repair):
72762024-02-01T17:30:41.731ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72772024-02-01T17:30:41.731Z 0 000-019 62 62 62 86 86 86 F F F
72782024-02-01T17:30:41.731Z 1 020-039 66 66 66 95 95 95 F F F
72792024-02-01T17:30:41.731Z 2 040-059 65 65 65 94 94 94 F F F
72802024-02-01T17:30:41.731Z 3 060-079 65 65 65 94 94 94 F F F
72812024-02-01T17:30:41.731Z 4 080-099 59 59 59 82 82 82 F F F
72822024-02-01T17:30:41.731Z 5 100-119 58 58 58 81 81 81 F F F
72832024-02-01T17:30:41.731Z 6 120-139 66 66 66 95 95 95 F F F
72842024-02-01T17:30:41.735Z 7 140-159 62 62 62 91 91 91 F F F
72852024-02-01T17:30:41.735Z 8 160-179 62 62 62 88 88 88 F F F
72862024-02-01T17:30:41.735Z 9 180-199 63 63 63 92 92 92 F F F
72872024-02-01T17:30:41.736Z 10 200-219 65 65 65 94 94 94 F F F
72882024-02-01T17:30:41.736Z 11 220-239 65 65 65 94 94 94 F F F
72892024-02-01T17:30:41.736Z 12 240-259 63 63 63 92 92 92 F F F
72902024-02-01T17:30:41.736Z 13 260-279 62 62 62 89 89 89 F F F
72912024-02-01T17:30:41.736Z 14 280-299 65 65 65 94 94 94 F F F
72922024-02-01T17:30:41.736Z 15 300-319 62 62 62 89 89 89 F F F
72932024-02-01T17:30:41.736Z 16 320-339 60 60 60 83 83 83 F F F
72942024-02-01T17:30:41.736Z 17 340-359 56 56 56 77 77 77 F F F
72952024-02-01T17:30:41.736Z 18 360-379 62 62 62 87 87 87 F F F
72962024-02-01T17:30:41.736Z 19 380-399 62 62 62 87 87 87 F F F
72972024-02-01T17:30:41.736Z 20 400-419 66 66 66 95 95 95 F F F
72982024-02-01T17:30:41.737Z 21 420-439 65 65 65 94 94 94 F F F
72992024-02-01T17:30:41.737Z 22 440-459 66 66 66 95 95 95 F F F
73002024-02-01T17:30:41.737Z 23 460-479 66 66 66 95 95 95 F F F
73012024-02-01T17:30:41.737Z 24 480-499 65 65 65 94 94 94 F F F
73022024-02-01T17:30:41.737Z 25 500-519 65 65 65 94 94 94 F F F
73032024-02-01T17:30:41.737Z 26 520-539 66 66 66 95 95 95 F F F
73042024-02-01T17:30:41.737Z 27 540-559 62 62 62 87 87 87 F F F
73052024-02-01T17:30:41.737Z 28 560-579 61 61 61 85 85 85 F F F
73062024-02-01T17:30:41.737Z 29 580-599 66 66 66 95 95 95 F F F
73072024-02-01T17:30:41.737ZMax gen: 66, Max flush: 95
73082024-02-01T17:30:41.741Z
73092024-02-01T17:30:41.741ZBegin loop 34
73102024-02-01T17:30:41.741ZDownstairs to restart: 0
73112024-02-01T17:30:41.767Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.034746555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
73122024-02-01T17:30:41.767Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.035328764Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
73132024-02-01T17:30:41.772Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.035399937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
73142024-02-01T17:30:41.772Z{"msg":"Crucible c71996c4-aaa2-4c8a-ac88-3f9e9ee1a9e0 has session id: 463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.035441504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73152024-02-01T17:30:41.773Z{"msg":"Upstairs opts: Upstairs UUID: c71996c4-aaa2-4c8a-ac88-3f9e9ee1a9e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.03561476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73162024-02-01T17:30:41.773Z{"msg":"Crucible stats registered with UUID: c71996c4-aaa2-4c8a-ac88-3f9e9ee1a9e0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.035670383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73172024-02-01T17:30:41.774ZCrucible runtime is spawned
73182024-02-01T17:30:41.775Z{"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.035862291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
73192024-02-01T17:30:41.775Z{"msg":"c71996c4-aaa2-4c8a-ac88-3f9e9ee1a9e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.03609853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73202024-02-01T17:30:41.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036162126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73212024-02-01T17:30:41.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036196485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73222024-02-01T17:30:41.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036221937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73232024-02-01T17:30:41.775Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036304729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73242024-02-01T17:30:41.775Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036376095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73252024-02-01T17:30:41.776Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.036715765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73262024-02-01T17:30:41.776Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.037034657Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73272024-02-01T17:30:41.776Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.037111877Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73282024-02-01T17:30:41.777Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.037185357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73292024-02-01T17:30:41.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.037232506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73302024-02-01T17:30:41.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.037330126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73312024-02-01T17:30:41.777Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.03737448Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"io task","client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73322024-02-01T17:30:41.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038194414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73332024-02-01T17:30:41.777Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038259058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73342024-02-01T17:30:41.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038285789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73352024-02-01T17:30:41.777Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038330816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73362024-02-01T17:30:41.777Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038355271Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73372024-02-01T17:30:41.777Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.038377414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73382024-02-01T17:30:41.777Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.039280742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73392024-02-01T17:30:41.777Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.039352032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73402024-02-01T17:30:41.783Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.039485025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73412024-02-01T17:30:41.784Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040303641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73422024-02-01T17:30:41.784Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040367018Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73432024-02-01T17:30:41.784Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040394536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73442024-02-01T17:30:41.784Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040559784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73452024-02-01T17:30:41.784Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040632361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73462024-02-01T17:30:41.784Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040661795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73472024-02-01T17:30:41.784Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040702984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73482024-02-01T17:30:41.784Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040727906Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73492024-02-01T17:30:41.784Z{"msg":"[0]R flush_numbers[0..12]: [86, 95, 94, 94, 82, 81, 95, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040769947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73502024-02-01T17:30:41.784Z{"msg":"[0]R generation[0..12]: [62, 66, 65, 65, 59, 58, 66, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040795248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73512024-02-01T17:30:41.785Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040845331Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73522024-02-01T17:30:41.785Z{"msg":"[1]R flush_numbers[0..12]: [86, 95, 94, 94, 82, 81, 95, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040889571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73532024-02-01T17:30:41.785Z{"msg":"[1]R generation[0..12]: [62, 66, 65, 65, 59, 58, 66, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.040914565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73542024-02-01T17:30:41.785Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.04094009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73552024-02-01T17:30:41.785Z{"msg":"[2]R flush_numbers[0..12]: [86, 95, 94, 94, 82, 81, 95, 91, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.04098575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73562024-02-01T17:30:41.785Z{"msg":"[2]R generation[0..12]: [62, 66, 65, 65, 59, 58, 66, 62, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041013012Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73572024-02-01T17:30:41.786Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
73582024-02-01T17:30:41.786Zv":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041078132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73592024-02-01T17:30:41.786Z{"msg":"Max found gen is 67","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041129115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73602024-02-01T17:30:41.786Z{"msg":"Generation requested: 68 >= found:67","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041153741Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73612024-02-01T17:30:41.787Z{"msg":"Next flush: 96","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041176747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73622024-02-01T17:30:41.787Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041202348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"":"downstairs","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73632024-02-01T17:30:41.787Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041243343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73642024-02-01T17:30:41.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.04126568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"0","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73652024-02-01T17:30:41.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041306814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"1","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73662024-02-01T17:30:41.787Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041330786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"client":"2","":"downstairs","session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73672024-02-01T17:30:41.787Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041352897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73682024-02-01T17:30:41.788Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041392396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73692024-02-01T17:30:41.788Z{"msg":"c71996c4-aaa2-4c8a-ac88-3f9e9ee1a9e0 is now active with session: 463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041416238Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73702024-02-01T17:30:41.788Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041462092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401,"session_id":"463f5cf1-5a76-4fe7-a9a4-14f3c7fdf87f"}
73712024-02-01T17:30:41.788Z{"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:41.041484599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
73722024-02-01T17:30:41.789ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
73732024-02-01T17:30:41.789ZLoading write count information from file "/tmp/test_repair_verify.data"
73742024-02-01T17:30:41.789ZRun Repair workload
73752024-02-01T17:30:41.789Z01/30 Write block 362 len 24576 data: 9 8 7 6 6 6
73762024-02-01T17:30:41.789Z02/30 Write block 16 len 24576 data: 3 5 6 5 6 7
73772024-02-01T17:30:41.789Z03/30 Write block 92 len 36864 data: 5 4 4 4 5 5 5 5 5
73782024-02-01T17:30:41.789Z04/30 Read block 363 len 40960
73792024-02-01T17:30:41.796Z05/30 Write block 143 len 16384 data: 3 3 3 3
73802024-02-01T17:30:41.796Z06/30 Flush
73812024-02-01T17:30:41.796Z07/30 Write block 272 len 36864 data: 6 6 4 5 7 5 7 5 5
73822024-02-01T17:30:41.796Z08/30 Read block 413 len 12288
73832024-02-01T17:30:41.796Z09/30 Write block 52 len 4096 data: 7
73842024-02-01T17:30:41.796Z10/30 Read block 359 len 16384
73852024-02-01T17:30:41.796Z11/30 Read block 504 len 8192
73862024-02-01T17:30:41.796Z12/30 Read block 330 len 36864
73872024-02-01T17:30:41.838Z13/30 Write block 457 len 40960 data: 6 6 8 7 6 6 5 5 6 8
73882024-02-01T17:30:41.838Z14/30 Flush
73892024-02-01T17:30:41.839Z15/30 Write block 433 len 8192 data: 6 7
73902024-02-01T17:30:41.839Z16/30 Read block 5 len 8192
73912024-02-01T17:30:41.839Z17/30 Write block 535 len 40960 data: 9 7 8 6 5 8 11 11 9 11
73922024-02-01T17:30:41.839Z18/30 Read block 313 len 12288
73932024-02-01T17:30:41.839Z19/30 Read block 184 len 12288
73942024-02-01T17:30:41.839Z20/30 Flush
73952024-02-01T17:30:41.839Z21/30 Write block 147 len 24576 data: 3 6 6 5 5 9
73962024-02-01T17:30:41.839Z22/30 Write block 585 len 20480 data: 6 7 7 9 8
73972024-02-01T17:30:41.839Z23/30 Read block 479 len 28672
73982024-02-01T17:30:41.839Z24/30 Read block 307 len 12288
73992024-02-01T17:30:41.840Z25/30 Read block 192 len 20480
74002024-02-01T17:30:41.840Z26/30 Write block 10 len 32768 data: 5 3 3 3 3 3 4 6
74012024-02-01T17:30:41.840Z27/30 Write block 478 len 28672 data: 4 5 5 7 7 7 7
74022024-02-01T17:30:41.840Z28/30 Read block 578 len 36864
74032024-02-01T17:30:41.840Z29/30 Read block 371 len 12288
74042024-02-01T17:30:41.840Z30/30 Read block 269 len 40960
74052024-02-01T17:30:41.840Z----------------------------------------------------------------
74062024-02-01T17:30:41.840Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
74072024-02-01T17:30:41.840ZStates: Active Active Active
74082024-02-01T17:30:41.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74092024-02-01T17:30:41.840Z 1 Acked 1000 Write 6 Sent Done Done false
74102024-02-01T17:30:41.840Z 2 Acked 1001 Write 6 Sent Done Done false
74112024-02-01T17:30:41.841Z 3 Acked 1002 Write 9 Sent Done Done false
74122024-02-01T17:30:41.841Z 4 Acked 1003 Read 10 Sent Done Done false
74132024-02-01T17:30:41.841Z 5 Acked 1004 Write 4 Sent Done Done false
74142024-02-01T17:30:41.841Z 6 Acked 1005 Flush 0 Sent Done Done false
74152024-02-01T17:30:41.841Z 7 Acked 1006 Write 9 Sent Done Done false
74162024-02-01T17:30:41.841Z 8 Acked 1007 Read 3 Sent Done Done false
74172024-02-01T17:30:41.841Z 9 Acked 1008 Write 1 Sent Done Done false
74182024-02-01T17:30:41.841Z 10 Acked 1009 Read 4 Sent Done Done false
74192024-02-01T17:30:41.841Z 11 Acked 1010 Read 2 Sent Done Done false
74202024-02-01T17:30:41.841Z 12 Acked 1011 Read 9 Sent Done Done false
74212024-02-01T17:30:41.841Z 13 Acked 1012 Write 10 Sent Done Done false
74222024-02-01T17:30:41.841Z 14 Acked 1013 Flush 0 Sent Done Done false
74232024-02-01T17:30:41.842Z 15 Acked 1014 Write 2 Sent Done Done false
74242024-02-01T17:30:41.842Z 16 Acked 1015 Read 2 Sent Done Done false
74252024-02-01T17:30:41.842Z 17 Acked 1016 Write 10 Sent Done Done false
74262024-02-01T17:30:41.842Z 18 Acked 1017 Read 3 Sent Done Done false
74272024-02-01T17:30:41.842Z 19 Acked 1018 Read 3 Sent Done Done false
74282024-02-01T17:30:41.842Z 20 Acked 1019 Flush 0 Sent Done Done false
74292024-02-01T17:30:41.842Z 21 Acked 1020 Write 6 Sent Done Done false
74302024-02-01T17:30:41.842Z 22 Acked 1021 Write 5 Sent Done Done false
74312024-02-01T17:30:41.842Z 23 Acked 1022 Read 7 Sent Done Done false
74322024-02-01T17:30:41.842Z 24 Acked 1023 Read 3 Sent Done Done false
74332024-02-01T17:30:41.842Z 25 Acked 1024 Read 5 Sent Done Done false
74342024-02-01T17:30:41.842Z 26 Acked 1025 Write 8 Sent Done Done false
74352024-02-01T17:30:41.842Z 27 Acked 1026 Write 7 Sent Done Done false
74362024-02-01T17:30:41.843Z 28 Acked 1027 Read 9 Sent Done Done false
74372024-02-01T17:30:41.843Z 29 Acked 1028 Read 3 Sent Done Done false
74382024-02-01T17:30:41.843Z 30 Acked 1029 Read 10 Sent Sent Done false
74392024-02-01T17:30:41.843Z STATES DS:0 DS:1 DS:2 TOTAL
74402024-02-01T17:30:41.843Z New 0 0 0 0
74412024-02-01T17:30:41.843Z Sent 30 1 0 31
74422024-02-01T17:30:41.843Z Done 0 29 30 59
74432024-02-01T17:30:41.843Z Skipped 0 0 0 0
74442024-02-01T17:30:41.843Z Error 0 0 0 0
74452024-02-01T17:30:41.843ZLast Flush: 0 1019 1019
74462024-02-01T17:30:41.843ZDownstairs last five completed:
74472024-02-01T17:30:41.843ZUpstairs last five completed: 30 29 28 27 26
74482024-02-01T17:30:41.843Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:41.094631241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1401}
74492024-02-01T17:30:41.844ZWrote out file "/tmp/test_repair_verify.data"
74502024-02-01T17:30:41.844Z
74512024-02-01T17:30:41.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
74522024-02-01T17:30:41.865ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74532024-02-01T17:30:41.866Z 0 000-019 62 68 68 86 96 96 F T T <---
74542024-02-01T17:30:41.866Z 1 020-039 66 68 68 95 96 96 F F F <---
74552024-02-01T17:30:41.866Z 2 040-059 65 68 68 94 97 97 F F F <---
74562024-02-01T17:30:41.866Z 3 060-079 65 65 65 94 94 94 F F F
74572024-02-01T17:30:41.866Z 4 080-099 59 68 68 82 96 96 F F F <---
74582024-02-01T17:30:41.866Z 5 100-119 58 68 68 81 96 96 F F F <---
74592024-02-01T17:30:41.867Z 6 120-139 66 66 66 95 95 95 F F F
74602024-02-01T17:30:41.867Z 7 140-159 62 68 68 91 96 96 F T T <---
74612024-02-01T17:30:41.867Z 8 160-179 62 62 62 88 88 88 F F F
74622024-02-01T17:30:41.867Z 9 180-199 63 63 63 92 92 92 F F F
74632024-02-01T17:30:41.867Z 10 200-219 65 65 65 94 94 94 F F F
74642024-02-01T17:30:41.873Z 11 220-239 65 65 65 94 94 94 F F F
74652024-02-01T17:30:41.873Z 12 240-259 63 63 63 92 92 92 F F F
74662024-02-01T17:30:41.873Z 13 260-279 62 68 68 89 97 97 F F F <---
74672024-02-01T17:30:41.873Z 14 280-299 65 68 68 94 97 97 F F F <---
74682024-02-01T17:30:41.873Z 15 300-319 62 62 62 89 89 89 F F F
74692024-02-01T17:30:41.874Z 16 320-339 60 60 60 83 83 83 F F F
74702024-02-01T17:30:41.874Z 17 340-359 56 56 56 77 77 77 F F F
74712024-02-01T17:30:41.874Z 18 360-379 62 68 68 87 96 96 F F F <---
74722024-02-01T17:30:41.874Z 19 380-399 62 62 62 87 87 87 F F F
74732024-02-01T17:30:41.874Z 20 400-419 66 66 66 95 95 95 F F F
74742024-02-01T17:30:41.874Z 21 420-439 65 68 68 94 98 98 F F F <---
74752024-02-01T17:30:41.874Z 22 440-459 66 68 68 95 97 97 F F F <---
74762024-02-01T17:30:41.874Z 23 460-479 66 68 68 95 97 97 F T T <---
74772024-02-01T17:30:41.874Z 24 480-499 65 65 65 94 94 94 F T T <---
74782024-02-01T17:30:41.875Z 25 500-519 65 65 65 94 94 94 F F F
74792024-02-01T17:30:41.875Z 26 520-539 66 68 68 95 98 98 F F F <---
74802024-02-01T17:30:41.875Z 27 540-559 62 68 68 87 98 98 F F F <---
74812024-02-01T17:30:41.875Z 28 560-579 61 61 61 85 85 85 F F F
74822024-02-01T17:30:41.875Z 29 580-599 66 66 66 95 95 95 F T T <---
74832024-02-01T17:30:41.875ZMax gen: 68, Max flush: 98
74842024-02-01T17:30:41.876ZError: Difference in extent metadata found!
74852024-02-01T17:30:41.876ZA Difference in extent metadata is expected here
74862024-02-01T17:30:41.876ZOn loop 34
74872024-02-01T17:30:41.876Z
74882024-02-01T17:30:41.894ZVerifying data now
74892024-02-01T17:30:47.324ZNo change after verify
74902024-02-01T17:30:47.325ZLoop: 34 Downstairs dump after verify (and repair):
74912024-02-01T17:30:47.339ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
74922024-02-01T17:30:47.340Z 0 000-019 69 69 69 99 99 99 F F F
74932024-02-01T17:30:47.340Z 1 020-039 68 68 68 96 96 96 F F F
74942024-02-01T17:30:47.340Z 2 040-059 68 68 68 97 97 97 F F F
74952024-02-01T17:30:47.340Z 3 060-079 65 65 65 94 94 94 F F F
74962024-02-01T17:30:47.340Z 4 080-099 68 68 68 96 96 96 F F F
74972024-02-01T17:30:47.340Z 5 100-119 68 68 68 96 96 96 F F F
74982024-02-01T17:30:47.340Z 6 120-139 66 66 66 95 95 95 F F F
74992024-02-01T17:30:47.340Z 7 140-159 69 69 69 99 99 99 F F F
75002024-02-01T17:30:47.340Z 8 160-179 62 62 62 88 88 88 F F F
75012024-02-01T17:30:47.340Z 9 180-199 63 63 63 92 92 92 F F F
75022024-02-01T17:30:47.340Z 10 200-219 65 65 65 94 94 94 F F F
75032024-02-01T17:30:47.340Z 11 220-239 65 65 65 94 94 94 F F F
75042024-02-01T17:30:47.340Z 12 240-259 63 63 63 92 92 92 F F F
75052024-02-01T17:30:47.341Z 13 260-279 68 68 68 97 97 97 F F F
75062024-02-01T17:30:47.341Z 14 280-299 68 68 68 97 97 97 F F F
75072024-02-01T17:30:47.341Z 15 300-319 62 62 62 89 89 89 F F F
75082024-02-01T17:30:47.341Z 16 320-339 60 60 60 83 83 83 F F F
75092024-02-01T17:30:47.341Z 17 340-359 56 56 56 77 77 77 F F F
75102024-02-01T17:30:47.341Z 18 360-379 68 68 68 96 96 96 F F F
75112024-02-01T17:30:47.341Z 19 380-399 62 62 62 87 87 87 F F F
75122024-02-01T17:30:47.341Z 20 400-419 66 66 66 95 95 95 F F F
75132024-02-01T17:30:47.341Z 21 420-439 68 68 68 98 98 98 F F F
75142024-02-01T17:30:47.341Z 22 440-459 68 68 68 97 97 97 F F F
75152024-02-01T17:30:47.341Z 23 460-479 69 69 69 99 99 99 F F F
75162024-02-01T17:30:47.342Z 24 480-499 69 69 69 99 99 99 F F F
75172024-02-01T17:30:47.342Z 25 500-519 65 65 65 94 94 94 F F F
75182024-02-01T17:30:47.342Z 26 520-539 68 68 68 98 98 98 F F F
75192024-02-01T17:30:47.342Z 27 540-559 68 68 68 98 98 98 F F F
75202024-02-01T17:30:47.342Z 28 560-579 61 61 61 85 85 85 F F F
75212024-02-01T17:30:47.342Z 29 580-599 69 69 69 99 99 99 F F F
75222024-02-01T17:30:47.342ZMax gen: 69, Max flush: 99
75232024-02-01T17:30:47.342Z
75242024-02-01T17:30:47.342ZBegin loop 35
75252024-02-01T17:30:47.342ZDownstairs to restart: 2
75262024-02-01T17:30:47.342Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.535548007Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
75272024-02-01T17:30:47.343Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536001516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
75282024-02-01T17:30:47.343Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536047582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
75292024-02-01T17:30:47.343Z{"msg":"Crucible bd346ce8-49bb-4a94-a373-75a5a49ed147 has session id: f2a5159c-7657-4385-b7eb-a653fbd93c62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536085487Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75302024-02-01T17:30:47.343ZCrucible runtime is spawned
75312024-02-01T17:30:47.343Z{"msg":"Upstairs opts: Upstairs UUID: bd346ce8-49bb-4a94-a373-75a5a49ed147, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536158266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75322024-02-01T17:30:47.343Z{"msg":"Crucible stats registered with UUID: bd346ce8-49bb-4a94-a373-75a5a49ed147","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536218597Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75332024-02-01T17:30:47.343Z{"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536244222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
75342024-02-01T17:30:47.343Z{"msg":"bd346ce8-49bb-4a94-a373-75a5a49ed147 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536356916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75352024-02-01T17:30:47.343Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536414816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75362024-02-01T17:30:47.343Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536449016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75372024-02-01T17:30:47.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536521438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75382024-02-01T17:30:47.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.53661835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75392024-02-01T17:30:47.345Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536670136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75402024-02-01T17:30:47.345Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536713725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75412024-02-01T17:30:47.345Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536751914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75422024-02-01T17:30:47.345Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536794429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75432024-02-01T17:30:47.345Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.536904929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75442024-02-01T17:30:47.345Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.537062035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75452024-02-01T17:30:47.346Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.53713881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75462024-02-01T17:30:47.346Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.537186566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75472024-02-01T17:30:47.346Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.538192808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75482024-02-01T17:30:47.346Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.538353672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75492024-02-01T17:30:47.346Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.538403848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75502024-02-01T17:30:47.346Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.538446542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75512024-02-01T17:30:47.346Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.539059261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75522024-02-01T17:30:47.346Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.539124259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75532024-02-01T17:30:47.346Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.539153765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75542024-02-01T17:30:47.346Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.539616562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75552024-02-01T17:30:47.346Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540147899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75562024-02-01T17:30:47.347Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540251307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75572024-02-01T17:30:47.347Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540317154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75582024-02-01T17:30:47.347Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540383904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75592024-02-01T17:30:47.347Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540692731Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75602024-02-01T17:30:47.347Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540780952Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75612024-02-01T17:30:47.348Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.540848277Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75622024-02-01T17:30:47.348Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541237006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75632024-02-01T17:30:47.348Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541318541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75642024-02-01T17:30:47.348Z{"msg":"[0]R flush_numbers[0..12]: [99, 96, 97, 94, 96, 96, 95, 99, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541357356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75652024-02-01T17:30:47.348Z{"msg":"[0]R generation[0..12]: [69, 68, 68, 65, 68, 68, 66, 69, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541402327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75662024-02-01T17:30:47.348Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541429081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75672024-02-01T17:30:47.349Z{"msg":"[1]R flush_numbers[0..12]: [99, 96, 97, 94, 96, 96, 95, 99, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541453139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75682024-02-01T17:30:47.349Z{"msg":"[1]R generation[0..12]: [69, 68, 68, 65, 68, 68, 66, 69, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541540302Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
75692024-02-01T17:30:47.349Zpid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75702024-02-01T17:30:47.349Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541590686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75712024-02-01T17:30:47.349Z{"msg":"[2]R flush_numbers[0..12]: [99, 96, 97, 94, 96, 96, 95, 99, 88, 92, 94, 94]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541650007Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75722024-02-01T17:30:47.349Z{"msg":"[2]R generation[0..12]: [69, 68, 68, 65, 68, 68, 66, 69, 62, 63, 65, 65]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541695471Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75732024-02-01T17:30:47.349Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541755622Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75742024-02-01T17:30:47.349Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541798539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75752024-02-01T17:30:47.350Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541858225Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75762024-02-01T17:30:47.350Z{"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541900746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75772024-02-01T17:30:47.351Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.541967596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"downstairs","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75782024-02-01T17:30:47.351Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542011757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75792024-02-01T17:30:47.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542065388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75802024-02-01T17:30:47.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542108983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"1","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75812024-02-01T17:30:47.351Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.54216239Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"client":"2","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75822024-02-01T17:30:47.351Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542206051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75832024-02-01T17:30:47.351Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
75842024-02-01T17:30:47.351Z,"time":"2024-02-01T17:30:46.542268909Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75852024-02-01T17:30:47.351Z{"msg":"bd346ce8-49bb-4a94-a373-75a5a49ed147 is now active with session: f2a5159c-7657-4385-b7eb-a653fbd93c62","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542311768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75862024-02-01T17:30:47.351Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542364769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
75872024-02-01T17:30:47.352Z{"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:46.542404721Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
75882024-02-01T17:30:47.362ZLoading write count information from file "/tmp/test_repair_verify.data"
75892024-02-01T17:30:47.362ZRun Repair workload
75902024-02-01T17:30:47.362Z01/30 Write block 352 len 36864 data: 6 5 8 6 5 5 7 7 7
75912024-02-01T17:30:47.496Z02/30 Read block 139 len 40960
75922024-02-01T17:30:47.496Z03/30 Write block 161 len 28672 data: 9 8 6 6 4 3 3
75932024-02-01T17:30:47.496Z04/30 Write block 426 len 8192 data: 9 5
75942024-02-01T17:30:47.497Z05/30 Flush
75952024-02-01T17:30:47.497Z06/30 Write block 158 len 4096 data: 7
75962024-02-01T17:30:47.497Z07/30 Read block 344 len 32768
75972024-02-01T17:30:47.497Z08/30 Read block 137 len 32768
75982024-02-01T17:30:47.497Z09/30 Write block 220 len 8192 data: 5 4
75992024-02-01T17:30:47.497Z10/30 Read block 540 len 20480
76002024-02-01T17:30:47.497Z11/30 Read block 535 len 4096
76012024-02-01T17:30:47.497Z12/30 Flush
76022024-02-01T17:30:47.497Z13/30 Write block 495 len 20480 data: 7 7 5 6 5
76032024-02-01T17:30:47.497Z14/30 Read block 332 len 8192
76042024-02-01T17:30:47.497Z15/30 Read block 102 len 36864
76052024-02-01T17:30:47.497Z16/30 Read block 134 len 28672
76062024-02-01T17:30:47.497Z17/30 Write block 178 len 40960 data: 7 7 6 8 10 11 10 11 8 8
76072024-02-01T17:30:47.497Z18/30 Write block 41 len 12288 data: 6 6 7
76082024-02-01T17:30:47.497Z19/30 Read block 567 len 20480
76092024-02-01T17:30:47.497Z20/30 Flush
76102024-02-01T17:30:47.498Z21/30 Write block 274 len 20480 data: 5 6 8 6 8
76112024-02-01T17:30:47.498Z22/30 Flush
76122024-02-01T17:30:47.498Z23/30 Write block 449 len 8192 data: 8 6
76132024-02-01T17:30:47.498Z24/30 Write block 102 len 8192 data: 5 4
76142024-02-01T17:30:47.498Z25/30 Read block 223 len 20480
76152024-02-01T17:30:47.499Z26/30 Write block 34 len 40960 data: 4 5 6 8 9 8 4 7 7 8
76162024-02-01T17:30:47.499Z27/30 Write block 185 len 28672 data: 12 9 9 6 7 7 6
76172024-02-01T17:30:47.499Z28/30 Read block 353 len 8192
76182024-02-01T17:30:47.499Z29/30 Read block 449 len 40960
76192024-02-01T17:30:47.499Z30/30 Write block 487 len 28672 data: 7 8 8 8 8 9 8
76202024-02-01T17:30:47.499Z----------------------------------------------------------------
76212024-02-01T17:30:47.499Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
76222024-02-01T17:30:47.499ZStates: Active Active Active
76232024-02-01T17:30:47.499ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
76242024-02-01T17:30:47.499Z 1 Acked 1000 Write 9 Done Done Done false
76252024-02-01T17:30:47.499Z 2 Acked 1001 Read 10 Done Done Done false
76262024-02-01T17:30:47.499Z 3 Acked 1002 Write 7 Done Done Sent false
76272024-02-01T17:30:47.499Z 4 Acked 1003 Write 2 Done Done Sent false
76282024-02-01T17:30:47.499Z 5 Acked 1004 Flush 0 Done Done Sent false
76292024-02-01T17:30:47.499Z 6 Acked 1005 Write 1 Done Done Sent false
76302024-02-01T17:30:47.499Z 7 Acked 1006 Read 8 Done Done Sent false
76312024-02-01T17:30:47.499Z 8 Acked 1007 Read 8 Done Done Sent false
76322024-02-01T17:30:47.499Z 9 Acked 1008 Write 2 Done Done Sent false
76332024-02-01T17:30:47.499Z 10 Acked 1009 Read 5 Done Done Sent false
76342024-02-01T17:30:47.499Z 11 Acked 1010 Read 1 Done Done Sent false
76352024-02-01T17:30:47.499Z 12 Acked 1011 Flush 0 Done Done Sent false
76362024-02-01T17:30:47.499Z 13 Acked 1012 Write 5 Done Done Sent false
76372024-02-01T17:30:47.499Z 14 Acked 1013 Read 2 Done Done Sent false
76382024-02-01T17:30:47.500Z 15 Acked 1014 Read 9 Done Done Sent false
76392024-02-01T17:30:47.500Z 16 Acked 1015 Read 7 Done Done Sent false
76402024-02-01T17:30:47.500Z 17 Acked 1016 Write 10 Done Done Sent false
76412024-02-01T17:30:47.500Z 18 Acked 1017 Write 3 Done Done Sent false
76422024-02-01T17:30:47.500Z 19 Acked 1018 Read 5 Done Done Sent false
76432024-02-01T17:30:47.500Z 20 Acked 1019 Flush 0 Done Done Sent false
76442024-02-01T17:30:47.500Z 21 Acked 1020 Write 5 Done Done Sent false
76452024-02-01T17:30:47.500Z 22 Acked 1021 Flush 0 Done Done Sent false
76462024-02-01T17:30:47.500Z 23 Acked 1022 Write 2 Done Done Sent false
76472024-02-01T17:30:47.500Z 24 Acked 1023 Write 2 Done Done Sent false
76482024-02-01T17:30:47.500Z 25 Acked 1024 Read 5 Done Done Sent false
76492024-02-01T17:30:47.500Z 26 Acked 1025 Write 10 Done Done Sent false
76502024-02-01T17:30:47.500Z 27 Acked 1026 Write 7 Done Done Sent false
76512024-02-01T17:30:47.500Z 28 Acked 1027 Read 2 Done Done Sent false
76522024-02-01T17:30:47.500Z 29 Acked 1028 Read 10 Done Done Sent false
76532024-02-01T17:30:47.500Z 30 Acked 1029 Write 7 Sent Sent Sent false
76542024-02-01T17:30:47.500Z STATES DS:0 DS:1 DS:2 TOTAL
76552024-02-01T17:30:47.500Z New 0 0 0 0
76562024-02-01T17:30:47.500Z Sent 1 1 28 30
76572024-02-01T17:30:47.500Z Done 29 29 2 60
76582024-02-01T17:30:47.500Z Skipped 0 0 0 0
76592024-02-01T17:30:47.500Z Error 0 0 0 0
76602024-02-01T17:30:47.500ZLast Flush: 1021 1021 0
76612024-02-01T17:30:47.501ZDownstairs last five completed:
76622024-02-01T17:30:47.501ZUpstairs last five completed: 30 29 28 27 26
76632024-02-01T17:30:47.501Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:46.590484405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409}
76642024-02-01T17:30:47.501ZWrote out file "/tmp/test_repair_verify.data"
76652024-02-01T17:30:47.501Z{"msg":"client response queue closed unexpectedly: channel closed; is the program exiting?","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:46.591262728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1409,"":"io task","client":"0","":"downstairs","session_id":"f2a5159c-7657-4385-b7eb-a653fbd93c62"}
76662024-02-01T17:30:47.501Z
76672024-02-01T17:30:47.501ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
76682024-02-01T17:30:47.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
76692024-02-01T17:30:47.502Z 0 000-019 69 69 69 99 99 99 F F F
76702024-02-01T17:30:47.502Z 1 020-039 68 68 68 96 96 96 T T F <---
76712024-02-01T17:30:47.502Z 2 040-059 70 70 68 102 102 97 T T F <---
76722024-02-01T17:30:47.502Z 3 060-079 65 65 65 94 94 94 F F F
76732024-02-01T17:30:47.502Z 4 080-099 68 68 68 96 96 96 F F F
76742024-02-01T17:30:47.502Z 5 100-119 68 68 68 96 96 96 T T F <---
76752024-02-01T17:30:47.502Z 6 120-139 66 66 66 95 95 95 F F F
76762024-02-01T17:30:47.502Z 7 140-159 70 70 69 101 101 99 F F F <---
76772024-02-01T17:30:47.502Z 8 160-179 70 70 62 102 102 88 F F F <---
76782024-02-01T17:30:47.502Z 9 180-199 70 70 63 102 102 92 T T F <---
76792024-02-01T17:30:47.502Z 10 200-219 65 65 65 94 94 94 F F F
76802024-02-01T17:30:47.502Z 11 220-239 70 70 65 101 101 94 F F F <---
76812024-02-01T17:30:47.502Z 12 240-259 63 63 63 92 92 92 F F F
76822024-02-01T17:30:47.502Z 13 260-279 70 70 68 103 103 97 F F F <---
76832024-02-01T17:30:47.503Z 14 280-299 68 68 68 97 97 97 F F F
76842024-02-01T17:30:47.503Z 15 300-319 62 62 62 89 89 89 F F F
76852024-02-01T17:30:47.503Z 16 320-339 60 60 60 83 83 83 F F F
76862024-02-01T17:30:47.503Z 17 340-359 70 70 56 100 100 77 F F T <---
76872024-02-01T17:30:47.503Z 18 360-379 70 70 68 100 100 96 F F T <---
76882024-02-01T17:30:47.503Z 19 380-399 62 62 62 87 87 87 F F F
76892024-02-01T17:30:47.503Z 20 400-419 66 66 66 95 95 95 F F F
76902024-02-01T17:30:47.503Z 21 420-439 70 70 68 100 100 98 F F F <---
76912024-02-01T17:30:47.515Z 22 440-459 68 68 68 97 97 97 T T F <---
76922024-02-01T17:30:47.515Z 23 460-479 69 69 69 99 99 99 F F F
76932024-02-01T17:30:47.515Z 24 480-499 70 70 69 102 102 99 T T F <---
76942024-02-01T17:30:47.515Z 25 500-519 65 65 65 94 94 94 F F F
76952024-02-01T17:30:47.515Z 26 520-539 68 68 68 98 98 98 F F F
76962024-02-01T17:30:47.515Z 27 540-559 68 68 68 98 98 98 F F F
76972024-02-01T17:30:47.515Z 28 560-579 61 61 61 85 85 85 F F F
76982024-02-01T17:30:47.515Z 29 580-599 69 69 69 99 99 99 F F F
76992024-02-01T17:30:47.515ZMax gen: 70, Max flush: 103
77002024-02-01T17:30:47.516ZError: Difference in extent metadata found!
77012024-02-01T17:30:47.516ZA Difference in extent metadata is expected here
77022024-02-01T17:30:47.516ZOn loop 35
77032024-02-01T17:30:47.516Z
77042024-02-01T17:30:47.516ZVerifying data now
77052024-02-01T17:30:52.483ZNo change after verify
77062024-02-01T17:30:52.483ZLoop: 35 Downstairs dump after verify (and repair):
77072024-02-01T17:30:52.513ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77082024-02-01T17:30:52.513Z 0 000-019 69 69 69 99 99 99 F F F
77092024-02-01T17:30:52.513Z 1 020-039 71 71 71 104 104 104 F F F
77102024-02-01T17:30:52.513Z 2 040-059 71 71 71 104 104 104 F F F
77112024-02-01T17:30:52.513Z 3 060-079 65 65 65 94 94 94 F F F
77122024-02-01T17:30:52.635Z 4 080-099 68 68 68 96 96 96 F F F
77132024-02-01T17:30:52.636Z 5 100-119 71 71 71 104 104 104 F F F
77142024-02-01T17:30:52.636Z 6 120-139 66 66 66 95 95 95 F F F
77152024-02-01T17:30:52.636Z 7 140-159 70 70 70 101 101 101 F F F
77162024-02-01T17:30:52.636Z 8 160-179 70 70 70 102 102 102 F F F
77172024-02-01T17:30:52.636Z 9 180-199 71 71 71 104 104 104 F F F
77182024-02-01T17:30:52.637Z 10 200-219 65 65 65 94 94 94 F F F
77192024-02-01T17:30:52.637Z 11 220-239 70 70 70 101 101 101 F F F
77202024-02-01T17:30:52.637Z 12 240-259 63 63 63 92 92 92 F F F
77212024-02-01T17:30:52.637Z 13 260-279 70 70 70 103 103 103 F F F
77222024-02-01T17:30:52.637Z 14 280-299 68 68 68 97 97 97 F F F
77232024-02-01T17:30:52.637Z 15 300-319 62 62 62 89 89 89 F F F
77242024-02-01T17:30:52.637Z 16 320-339 60 60 60 83 83 83 F F F
77252024-02-01T17:30:52.637Z 17 340-359 70 70 70 100 100 100 F F F
77262024-02-01T17:30:52.637Z 18 360-379 70 70 70 100 100 100 F F F
77272024-02-01T17:30:52.637Z 19 380-399 62 62 62 87 87 87 F F F
77282024-02-01T17:30:52.637Z 20 400-419 66 66 66 95 95 95 F F F
77292024-02-01T17:30:52.637Z 21 420-439 70 70 70 100 100 100 F F F
77302024-02-01T17:30:52.637Z 22 440-459 71 71 71 104 104 104 F F F
77312024-02-01T17:30:52.637Z 23 460-479 69 69 69 99 99 99 F F F
77322024-02-01T17:30:52.637Z 24 480-499 71 71 71 104 104 104 F F F
77332024-02-01T17:30:52.637Z 25 500-519 65 65 65 94 94 94 F F F
77342024-02-01T17:30:52.637Z 26 520-539 68 68 68 98 98 98 F F F
77352024-02-01T17:30:52.637Z 27 540-559 68 68 68 98 98 98 F F F
77362024-02-01T17:30:52.637Z 28 560-579 61 61 61 85 85 85 F F F
77372024-02-01T17:30:52.637Z 29 580-599 69 69 69 99 99 99 F F F
77382024-02-01T17:30:52.638ZMax gen: 71, Max flush: 104
77392024-02-01T17:30:52.638Z
77402024-02-01T17:30:52.638ZBegin loop 36
77412024-02-01T17:30:52.638ZDownstairs to restart: 1
77422024-02-01T17:30:52.646Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.809443165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
77432024-02-01T17:30:52.647Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}"Crucible runtime is spawned
77442024-02-01T17:30:52.647Z,"v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810054857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
77452024-02-01T17:30:52.648Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810129159Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
77462024-02-01T17:30:52.648Z{"msg":"Crucible ce795192-5d17-40f6-9608-5d5454ec9ddf has session id: 950adccd-3a5e-48ea-994c-7c9f0721d76f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.81016988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77472024-02-01T17:30:52.648Z{"msg":"Upstairs opts: Upstairs UUID: ce795192-5d17-40f6-9608-5d5454ec9ddf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.81025363Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77482024-02-01T17:30:52.648Z{"msg":"Crucible stats registered with UUID: ce795192-5d17-40f6-9608-5d5454ec9ddf","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810327571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77492024-02-01T17:30:52.648Z{"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810369786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
77502024-02-01T17:30:52.648Z{"msg":"ce795192-5d17-40f6-9608-5d5454ec9ddf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810432576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77512024-02-01T17:30:52.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810472814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77522024-02-01T17:30:52.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810556323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77532024-02-01T17:30:52.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810589244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77542024-02-01T17:30:52.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810626921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77552024-02-01T17:30:52.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810724521Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77562024-02-01T17:30:52.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810772163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77572024-02-01T17:30:52.648Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.810919972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77582024-02-01T17:30:52.649Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.811051935Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77592024-02-01T17:30:52.649Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.811130871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77602024-02-01T17:30:52.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.811180014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77612024-02-01T17:30:52.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.811259479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77622024-02-01T17:30:52.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.811351515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"io task","client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77632024-02-01T17:30:52.649Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.812307462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77642024-02-01T17:30:52.649Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.812421003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77652024-02-01T17:30:52.649Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.812475067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77662024-02-01T17:30:52.649Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.812516898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77672024-02-01T17:30:52.649Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.812575542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77682024-02-01T17:30:52.649Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.8126061Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77692024-02-01T17:30:52.649Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.813502692Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77702024-02-01T17:30:52.650Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.813567241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77712024-02-01T17:30:52.650Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.813894493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77722024-02-01T17:30:52.650Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814288304Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77732024-02-01T17:30:52.651Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.81437942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77742024-02-01T17:30:52.651Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814444608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77752024-02-01T17:30:52.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814470967Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77762024-02-01T17:30:52.651Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814530053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77772024-02-01T17:30:52.651Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814561576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77782024-02-01T17:30:52.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814723702Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77792024-02-01T17:30:52.651Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814780929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77802024-02-01T17:30:52.651Z{"msg":"[0]R flush_numbers[0..12]: [99, 104, 104, 94, 96, 104, 95, 101, 102, 104, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814808695Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77812024-02-01T17:30:52.651Z{"msg":"[0]R generation[0..12]: [69, 71, 71, 65, 68, 71, 66, 70, 70, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814868994Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77822024-02-01T17:30:52.651Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814903925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77832024-02-01T17:30:52.651Z{"msg":"[1]R flush_numbers[0..12]: [99, 104, 104, 94, 96, 104, 95, 101, 102, 104, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814960387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77842024-02-01T17:30:52.652Z{"msg":"[1]R generation[0..12]: [69, 71, 71, 65, 68, 71, 66, 70, 70, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.814992604Z","hostname":"Wait for a query_work_queue command to finish before sending IO
77852024-02-01T17:30:52.652Zip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77862024-02-01T17:30:52.652Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815114844Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77872024-02-01T17:30:52.652Z{"msg":"[2]R flush_numbers[0..12]: [99, 104, 104, 94, 96, 104, 95, 101, 102, 104, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815152547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77882024-02-01T17:30:52.652Z{"msg":"[2]R generation[0..12]: [69, 71, 71, 65, 68, 71, 66, 70, 70, 71, 65, 70]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815209346Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77892024-02-01T17:30:52.652Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815241126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77902024-02-01T17:30:52.652Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815297386Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77912024-02-01T17:30:52.652Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815323947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77922024-02-01T17:30:52.652Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815349655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77932024-02-01T17:30:52.652Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.81538645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"":"downstairs","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77942024-02-01T17:30:52.652Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815443864Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77952024-02-01T17:30:52.652Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815515894Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"0","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77962024-02-01T17:30:52.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815543221Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"1","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77972024-02-01T17:30:52.653Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815568533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"client":"2","":"downstairs","session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77982024-02-01T17:30:52.653Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815647343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
77992024-02-01T17:30:52.653Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815672558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
78002024-02-01T17:30:52.653Z{"msg":"ce795192-5d17-40f6-9608-5d5454ec9ddf is now active with session: 950adccd-3a5e-48ea-994c-7c9f0721d76f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815729955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
78012024-02-01T17:30:52.653Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.815786666Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417,"session_id":"950adccd-3a5e-48ea-994c-7c9f0721d76f"}
78022024-02-01T17:30:52.653Z{"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:51.8158128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
78032024-02-01T17:30:52.653ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
78042024-02-01T17:30:52.654ZLoading write count information from file "/tmp/test_repair_verify.data"
78052024-02-01T17:30:52.654ZRun Repair workload
78062024-02-01T17:30:52.654Z01/30 Write block 138 len 36864 data: 7 5 5 5 4 4 4 4 4
78072024-02-01T17:30:52.655Z02/30 Read block 35 len 8192
78082024-02-01T17:30:52.655Z03/30 Read block 197 len 28672
78092024-02-01T17:30:52.655Z04/30 Write block 229 len 32768 data: 4 4 4 4 3 4 3 4
78102024-02-01T17:30:52.655Z05/30 Read block 552 len 16384
78112024-02-01T17:30:52.655Z06/30 Write block 219 len 20480 data: 6 6 5 4 3
78122024-02-01T17:30:52.655Z07/30 Read block 590 len 36864
78132024-02-01T17:30:52.655Z08/30 Write block 182 len 40960 data: 11 12 11 13 10 10 7 8 8 7
78142024-02-01T17:30:52.655Z09/30 Write block 304 len 40960 data: 6 6 4 3 4 3 3 3 6 8
78152024-02-01T17:30:52.655Z10/30 Read block 436 len 28672
78162024-02-01T17:30:52.655Z11/30 Write block 542 len 24576 data: 12 10 12 10 11 10
78172024-02-01T17:30:52.655Z12/30 Flush
78182024-02-01T17:30:52.655Z13/30 Write block 373 len 20480 data: 5 7 9 11 11
78192024-02-01T17:30:52.655Z14/30 Read block 83 len 4096
78202024-02-01T17:30:52.656Z15/30 Read block 296 len 4096
78212024-02-01T17:30:52.656Z16/30 Read block 373 len 24576
78222024-02-01T17:30:52.656Z17/30 Write block 239 len 40960 data: 4 4 3 5 5 5 5 6 6 6
78232024-02-01T17:30:52.656Z18/30 Write block 238 len 12288 data: 4 5 5
78242024-02-01T17:30:52.656Z19/30 Read block 305 len 24576
78252024-02-01T17:30:52.656Z20/30 Read block 475 len 28672
78262024-02-01T17:30:52.656Z21/30 Read block 348 len 4096
78272024-02-01T17:30:52.656Z22/30 Read block 248 len 28672
78282024-02-01T17:30:52.656Z23/30 Write block 200 len 40960 data: 6 6 8 9 9 8 9 12 12 10
78292024-02-01T17:30:52.656Z24/30 Write block 62 len 32768 data: 6 6 5 4 5 5 5 7
78302024-02-01T17:30:52.656Z25/30 Write block 136 len 12288 data: 7 8 8
78312024-02-01T17:30:52.656Z26/30 Read block 165 len 8192
78322024-02-01T17:30:52.656Z27/30 Read block 490 len 32768
78332024-02-01T17:30:52.657Z28/30 Write block 140 len 40960 data: 6 6 5 5 5 5 5 4 7 7
78342024-02-01T17:30:52.657Z29/30 Read block 177 len 32768
78352024-02-01T17:30:52.657Z30/30 Read block 486 len 32768
78362024-02-01T17:30:52.657Z----------------------------------------------------------------
78372024-02-01T17:30:52.657Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
78382024-02-01T17:30:52.657ZStates: Active Active Active
78392024-02-01T17:30:52.657ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
78402024-02-01T17:30:52.657Z 1 Acked 1000 Write 9 Done Done Done false
78412024-02-01T17:30:52.657Z 2 Acked 1001 Read 2 Done Done Done false
78422024-02-01T17:30:52.663Z 3 Acked 1002 Read 7 Done Done Done false
78432024-02-01T17:30:52.663Z 4 Acked 1003 Write 8 Done Done Done false
78442024-02-01T17:30:52.663Z 5 Acked 1004 Read 4 Done Done Done false
78452024-02-01T17:30:52.663Z 6 Acked 1005 Write 5 Done Done Done false
78462024-02-01T17:30:52.663Z 7 Acked 1006 Read 9 Done Done Done false
78472024-02-01T17:30:52.663Z 8 Acked 1007 Write 10 Done Sent Done false
78482024-02-01T17:30:52.663Z 9 Acked 1008 Write 10 Done Sent Done false
78492024-02-01T17:30:52.663Z 10 Acked 1009 Read 7 Done Sent Done false
78502024-02-01T17:30:52.663Z 11 Acked 1010 Write 6 Done Sent Done false
78512024-02-01T17:30:52.664Z 12 Acked 1011 Flush 0 Done Sent Done false
78522024-02-01T17:30:52.664Z 13 Acked 1012 Write 5 Done Sent Done false
78532024-02-01T17:30:52.664Z 14 Acked 1013 Read 1 Done Sent Done false
78542024-02-01T17:30:52.664Z 15 Acked 1014 Read 1 Done Sent Done false
78552024-02-01T17:30:52.664Z 16 Acked 1015 Read 6 Done Sent Done false
78562024-02-01T17:30:52.664Z 17 Acked 1016 Write 10 Done Sent Done false
78572024-02-01T17:30:52.664Z 18 Acked 1017 Write 3 Done Sent Done false
78582024-02-01T17:30:52.664Z 19 Acked 1018 Read 6 Done Sent Done false
78592024-02-01T17:30:52.664Z 20 Acked 1019 Read 7 Done Sent Done false
78602024-02-01T17:30:52.664Z 21 Acked 1020 Read 1 Done Sent Done false
78612024-02-01T17:30:52.664Z 22 Acked 1021 Read 7 Done Sent Done false
78622024-02-01T17:30:52.664Z 23 Acked 1022 Write 10 Done Sent Done false
78632024-02-01T17:30:52.664Z 24 Acked 1023 Write 8 Done Sent Done false
78642024-02-01T17:30:52.665Z 25 Acked 1024 Write 3 Done Sent Done false
78652024-02-01T17:30:52.665Z 26 Acked 1025 Read 2 Done Sent Done false
78662024-02-01T17:30:52.665Z 27 Acked 1026 Read 8 Done Sent Done false
78672024-02-01T17:30:52.665Z 28 Acked 1027 Write 10 Done Sent Done false
78682024-02-01T17:30:52.665Z 29 Acked 1028 Read 8 Done Sent Done false
78692024-02-01T17:30:52.665Z 30 Acked 1029 Read 8 Done Sent Done false
78702024-02-01T17:30:52.665Z STATES DS:0 DS:1 DS:2 TOTAL
78712024-02-01T17:30:52.665Z New 0 0 0 0
78722024-02-01T17:30:52.665Z Sent 0 23 0 23
78732024-02-01T17:30:52.665Z Done 30 7 30 67
78742024-02-01T17:30:52.665Z Skipped 0 0 0 0
78752024-02-01T17:30:52.665Z Error 0 0 0 0
78762024-02-01T17:30:52.665ZLast Flush: 1011 0 1011
78772024-02-01T17:30:52.667ZDownstairs last five completed:
78782024-02-01T17:30:52.667ZUpstairs last five completed: 30 29 28 27 26
78792024-02-01T17:30:52.667Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:51.864928065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1417}
78802024-02-01T17:30:52.667ZWrote out file "/tmp/test_repair_verify.data"
78812024-02-01T17:30:52.667Z
78822024-02-01T17:30:52.667ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
78832024-02-01T17:30:52.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78842024-02-01T17:30:52.667Z 0 000-019 69 69 69 99 99 99 F F F
78852024-02-01T17:30:52.667Z 1 020-039 71 71 71 104 104 104 F F F
78862024-02-01T17:30:52.667Z 2 040-059 71 71 71 104 104 104 F F F
78872024-02-01T17:30:52.667Z 3 060-079 65 65 65 94 94 94 T F T <---
78882024-02-01T17:30:52.668Z 4 080-099 68 68 68 96 96 96 F F F
78892024-02-01T17:30:52.668Z 5 100-119 71 71 71 104 104 104 F F F
78902024-02-01T17:30:52.668Z 6 120-139 72 66 72 105 95 105 T T T <---
78912024-02-01T17:30:52.668Z 7 140-159 72 70 72 105 101 105 T T T <---
78922024-02-01T17:30:52.668Z 8 160-179 70 70 70 102 102 102 F F F
78932024-02-01T17:30:52.668Z 9 180-199 72 71 72 105 104 105 F F F <---
78942024-02-01T17:30:52.668Z 10 200-219 72 65 72 105 94 105 T T T <---
78952024-02-01T17:30:52.668Z 11 220-239 72 70 72 105 101 105 T T T <---
78962024-02-01T17:30:52.668Z 12 240-259 63 63 63 92 92 92 T F T <---
78972024-02-01T17:30:52.668Z 13 260-279 70 70 70 103 103 103 F F F
78982024-02-01T17:30:52.668Z 14 280-299 68 68 68 97 97 97 F F F
78992024-02-01T17:30:52.668Z 15 300-319 72 62 72 105 89 105 F F F <---
79002024-02-01T17:30:52.668Z 16 320-339 60 60 60 83 83 83 F F F
79012024-02-01T17:30:52.669Z 17 340-359 70 70 70 100 100 100 F F F
79022024-02-01T17:30:52.669Z 18 360-379 70 70 70 100 100 100 T F T <---
79032024-02-01T17:30:52.669Z 19 380-399 62 62 62 87 87 87 F F F
79042024-02-01T17:30:52.670Z 20 400-419 66 66 66 95 95 95 F F F
79052024-02-01T17:30:52.670Z 21 420-439 70 70 70 100 100 100 F F F
79062024-02-01T17:30:52.670Z 22 440-459 71 71 71 104 104 104 F F F
79072024-02-01T17:30:52.670Z 23 460-479 69 69 69 99 99 99 F F F
79082024-02-01T17:30:52.670Z 24 480-499 71 71 71 104 104 104 F F F
79092024-02-01T17:30:52.670Z 25 500-519 65 65 65 94 94 94 F F F
79102024-02-01T17:30:52.670Z 26 520-539 68 68 68 98 98 98 F F F
79112024-02-01T17:30:52.670Z 27 540-559 72 68 72 105 98 105 F F F <---
79122024-02-01T17:30:52.670Z 28 560-579 61 61 61 85 85 85 F F F
79132024-02-01T17:30:52.670Z 29 580-599 69 69 69 99 99 99 F F F
79142024-02-01T17:30:52.670ZMax gen: 72, Max flush: 105
79152024-02-01T17:30:52.670ZError: Difference in extent metadata found!
79162024-02-01T17:30:52.670ZA Difference in extent metadata is expected here
79172024-02-01T17:30:52.671ZOn loop 36
79182024-02-01T17:30:52.671Z
79192024-02-01T17:30:52.671ZVerifying data now
79202024-02-01T17:30:58.544ZNo change after verify
79212024-02-01T17:30:58.544ZLoop: 36 Downstairs dump after verify (and repair):
79222024-02-01T17:30:58.564ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
79232024-02-01T17:30:58.565Z 0 000-019 69 69 69 99 99 99 F F F
79242024-02-01T17:30:58.565Z 1 020-039 71 71 71 104 104 104 F F F
79252024-02-01T17:30:58.572Z 2 040-059 71 71 71 104 104 104 F F F
79262024-02-01T17:30:58.576Z 3 060-079 73 73 73 106 106 106 F F F
79272024-02-01T17:30:58.576Z 4 080-099 68 68 68 96 96 96 F F F
79282024-02-01T17:30:58.576Z 5 100-119 71 71 71 104 104 104 F F F
79292024-02-01T17:30:58.576Z 6 120-139 73 73 73 106 106 106 F F F
79302024-02-01T17:30:58.576Z 7 140-159 73 73 73 106 106 106 F F F
79312024-02-01T17:30:58.576Z 8 160-179 70 70 70 102 102 102 F F F
79322024-02-01T17:30:58.576Z 9 180-199 72 72 72 105 105 105 F F F
79332024-02-01T17:30:58.577Z 10 200-219 73 73 73 106 106 106 F F F
79342024-02-01T17:30:58.577Z 11 220-239 73 73 73 106 106 106 F F F
79352024-02-01T17:30:58.577Z 12 240-259 73 73 73 106 106 106 F F F
79362024-02-01T17:30:58.577Z 13 260-279 70 70 70 103 103 103 F F F
79372024-02-01T17:30:58.577Z 14 280-299 68 68 68 97 97 97 F F F
79382024-02-01T17:30:58.577Z 15 300-319 72 72 72 105 105 105 F F F
79392024-02-01T17:30:58.577Z 16 320-339 60 60 60 83 83 83 F F F
79402024-02-01T17:30:58.577Z 17 340-359 70 70 70 100 100 100 F F F
79412024-02-01T17:30:58.577Z 18 360-379 73 73 73 106 106 106 F F F
79422024-02-01T17:30:58.577Z 19 380-399 62 62 62 87 87 87 F F F
79432024-02-01T17:30:58.577Z 20 400-419 66 66 66 95 95 95 F F F
79442024-02-01T17:30:58.577Z 21 420-439 70 70 70 100 100 100 F F F
79452024-02-01T17:30:58.577Z 22 440-459 71 71 71 104 104 104 F F F
79462024-02-01T17:30:58.577Z 23 460-479 69 69 69 99 99 99 F F F
79472024-02-01T17:30:58.578Z 24 480-499 71 71 71 104 104 104 F F F
79482024-02-01T17:30:58.578Z 25 500-519 65 65 65 94 94 94 F F F
79492024-02-01T17:30:58.578Z 26 520-539 68 68 68 98 98 98 F F F
79502024-02-01T17:30:58.579Z 27 540-559 72 72 72 105 105 105 F F F
79512024-02-01T17:30:58.579Z 28 560-579 61 61 61 85 85 85 F F F
79522024-02-01T17:30:58.579Z 29 580-599 69 69 69 99 99 99 F F F
79532024-02-01T17:30:58.579ZMax gen: 73, Max flush: 106
79542024-02-01T17:30:58.579Z
79552024-02-01T17:30:58.579ZBegin loop 37
79562024-02-01T17:30:58.579ZDownstairs to restart: 0
79572024-02-01T17:30:58.600Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.869194485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
79582024-02-01T17:30:58.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.869680436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
79592024-02-01T17:30:58.601Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.869752178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
79602024-02-01T17:30:58.606Z{"msg":"Crucible 62314d46-c584-4585-9cff-f99fe3016199 has session id: fe25f76a-be47-4f58-8ca8-c0de4d035718","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.869848881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79612024-02-01T17:30:58.606Z{"msg":"Upstairs opts: Upstairs UUID: 62314d46-c584-4585-9cff-f99fe3016199, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.869934124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79622024-02-01T17:30:58.606Z{Crucible runtime is spawned
79632024-02-01T17:30:58.606Z"msg":"Crucible stats registered with UUID: 62314d46-c584-4585-9cff-f99fe3016199","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870023102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79642024-02-01T17:30:58.606Z{"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870116878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
79652024-02-01T17:30:58.606Z{"msg":"62314d46-c584-4585-9cff-f99fe3016199 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870342724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79662024-02-01T17:30:58.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870417104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79672024-02-01T17:30:58.606Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870453742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79682024-02-01T17:30:58.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870514839Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79692024-02-01T17:30:58.606Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870543764Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79702024-02-01T17:30:58.606Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870602563Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79712024-02-01T17:30:58.607Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.870919614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79722024-02-01T17:30:58.607Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871020202Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79732024-02-01T17:30:58.607Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871101964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79742024-02-01T17:30:58.607Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871152198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79752024-02-01T17:30:58.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871271555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79762024-02-01T17:30:58.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871319359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79772024-02-01T17:30:58.608Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.871397916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"io task","client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79782024-02-01T17:30:58.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872264754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79792024-02-01T17:30:58.608Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872348677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79802024-02-01T17:30:58.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872378361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79812024-02-01T17:30:58.608Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872442999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79822024-02-01T17:30:58.608Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872487124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79832024-02-01T17:30:58.609Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.872532193Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79842024-02-01T17:30:58.609Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.8734237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79852024-02-01T17:30:58.609Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.873493826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79862024-02-01T17:30:58.609Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.873534823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79872024-02-01T17:30:58.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874395412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79882024-02-01T17:30:58.614Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874480563Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79892024-02-01T17:30:58.614Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874528367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79902024-02-01T17:30:58.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874582261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79912024-02-01T17:30:58.614Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874625804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79922024-02-01T17:30:58.614Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874680213Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79932024-02-01T17:30:58.615Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874720928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79942024-02-01T17:30:58.615Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874782369Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79952024-02-01T17:30:58.615Z{"msg":"[0]R flush_numbers[0..12]: [99, 104, 104, 106, 96, 104, 106, 106, 102, 105, 106, 106]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
79962024-02-01T17:30:58.615Z,"time":"2024-02-01T17:30:57.874828045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79972024-02-01T17:30:58.615Z{"msg":"[0]R generation[0..12]: [69, 71, 71, 73, 68, 71, 73, 73, 70, 72, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.874934649Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79982024-02-01T17:30:58.615Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875000171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
79992024-02-01T17:30:58.615Z{"msg":"[1]R flush_numbers[0..12]: [99, 104, 104, 106, 96, 104, 106, 106, 102, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875042168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80002024-02-01T17:30:58.615Z{"msg":"[1]R generation[0..12]: [69, 71, 71, 73, 68, 71, 73, 73, 70, 72, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875083992Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80012024-02-01T17:30:58.615Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875145014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80022024-02-01T17:30:58.615Z{"msg":"[2]R flush_numbers[0..12]: [99, 104, 104, 106, 96, 104, 106, 106, 102, 105, 106, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875189783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80032024-02-01T17:30:58.615Z{"msg":"[2]R generation[0..12]: [69, 71, 71, 73, 68, 71, 73, 73, 70, 72, 73, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875245978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80042024-02-01T17:30:58.615Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875292359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80052024-02-01T17:30:58.616Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875350305Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80062024-02-01T17:30:58.616Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875394057Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80072024-02-01T17:30:58.616Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.87544631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80082024-02-01T17:30:58.616Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875498406Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"":"downstairs","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80092024-02-01T17:30:58.616Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875541824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80102024-02-01T17:30:58.616Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875578475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"0","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80112024-02-01T17:30:58.616Z{"msg"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
80122024-02-01T17:30:58.616Z:"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875641072Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"1","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80132024-02-01T17:30:58.616Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875707532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"client":"2","":"downstairs","session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80142024-02-01T17:30:58.616Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875749643Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80152024-02-01T17:30:58.616Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875801887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80162024-02-01T17:30:58.617Z{"msg":"62314d46-c584-4585-9cff-f99fe3016199 is now active with session: fe25f76a-be47-4f58-8ca8-c0de4d035718","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875844268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80172024-02-01T17:30:58.617Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.87590048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425,"session_id":"fe25f76a-be47-4f58-8ca8-c0de4d035718"}
80182024-02-01T17:30:58.617Z{"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:30:57.875940486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
80192024-02-01T17:30:58.617ZLoading write count information from file "/tmp/test_repair_verify.data"
80202024-02-01T17:30:58.617ZRun Repair workload
80212024-02-01T17:30:58.617Z01/30 Write block 286 len 24576 data: 9 7 7 7 5 6
80222024-02-01T17:30:58.618Z02/30 Flush
80232024-02-01T17:30:58.618Z03/30 Write block 436 len 8192 data: 7 8
80242024-02-01T17:30:58.621Z04/30 Write block 560 len 8192 data: 9 8
80252024-02-01T17:30:58.621Z05/30 Read block 391 len 36864
80262024-02-01T17:30:58.622Z06/30 Read block 317 len 12288
80272024-02-01T17:30:58.622Z07/30 Flush
80282024-02-01T17:30:58.622Z08/30 Write block 63 len 28672 data: 7 6 5 6 6 6 8
80292024-02-01T17:30:58.622Z09/30 Read block 260 len 28672
80302024-02-01T17:30:58.625Z10/30 Read block 499 len 16384
80312024-02-01T17:30:58.625Z11/30 Write block 357 len 40960 data: 6 8 8 8 9 10 9 8 7 7
80322024-02-01T17:30:58.628Z12/30 Write block 305 len 4096 data: 7
80332024-02-01T17:30:58.628Z13/30 Read block 408 len 36864
80342024-02-01T17:30:58.628Z14/30 Read block 122 len 16384
80352024-02-01T17:30:58.633Z15/30 Read block 488 len 28672
80362024-02-01T17:30:58.633Z16/30 Read block 264 len 4096
80372024-02-01T17:30:58.634Z17/30 Read block 512 len 8192
80382024-02-01T17:30:58.634Z18/30 Flush
80392024-02-01T17:30:58.637Z19/30 Write block 209 len 4096 data: 11
80402024-02-01T17:30:58.638Z20/30 Write block 0 len 36864 data: 3 3 3 4 5 5 6 6 6
80412024-02-01T17:30:58.638Z21/30 Read block 563 len 32768
80422024-02-01T17:30:58.654Z22/30 Write block 134 len 24576 data: 5 6 8 9 9 6
80432024-02-01T17:30:58.654Z23/30 Flush
80442024-02-01T17:30:58.654Z24/30 Write block 140 len 28672 data: 7 7 6 6 6 6 6
80452024-02-01T17:30:58.654Z25/30 Read block 354 len 12288
80462024-02-01T17:30:58.654Z26/30 Write block 508 len 24576 data: 4 5 4 5 5 5
80472024-02-01T17:30:58.654Z27/30 Read block 356 len 12288
80482024-02-01T17:30:58.654Z28/30 Write block 470 len 20480 data: 7 7 7 5 3
80492024-02-01T17:30:58.654Z29/30 Read block 254 len 28672
80502024-02-01T17:30:58.655Z30/30 Write block 324 len 4096 data: 5
80512024-02-01T17:30:58.655Z----------------------------------------------------------------
80522024-02-01T17:30:58.655Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:28
80532024-02-01T17:30:58.655ZStates: Active Active Active
80542024-02-01T17:30:58.655ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80552024-02-01T17:30:58.655Z 3 Acked 1002 Write 2 Sent Done Done false
80562024-02-01T17:30:58.655Z 4 Acked 1003 Write 2 Sent Done Done false
80572024-02-01T17:30:58.655Z 5 Acked 1004 Read 9 Sent Done Done false
80582024-02-01T17:30:58.655Z 6 Acked 1005 Read 3 Sent Done Done false
80592024-02-01T17:30:58.655Z 7 Acked 1006 Flush 0 Sent Done Done false
80602024-02-01T17:30:58.655Z 8 Acked 1007 Write 7 Sent Done Done false
80612024-02-01T17:30:58.655Z 9 Acked 1008 Read 7 Sent Done Done false
80622024-02-01T17:30:58.655Z 10 Acked 1009 Read 4 Sent Done Done false
80632024-02-01T17:30:58.655Z 11 Acked 1010 Write 10 Sent Done Done false
80642024-02-01T17:30:58.655Z 12 Acked 1011 Write 1 Sent Done Done false
80652024-02-01T17:30:58.655Z 13 Acked 1012 Read 9 Sent Done Done false
80662024-02-01T17:30:58.655Z 14 Acked 1013 Read 4 Sent Done Done false
80672024-02-01T17:30:58.655Z 15 Acked 1014 Read 7 Sent Done Done false
80682024-02-01T17:30:58.655Z 16 Acked 1015 Read 1 Sent Done Done false
80692024-02-01T17:30:58.655Z 17 Acked 1016 Read 2 Sent Done Done false
80702024-02-01T17:30:58.655Z 18 Acked 1017 Flush 0 Sent Done Done false
80712024-02-01T17:30:58.655Z 19 Acked 1018 Write 1 Sent Done Done false
80722024-02-01T17:30:58.656Z 20 Acked 1019 Write 9 Sent Done Done false
80732024-02-01T17:30:58.656Z 21 Acked 1020 Read 8 Sent Done Done false
80742024-02-01T17:30:58.656Z 22 Acked 1021 Write 6 Sent Done Done false
80752024-02-01T17:30:58.656Z 23 Acked 1022 Flush 0 Sent Done Done false
80762024-02-01T17:30:58.656Z 24 Acked 1023 Write 7 Sent Done Done false
80772024-02-01T17:30:58.656Z 25 Acked 1024 Read 3 Sent Done Done false
80782024-02-01T17:30:58.656Z 26 Acked 1025 Write 6 Sent Done Done false
80792024-02-01T17:30:58.656Z 27 Acked 1026 Read 3 Sent Done Done false
80802024-02-01T17:30:58.656Z 28 Acked 1027 Write 5 Sent Done Done false
80812024-02-01T17:30:58.656Z 29 Acked 1028 Read 7 Sent Done Sent false
80822024-02-01T17:30:58.656Z 30 Acked 1029 Write 1 Sent Sent Sent false
80832024-02-01T17:30:58.656Z STATES DS:0 DS:1 DS:2 TOTAL
80842024-02-01T17:30:58.656Z New 0 0 0 0
80852024-02-01T17:30:58.656Z Sent 28 1 2 31
80862024-02-01T17:30:58.656Z Done 0 27 26 53
80872024-02-01T17:30:58.656Z Skipped 0 0 0 0
80882024-02-01T17:30:58.656Z Error 0 0 0 0
80892024-02-01T17:30:58.656ZLast Flush: 1001 1022 1022
80902024-02-01T17:30:58.657ZDownstairs last five completed: 1001 1000
80912024-02-01T17:30:58.657ZUpstairs last five completed: 30 29 28 27 26
80922024-02-01T17:30:58.661Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:30:57.923088754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1425}
80932024-02-01T17:30:58.661ZWrote out file "/tmp/test_repair_verify.data"
80942024-02-01T17:30:58.661Z
80952024-02-01T17:30:58.667ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80962024-02-01T17:30:58.693ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80972024-02-01T17:30:58.693Z 0 000-019 69 74 74 99 110 110 F F F <---
80982024-02-01T17:30:58.702Z 1 020-039 71 71 71 104 104 104 F F F
80992024-02-01T17:30:58.702Z 2 040-059 71 71 71 104 104 104 F F F
81002024-02-01T17:30:58.702Z 3 060-079 73 74 74 106 109 109 F F F <---
81012024-02-01T17:30:58.702Z 4 080-099 68 68 68 96 96 96 F F F
81022024-02-01T17:30:58.702Z 5 100-119 71 71 71 104 104 104 F F F
81032024-02-01T17:30:58.702Z 6 120-139 73 74 74 106 110 110 F F F <---
81042024-02-01T17:30:58.702Z 7 140-159 73 73 73 106 106 106 F T T <---
81052024-02-01T17:30:58.702Z 8 160-179 70 70 70 102 102 102 F F F
81062024-02-01T17:30:58.702Z 9 180-199 72 72 72 105 105 105 F F F
81072024-02-01T17:30:58.702Z 10 200-219 73 74 74 106 110 110 F F F <---
81082024-02-01T17:30:58.702Z 11 220-239 73 73 73 106 106 106 F F F
81092024-02-01T17:30:58.704Z 12 240-259 73 73 73 106 106 106 F F F
81102024-02-01T17:30:58.704Z 13 260-279 70 70 70 103 103 103 F F F
81112024-02-01T17:30:58.704Z 14 280-299 74 74 74 107 107 107 F F F
81122024-02-01T17:30:58.704Z 15 300-319 72 74 74 105 109 109 F F F <---
81132024-02-01T17:30:58.704Z 16 320-339 60 60 60 83 83 83 F T T <---
81142024-02-01T17:30:58.704Z 17 340-359 70 74 74 100 109 109 F F F <---
81152024-02-01T17:30:58.704Z 18 360-379 73 74 74 106 109 109 F F F <---
81162024-02-01T17:30:58.704Z 19 380-399 62 62 62 87 87 87 F F F
81172024-02-01T17:30:58.704Z 20 400-419 66 66 66 95 95 95 F F F
81182024-02-01T17:30:58.704Z 21 420-439 70 74 74 100 108 108 F F F <---
81192024-02-01T17:30:58.704Z 22 440-459 71 71 71 104 104 104 F F F
81202024-02-01T17:30:58.704Z 23 460-479 69 69 69 99 99 99 F T T <---
81212024-02-01T17:30:58.705Z 24 480-499 71 71 71 104 104 104 F F F
81222024-02-01T17:30:58.705Z 25 500-519 65 65 65 94 94 94 F T T <---
81232024-02-01T17:30:58.705Z 26 520-539 68 68 68 98 98 98 F F F
81242024-02-01T17:30:58.705Z 27 540-559 72 72 72 105 105 105 F F F
81252024-02-01T17:30:58.705Z 28 560-579 61 74 74 85 108 108 F F F <---
81262024-02-01T17:30:58.705Z 29 580-599 69 69 69 99 99 99 F F F
81272024-02-01T17:30:58.705ZMax gen: 74, Max flush: 110
81282024-02-01T17:30:58.705ZError: Difference in extent metadata found!
81292024-02-01T17:30:58.709ZA Difference in extent metadata is expected here
81302024-02-01T17:30:58.710ZOn loop 37
81312024-02-01T17:30:58.710Z
81322024-02-01T17:30:58.710ZVerifying data now
81332024-02-01T17:31:03.812ZNo change after verify
81342024-02-01T17:31:03.813ZLoop: 37 Downstairs dump after verify (and repair):
81352024-02-01T17:31:03.835ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81362024-02-01T17:31:03.835Z 0 000-019 74 74 74 110 110 110 F F F
81372024-02-01T17:31:03.839Z 1 020-039 71 71 71 104 104 104 F F F
81382024-02-01T17:31:03.839Z 2 040-059 71 71 71 104 104 104 F F F
81392024-02-01T17:31:03.839Z 3 060-079 74 74 74 109 109 109 F F F
81402024-02-01T17:31:03.839Z 4 080-099 68 68 68 96 96 96 F F F
81412024-02-01T17:31:03.839Z 5 100-119 71 71 71 104 104 104 F F F
81422024-02-01T17:31:03.839Z 6 120-139 74 74 74 110 110 110 F F F
81432024-02-01T17:31:03.839Z 7 140-159 75 75 75 111 111 111 F F F
81442024-02-01T17:31:03.839Z 8 160-179 70 70 70 102 102 102 F F F
81452024-02-01T17:31:03.839Z 9 180-199 72 72 72 105 105 105 F F F
81462024-02-01T17:31:03.839Z 10 200-219 74 74 74 110 110 110 F F F
81472024-02-01T17:31:03.840Z 11 220-239 73 73 73 106 106 106 F F F
81482024-02-01T17:31:03.840Z 12 240-259 73 73 73 106 106 106 F F F
81492024-02-01T17:31:03.840Z 13 260-279 70 70 70 103 103 103 F F F
81502024-02-01T17:31:03.840Z 14 280-299 74 74 74 107 107 107 F F F
81512024-02-01T17:31:03.840Z 15 300-319 74 74 74 109 109 109 F F F
81522024-02-01T17:31:03.840Z 16 320-339 75 75 75 111 111 111 F F F
81532024-02-01T17:31:03.845Z 17 340-359 74 74 74 109 109 109 F F F
81542024-02-01T17:31:03.845Z 18 360-379 74 74 74 109 109 109 F F F
81552024-02-01T17:31:03.845Z 19 380-399 62 62 62 87 87 87 F F F
81562024-02-01T17:31:03.845Z 20 400-419 66 66 66 95 95 95 F F F
81572024-02-01T17:31:03.845Z 21 420-439 74 74 74 108 108 108 F F F
81582024-02-01T17:31:03.845Z 22 440-459 71 71 71 104 104 104 F F F
81592024-02-01T17:31:03.845Z 23 460-479 75 75 75 111 111 111 F F F
81602024-02-01T17:31:03.845Z 24 480-499 71 71 71 104 104 104 F F F
81612024-02-01T17:31:03.845Z 25 500-519 75 75 75 111 111 111 F F F
81622024-02-01T17:31:03.845Z 26 520-539 68 68 68 98 98 98 F F F
81632024-02-01T17:31:03.845Z 27 540-559 72 72 72 105 105 105 F F F
81642024-02-01T17:31:03.845Z 28 560-579 74 74 74 108 108 108 F F F
81652024-02-01T17:31:03.846Z 29 580-599 69 69 69 99 99 99 F F F
81662024-02-01T17:31:03.846ZMax gen: 75, Max flush: 111
81672024-02-01T17:31:03.849Z
81682024-02-01T17:31:03.849ZBegin loop 38
81692024-02-01T17:31:03.849ZDownstairs to restart: 0
81702024-02-01T17:31:03.869Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138038792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
81712024-02-01T17:31:03.874Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138534159Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
81722024-02-01T17:31:03.874Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138652068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
81732024-02-01T17:31:03.874Z{"msg":"Crucible 443f4291-bfe9-42e5-829e-72808195b756 has session id: 9a4a4b40-73b4-4667-9252-212a3973144c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138695447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81742024-02-01T17:31:03.874Z{"msg":"Crucible runtime is spawned
81752024-02-01T17:31:03.875ZUpstairs opts: Upstairs UUID: 443f4291-bfe9-42e5-829e-72808195b756, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138913123Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81762024-02-01T17:31:03.875Z{"msg":"Crucible stats registered with UUID: 443f4291-bfe9-42e5-829e-72808195b756","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138947778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81772024-02-01T17:31:03.875Z{"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.138970187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
81782024-02-01T17:31:03.875Z{"msg":"443f4291-bfe9-42e5-829e-72808195b756 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139197751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81792024-02-01T17:31:03.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139274683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81802024-02-01T17:31:03.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139393762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81812024-02-01T17:31:03.875Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139472783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81822024-02-01T17:31:03.875Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139520561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81832024-02-01T17:31:03.878Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139649969Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81842024-02-01T17:31:03.878Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139727688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81852024-02-01T17:31:03.879Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.139983801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81862024-02-01T17:31:03.879Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.140063089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81872024-02-01T17:31:03.879Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.140137799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81882024-02-01T17:31:03.879Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.140184018Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81892024-02-01T17:31:03.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.14025085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81902024-02-01T17:31:03.880Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.140309137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"io task","client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81912024-02-01T17:31:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141044632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81922024-02-01T17:31:03.880Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141148649Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81932024-02-01T17:31:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141176152Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81942024-02-01T17:31:03.880Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141203098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81952024-02-01T17:31:03.880Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141272536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81962024-02-01T17:31:03.880Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.141317382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81972024-02-01T17:31:03.880Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.142137242Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81982024-02-01T17:31:03.880Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.1422032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
81992024-02-01T17:31:03.888Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.142414569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82002024-02-01T17:31:03.888Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143250477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82012024-02-01T17:31:03.888Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143314636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82022024-02-01T17:31:03.888Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143341914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82032024-02-01T17:31:03.889Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143363342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82042024-02-01T17:31:03.889Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143404346Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82052024-02-01T17:31:03.889Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143428833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82062024-02-01T17:31:03.889Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143466943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82072024-02-01T17:31:03.889Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143491002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82082024-02-01T17:31:03.889Z{"msg":"[0]R flush_numbers[0..12]: [110, 104, 104, 109, 96, 104, 110, 111, 102, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143531924Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82092024-02-01T17:31:03.889Z{"msg":"[0]R generation[0..12]: [74, 71, 71, 74, 68, 71, 74, 75, 70, 72, 74, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143556843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82102024-02-01T17:31:03.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO
82112024-02-01T17:31:03.889Ztime":"2024-02-01T17:31:03.143598719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82122024-02-01T17:31:03.889Z{"msg":"[1]R flush_numbers[0..12]: [110, 104, 104, 109, 96, 104, 110, 111, 102, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143628344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82132024-02-01T17:31:03.889Z{"msg":"[1]R generation[0..12]: [74, 71, 71, 74, 68, 71, 74, 75, 70, 72, 74, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143669343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82142024-02-01T17:31:03.889Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.14369466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82152024-02-01T17:31:03.889Z{"msg":"[2]R flush_numbers[0..12]: [110, 104, 104, 109, 96, 104, 110, 111, 102, 105, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143718244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82162024-02-01T17:31:03.889Z{"msg":"[2]R generation[0..12]: [74, 71, 71, 74, 68, 71, 74, 75, 70, 72, 74, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143762529Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82172024-02-01T17:31:03.889Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.14378845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82182024-02-01T17:31:03.890Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143828185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82192024-02-01T17:31:03.890Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143852351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82202024-02-01T17:31:03.890Z{"msg":"Next flush: 112","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.14389184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82212024-02-01T17:31:03.890Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143916027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"":"downstairs","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82222024-02-01T17:31:03.890Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143956128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82232024-02-01T17:31:03.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.143977988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"0","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82242024-02-01T17:31:03.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144017918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"1","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82252024-02-01T17:31:03.890Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144048999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"client":"2","":"downstairs","session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82262024-02-01T17:31:03.890Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144074607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82272024-02-01T17:31:03.890Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144101416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82282024-02-01T17:31:03.890Z{"msg":"443f4291-bfe9-42e5-829e-72808195b756 is now active with session: 9a4a4b40-73b4-4667-9252-212a3973144c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144149866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82292024-02-01T17:31:03.890Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144171934Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433,"session_id":"9a4a4b40-73b4-4667-9252-212a3973144c"}
82302024-02-01T17:31:03.890Z{"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:03.144194025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
82312024-02-01T17:31:03.890ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82322024-02-01T17:31:03.890ZLoading write count information from file "/tmp/test_repair_verify.data"
82332024-02-01T17:31:03.891ZRun Repair workload
82342024-02-01T17:31:03.892Z01/30 Write block 56 len 16384 data: 8 7 5 7
82352024-02-01T17:31:03.892Z02/30 Read block 125 len 24576
82362024-02-01T17:31:03.892Z03/30 Write block 250 len 28672 data: 6 6 5 4 7 7 6
82372024-02-01T17:31:03.892Z04/30 Write block 586 len 4096 data: 8
82382024-02-01T17:31:03.892Z05/30 Read block 278 len 4096
82392024-02-01T17:31:03.892Z06/30 Flush
82402024-02-01T17:31:03.898Z07/30 Write block 362 len 8192 data: 11 10
82412024-02-01T17:31:03.898Z08/30 Read block 505 len 8192
82422024-02-01T17:31:03.898Z09/30 Flush
82432024-02-01T17:31:03.898Z10/30 Write block 292 len 40960 data: 6 7 7 8 8 7 10 9 8 6
82442024-02-01T17:31:03.898Z11/30 Read block 62 len 36864
82452024-02-01T17:31:03.898Z12/30 Write block 557 len 12288 data: 8 9 9
82462024-02-01T17:31:03.898Z13/30 Read block 208 len 24576
82472024-02-01T17:31:03.902Z14/30 Write block 455 len 28672 data: 3 5 7 7 9 8 7
82482024-02-01T17:31:03.902Z15/30 Write block 88 len 4096 data: 4
82492024-02-01T17:31:03.902Z16/30 Read block 542 len 16384
82502024-02-01T17:31:03.902Z17/30 Read block 200 len 24576
82512024-02-01T17:31:03.903Z18/30 Read block 93 len 40960
82522024-02-01T17:31:03.906Z19/30 Read block 58 len 4096
82532024-02-01T17:31:03.906Z20/30 Write block 32 len 20480 data: 5 4 5 6 7
82542024-02-01T17:31:03.906Z21/30 Write block 46 len 28672 data: 7 5 8 7 10 10 8
82552024-02-01T17:31:03.910Z22/30 Flush
82562024-02-01T17:31:03.913Z23/30 Write block 196 len 4096 data: 7
82572024-02-01T17:31:03.913Z24/30 Read block 234 len 4096
82582024-02-01T17:31:03.913Z25/30 Flush
82592024-02-01T17:31:03.917Z26/30 Write block 331 len 28672 data: 7 8 9 11 11 10 7
82602024-02-01T17:31:03.917Z27/30 Write block 255 len 16384 data: 8 7 7 8
82612024-02-01T17:31:03.917Z28/30 Read block 8 len 20480
82622024-02-01T17:31:03.920Z29/30 Read block 419 len 40960
82632024-02-01T17:31:03.920Z30/30 Read block 543 len 32768
82642024-02-01T17:31:03.925Z----------------------------------------------------------------
82652024-02-01T17:31:03.925Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
82662024-02-01T17:31:03.925ZStates: Active Active Active
82672024-02-01T17:31:03.925ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82682024-02-01T17:31:03.925Z 1 Acked 1000 Write 4 Done Done Done false
82692024-02-01T17:31:03.925Z 2 Acked 1001 Read 6 Sent Done Done false
82702024-02-01T17:31:03.925Z 3 Acked 1002 Write 7 Sent Done Done false
82712024-02-01T17:31:03.926Z 4 Acked 1003 Write 1 Sent Done Done false
82722024-02-01T17:31:03.926Z 5 Acked 1004 Read 1 Sent Done Done false
82732024-02-01T17:31:03.926Z 6 Acked 1005 Flush 0 Sent Done Done false
82742024-02-01T17:31:03.926Z 7 Acked 1006 Write 2 Sent Done Done false
82752024-02-01T17:31:03.926Z 8 Acked 1007 Read 2 Sent Done Done false
82762024-02-01T17:31:03.926Z 9 Acked 1008 Flush 0 Sent Done Done false
82772024-02-01T17:31:03.926Z 10 Acked 1009 Write 10 Sent Done Done false
82782024-02-01T17:31:03.926Z 11 Acked 1010 Read 9 Sent Done Done false
82792024-02-01T17:31:03.926Z 12 Acked 1011 Write 3 Sent Done Done false
82802024-02-01T17:31:03.926Z 13 Acked 1012 Read 6 Sent Done Done false
82812024-02-01T17:31:03.926Z 14 Acked 1013 Write 7 Sent Done Done false
82822024-02-01T17:31:03.928Z 15 Acked 1014 Write 1 Sent Done Done false
82832024-02-01T17:31:03.928Z 16 Acked 1015 Read 4 Sent Done Done false
82842024-02-01T17:31:03.928Z 17 Acked 1016 Read 6 Sent Done Done false
82852024-02-01T17:31:03.928Z 18 Acked 1017 Read 10 Sent Done Done false
82862024-02-01T17:31:03.928Z 19 Acked 1018 Read 1 Sent Done Done false
82872024-02-01T17:31:03.928Z 20 Acked 1019 Write 5 Sent Done Done false
82882024-02-01T17:31:03.928Z 21 Acked 1020 Write 7 Sent Done Done false
82892024-02-01T17:31:03.928Z 22 Acked 1021 Flush 0 Sent Done Done false
82902024-02-01T17:31:03.928Z 23 Acked 1022 Write 1 Sent Done Done false
82912024-02-01T17:31:03.928Z 24 Acked 1023 Read 1 Sent Done Done false
82922024-02-01T17:31:03.928Z 25 Acked 1024 Flush 0 Sent Done Done false
82932024-02-01T17:31:03.928Z 26 Acked 1025 Write 7 Sent Done Done false
82942024-02-01T17:31:03.928Z 27 Acked 1026 Write 4 Sent Done Done false
82952024-02-01T17:31:03.929Z 28 Acked 1027 Read 5 Sent Done Done false
82962024-02-01T17:31:03.929Z 29 Acked 1028 Read 10 Sent Done Done false
82972024-02-01T17:31:03.929Z 30 Acked 1029 Read 8 Sent Done Done false
82982024-02-01T17:31:03.929Z STATES DS:0 DS:1 DS:2 TOTAL
82992024-02-01T17:31:03.929Z New 0 0 0 0
83002024-02-01T17:31:03.929Z Sent 29 0 0 29
83012024-02-01T17:31:03.929Z Done 1 30 30 61
83022024-02-01T17:31:03.929Z Skipped 0 0 0 0
83032024-02-01T17:31:03.929Z Error 0 0 0 0
83042024-02-01T17:31:03.929ZLast Flush: 0 1024 1024
83052024-02-01T17:31:03.929ZDownstairs last five completed:
83062024-02-01T17:31:03.929ZUpstairs last five completed: 30 29 28 27 26
83072024-02-01T17:31:03.929Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:03.191371562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1433}
83082024-02-01T17:31:03.929ZWrote out file "/tmp/test_repair_verify.data"
83092024-02-01T17:31:03.933Z
83102024-02-01T17:31:03.938ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83112024-02-01T17:31:03.958ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83122024-02-01T17:31:03.958Z 0 000-019 74 74 74 110 110 110 F F F
83132024-02-01T17:31:03.963Z 1 020-039 71 76 76 104 114 114 F F F <---
83142024-02-01T17:31:03.963Z 2 040-059 71 76 76 104 114 114 T F F <---
83152024-02-01T17:31:03.963Z 3 060-079 74 74 74 109 109 109 F F F
83162024-02-01T17:31:03.963Z 4 080-099 68 76 76 96 114 114 F F F <---
83172024-02-01T17:31:03.963Z 5 100-119 71 71 71 104 104 104 F F F
83182024-02-01T17:31:03.963Z 6 120-139 74 74 74 110 110 110 F F F
83192024-02-01T17:31:03.964Z 7 140-159 75 75 75 111 111 111 F F F
83202024-02-01T17:31:03.964Z 8 160-179 70 70 70 102 102 102 F F F
83212024-02-01T17:31:03.964Z 9 180-199 72 76 76 105 115 115 F F F <---
83222024-02-01T17:31:03.964Z 10 200-219 74 74 74 110 110 110 F F F
83232024-02-01T17:31:03.964Z 11 220-239 73 73 73 106 106 106 F F F
83242024-02-01T17:31:03.964Z 12 240-259 73 76 76 106 112 112 F T T <---
83252024-02-01T17:31:03.964Z 13 260-279 70 70 70 103 103 103 F F F
83262024-02-01T17:31:03.964Z 14 280-299 74 76 76 107 114 114 F F F <---
83272024-02-01T17:31:03.964Z 15 300-319 74 76 76 109 114 114 F F F <---
83282024-02-01T17:31:03.964Z 16 320-339 75 75 75 111 111 111 F T T <---
83292024-02-01T17:31:03.964Z 17 340-359 74 74 74 109 109 109 F F F
83302024-02-01T17:31:03.964Z 18 360-379 74 76 76 109 113 113 F F F <---
83312024-02-01T17:31:03.964Z 19 380-399 62 62 62 87 87 87 F F F
83322024-02-01T17:31:03.965Z 20 400-419 66 66 66 95 95 95 F F F
83332024-02-01T17:31:03.967Z 21 420-439 74 74 74 108 108 108 F F F
83342024-02-01T17:31:03.967Z 22 440-459 71 76 76 104 114 114 F F F <---
83352024-02-01T17:31:03.967Z 23 460-479 75 76 76 111 114 114 F F F <---
83362024-02-01T17:31:03.967Z 24 480-499 71 71 71 104 104 104 F F F
83372024-02-01T17:31:03.967Z 25 500-519 75 75 75 111 111 111 F F F
83382024-02-01T17:31:03.967Z 26 520-539 68 68 68 98 98 98 F F F
83392024-02-01T17:31:03.967Z 27 540-559 72 76 76 105 114 114 F F F <---
83402024-02-01T17:31:03.967Z 28 560-579 74 74 74 108 108 108 F F F
83412024-02-01T17:31:03.967Z 29 580-599 69 76 76 99 112 112 F F F <---
83422024-02-01T17:31:03.967ZMax gen: 76, Max flush: 115
83432024-02-01T17:31:03.967ZError: Difference in extent metadata found!
83442024-02-01T17:31:03.967ZA Difference in extent metadata is expected here
83452024-02-01T17:31:03.968ZOn loop 38
83462024-02-01T17:31:03.968Z
83472024-02-01T17:31:03.971ZVerifying data now
83482024-02-01T17:31:09.067ZNo change after verify
83492024-02-01T17:31:09.067ZLoop: 38 Downstairs dump after verify (and repair):
83502024-02-01T17:31:09.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83512024-02-01T17:31:09.089Z 0 000-019 74 74 74 110 110 110 F F F
83522024-02-01T17:31:09.090Z 1 020-039 76 76 76 114 114 114 F F F
83532024-02-01T17:31:09.090Z 2 040-059 76 76 76 114 114 114 F F F
83542024-02-01T17:31:09.091Z 3 060-079 74 74 74 109 109 109 F F F
83552024-02-01T17:31:09.091Z 4 080-099 76 76 76 114 114 114 F F F
83562024-02-01T17:31:09.094Z 5 100-119 71 71 71 104 104 104 F F F
83572024-02-01T17:31:09.094Z 6 120-139 74 74 74 110 110 110 F F F
83582024-02-01T17:31:09.094Z 7 140-159 75 75 75 111 111 111 F F F
83592024-02-01T17:31:09.094Z 8 160-179 70 70 70 102 102 102 F F F
83602024-02-01T17:31:09.094Z 9 180-199 76 76 76 115 115 115 F F F
83612024-02-01T17:31:09.094Z 10 200-219 74 74 74 110 110 110 F F F
83622024-02-01T17:31:09.094Z 11 220-239 73 73 73 106 106 106 F F F
83632024-02-01T17:31:09.094Z 12 240-259 77 77 77 116 116 116 F F F
83642024-02-01T17:31:09.095Z 13 260-279 70 70 70 103 103 103 F F F
83652024-02-01T17:31:09.095Z 14 280-299 76 76 76 114 114 114 F F F
83662024-02-01T17:31:09.095Z 15 300-319 76 76 76 114 114 114 F F F
83672024-02-01T17:31:09.095Z 16 320-339 77 77 77 116 116 116 F F F
83682024-02-01T17:31:09.095Z 17 340-359 74 74 74 109 109 109 F F F
83692024-02-01T17:31:09.095Z 18 360-379 76 76 76 113 113 113 F F F
83702024-02-01T17:31:09.095Z 19 380-399 62 62 62 87 87 87 F F F
83712024-02-01T17:31:09.095Z 20 400-419 66 66 66 95 95 95 F F F
83722024-02-01T17:31:09.095Z 21 420-439 74 74 74 108 108 108 F F F
83732024-02-01T17:31:09.095Z 22 440-459 76 76 76 114 114 114 F F F
83742024-02-01T17:31:09.095Z 23 460-479 76 76 76 114 114 114 F F F
83752024-02-01T17:31:09.095Z 24 480-499 71 71 71 104 104 104 F F F
83762024-02-01T17:31:09.095Z 25 500-519 75 75 75 111 111 111 F F F
83772024-02-01T17:31:09.095Z 26 520-539 68 68 68 98 98 98 F F F
83782024-02-01T17:31:09.096Z 27 540-559 76 76 76 114 114 114 F F F
83792024-02-01T17:31:09.096Z 28 560-579 74 74 74 108 108 108 F F F
83802024-02-01T17:31:09.096Z 29 580-599 76 76 76 112 112 112 F F F
83812024-02-01T17:31:09.096ZMax gen: 77, Max flush: 116
83822024-02-01T17:31:09.097Z
83832024-02-01T17:31:09.097ZBegin loop 39
83842024-02-01T17:31:09.097ZDownstairs to restart: 2
83852024-02-01T17:31:09.125Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.394707518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
83862024-02-01T17:31:09.125Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
83872024-02-01T17:31:09.130Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395274481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
83882024-02-01T17:31:09.130Z{"msg":"Crucible ea23474b-3084-4ab8-bc6d-b58a821f1073 has session id: 7a366b39-3599-42b5-973c-469515f4131c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395341754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83892024-02-01T17:31:09.130Z{"msg":"Upstairs opts: Upstairs UUID: ea23474b-3084-4ab8-bc6d-b58a821f1073, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395473212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83902024-02-01T17:31:09.130Z{"msg":"Crucible stats registered with UUID: ea23474b-3084-4ab8-bc6d-b58a821f1073","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395590833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83912024-02-01T17:31:09.130ZCrucible runtime is spawned
83922024-02-01T17:31:09.130Z{"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.395811918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
83932024-02-01T17:31:09.131Z{"msg":"ea23474b-3084-4ab8-bc6d-b58a821f1073 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.39594291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83942024-02-01T17:31:09.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.39606732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83952024-02-01T17:31:09.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396100201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83962024-02-01T17:31:09.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396203141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83972024-02-01T17:31:09.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396408342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83982024-02-01T17:31:09.131Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396488635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
83992024-02-01T17:31:09.131Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396558496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84002024-02-01T17:31:09.131Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.39660321Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84012024-02-01T17:31:09.131Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396650269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84022024-02-01T17:31:09.131Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396724888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84032024-02-01T17:31:09.132Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396768328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84042024-02-01T17:31:09.132Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396844318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84052024-02-01T17:31:09.132Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.396921069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"io task","client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84062024-02-01T17:31:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398164641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84072024-02-01T17:31:09.132Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398237919Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84082024-02-01T17:31:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398264555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84092024-02-01T17:31:09.132Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398287224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84102024-02-01T17:31:09.132Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398318568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84112024-02-01T17:31:09.132Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.398360192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84122024-02-01T17:31:09.138Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.399480565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84132024-02-01T17:31:09.138Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.399626259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84142024-02-01T17:31:09.138Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.399800508Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84152024-02-01T17:31:09.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400440168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84162024-02-01T17:31:09.138Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400484262Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84172024-02-01T17:31:09.138Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.40050947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84182024-02-01T17:31:09.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400603707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84192024-02-01T17:31:09.138Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400708816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84202024-02-01T17:31:09.138Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400808174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84212024-02-01T17:31:09.138Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400869655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84222024-02-01T17:31:09.138Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.400931276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84232024-02-01T17:31:09.138Z{"msg":"[0]R flush_numbers[0..12]: [110, 114, 114, 109, 114, 104, 110, 111, 102, 115, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401006442Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84242024-02-01T17:31:09.138Z{"msg":"[0]R generation[0..12]: [74, 76, 76, 74, 76, 71, 74, 75, 70, 76, 74, 73]","v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO
84252024-02-01T17:31:09.138Z":30,"time":"2024-02-01T17:31:08.401095745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84262024-02-01T17:31:09.138Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401159462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84272024-02-01T17:31:09.138Z{"msg":"[1]R flush_numbers[0..12]: [110, 114, 114, 109, 114, 104, 110, 111, 102, 115, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401192389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84282024-02-01T17:31:09.139Z{"msg":"[1]R generation[0..12]: [74, 76, 76, 74, 76, 71, 74, 75, 70, 76, 74, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401251546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84292024-02-01T17:31:09.139Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401285772Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84302024-02-01T17:31:09.139Z{"msg":"[2]R flush_numbers[0..12]: [110, 114, 114, 109, 114, 104, 110, 111, 102, 115, 110, 106]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401370145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84312024-02-01T17:31:09.139Z{"msg":"[2]R generation[0..12]: [74, 76, 76, 74, 76, 71, 74, 75, 70, 76, 74, 73]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401409596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84322024-02-01T17:31:09.139Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.40150081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84332024-02-01T17:31:09.139Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401545947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84342024-02-01T17:31:09.139Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401682548Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84352024-02-01T17:31:09.140Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401727281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84362024-02-01T17:31:09.140Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401796395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"":"downstairs","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84372024-02-01T17:31:09.140Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401838837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84382024-02-01T17:31:09.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401906439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"0","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84392024-02-01T17:31:09.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.401982886Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"1","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84402024-02-01T17:31:09.140Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.402030745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"client":"2","":"downstairs","session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84412024-02-01T17:31:09.140Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84422024-02-01T17:31:09.140Z,"level":30,"time":"2024-02-01T17:31:08.402139748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84432024-02-01T17:31:09.140Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.40219395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84442024-02-01T17:31:09.140Z{"msg":"ea23474b-3084-4ab8-bc6d-b58a821f1073 is now active with session: 7a366b39-3599-42b5-973c-469515f4131c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.40222016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84452024-02-01T17:31:09.140Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.402259683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441,"session_id":"7a366b39-3599-42b5-973c-469515f4131c"}
84462024-02-01T17:31:09.140Z{"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:08.402282377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
84472024-02-01T17:31:09.140ZLoading write count information from file "/tmp/test_repair_verify.data"
84482024-02-01T17:31:09.140ZRun Repair workload
84492024-02-01T17:31:09.140Z01/30 Flush
84502024-02-01T17:31:09.140Z02/30 Write block 318 len 16384 data: 7 7 8 7
84512024-02-01T17:31:09.144Z03/30 Write block 590 len 16384 data: 8 7 5 5
84522024-02-01T17:31:09.144Z04/30 Read block 66 len 20480
84532024-02-01T17:31:09.144Z05/30 Flush
84542024-02-01T17:31:09.144Z06/30 Write block 573 len 40960 data: 3 3 5 5 5 5 5 5 5 6
84552024-02-01T17:31:09.144Z07/30 Read block 132 len 12288
84562024-02-01T17:31:09.147Z08/30 Read block 464 len 40960
84572024-02-01T17:31:09.150Z09/30 Write block 291 len 32768 data: 7 7 8 8 9 9 8 11
84582024-02-01T17:31:09.150Z10/30 Read block 360 len 12288
84592024-02-01T17:31:09.154Z11/30 Read block 96 len 12288
84602024-02-01T17:31:09.154Z12/30 Read block 345 len 28672
84612024-02-01T17:31:09.158Z13/30 Read block 518 len 32768
84622024-02-01T17:31:09.158Z14/30 Write block 576 len 12288 data: 6 6 6
84632024-02-01T17:31:09.158Z15/30 Write block 483 len 12288 data: 8 8 7
84642024-02-01T17:31:09.158Z16/30 Read block 463 len 12288
84652024-02-01T17:31:09.161Z17/30 Write block 366 len 32768 data: 8 7 5 4 5 5 5 6
84662024-02-01T17:31:09.161Z18/30 Write block 218 len 32768 data: 6 7 7 6 5 4 4 3
84672024-02-01T17:31:09.164Z19/30 Read block 539 len 36864
84682024-02-01T17:31:09.167Z20/30 Write block 456 len 8192 data: 6 8
84692024-02-01T17:31:09.167Z21/30 Write block 490 len 12288 data: 9 9 10
84702024-02-01T17:31:09.167Z22/30 Write block 25 len 8192 data: 8 6
84712024-02-01T17:31:09.167Z23/30 Read block 318 len 4096
84722024-02-01T17:31:09.171Z24/30 Read block 331 len 8192
84732024-02-01T17:31:09.171Z25/30 Write block 534 len 8192 data: 11 10
84742024-02-01T17:31:09.171Z26/30 Write block 147 len 20480 data: 5 8 8 6 6
84752024-02-01T17:31:09.175Z27/30 Write block 211 len 20480 data: 8 4 4 5 5
84762024-02-01T17:31:09.175Z28/30 Read block 26 len 40960
84772024-02-01T17:31:09.175Z29/30 Write block 280 len 4096 data: 6
84782024-02-01T17:31:09.179Z30/30 Write block 424 len 32768 data: 7 8 10 6 5 4 5 5
84792024-02-01T17:31:09.179Z----------------------------------------------------------------
84802024-02-01T17:31:09.179Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
84812024-02-01T17:31:09.179ZStates: Active Active Active
84822024-02-01T17:31:09.179ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84832024-02-01T17:31:09.179Z 1 Acked 1000 Flush 0 Done Done Sent false
84842024-02-01T17:31:09.179Z 2 Acked 1001 Write 4 Done Done Sent false
84852024-02-01T17:31:09.179Z 3 Acked 1002 Write 4 Done Done Sent false
84862024-02-01T17:31:09.179Z 4 Acked 1003 Read 5 Done Done Sent false
84872024-02-01T17:31:09.179Z 5 Acked 1004 Flush 0 Done Done Sent false
84882024-02-01T17:31:09.179Z 6 Acked 1005 Write 10 Done Done Sent false
84892024-02-01T17:31:09.180Z 7 Acked 1006 Read 3 Done Done Sent false
84902024-02-01T17:31:09.180Z 8 Acked 1007 Read 10 Done Done Sent false
84912024-02-01T17:31:09.180Z 9 Acked 1008 Write 8 Done Done Sent false
84922024-02-01T17:31:09.180Z 10 Acked 1009 Read 3 Done Done Sent false
84932024-02-01T17:31:09.180Z 11 Acked 1010 Read 3 Done Done Sent false
84942024-02-01T17:31:09.180Z 12 Acked 1011 Read 7 Done Done Sent false
84952024-02-01T17:31:09.180Z 13 Acked 1012 Read 8 Done Done Sent false
84962024-02-01T17:31:09.180Z 14 Acked 1013 Write 3 Done Done Sent false
84972024-02-01T17:31:09.180Z 15 Acked 1014 Write 3 Done Done Sent false
84982024-02-01T17:31:09.180Z 16 Acked 1015 Read 3 Done Done Sent false
84992024-02-01T17:31:09.180Z 17 Acked 1016 Write 8 Done Done Sent false
85002024-02-01T17:31:09.180Z 18 Acked 1017 Write 8 Done Done Sent false
85012024-02-01T17:31:09.180Z 19 Acked 1018 Read 9 Done Done Sent false
85022024-02-01T17:31:09.180Z 20 Acked 1019 Write 2 Done Done Sent false
85032024-02-01T17:31:09.180Z 21 Acked 1020 Write 3 Done Done Sent false
85042024-02-01T17:31:09.180Z 22 Acked 1021 Write 2 Done Done Sent false
85052024-02-01T17:31:09.180Z 23 Acked 1022 Read 1 Done Done Sent false
85062024-02-01T17:31:09.180Z 24 Acked 1023 Read 2 Done Done Sent false
85072024-02-01T17:31:09.180Z 25 Acked 1024 Write 2 Done Done Sent false
85082024-02-01T17:31:09.180Z 26 Acked 1025 Write 5 Done Done Sent false
85092024-02-01T17:31:09.180Z 27 Acked 1026 Write 5 Done Done Sent false
85102024-02-01T17:31:09.180Z 28 Acked 1027 Read 10 Done Done Sent false
85112024-02-01T17:31:09.180Z 29 Acked 1028 Write 1 Done Done Sent false
85122024-02-01T17:31:09.181Z 30 Acked 1029 Write 8 Sent Sent Sent false
85132024-02-01T17:31:09.181Z STATES DS:0 DS:1 DS:2 TOTAL
85142024-02-01T17:31:09.181Z New 0 0 0 0
85152024-02-01T17:31:09.181Z Sent 1 1 30 32
85162024-02-01T17:31:09.181Z Done 29 29 0 58
85172024-02-01T17:31:09.181Z Skipped 0 0 0 0
85182024-02-01T17:31:09.181Z Error 0 0 0 0
85192024-02-01T17:31:09.181ZLast Flush: 1004 1004 0
85202024-02-01T17:31:09.181ZDownstairs last five completed:
85212024-02-01T17:31:09.181ZUpstairs last five completed: 30 29 28 27 26
85222024-02-01T17:31:09.181Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:08.447556711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1441}
85232024-02-01T17:31:09.181ZWrote out file "/tmp/test_repair_verify.data"
85242024-02-01T17:31:09.185Z
85252024-02-01T17:31:09.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85262024-02-01T17:31:09.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85272024-02-01T17:31:09.217Z 0 000-019 74 74 74 110 110 110 F F F
85282024-02-01T17:31:09.217Z 1 020-039 76 76 76 114 114 114 T T F <---
85292024-02-01T17:31:09.217Z 2 040-059 76 76 76 114 114 114 F F F
85302024-02-01T17:31:09.221Z 3 060-079 74 74 74 109 109 109 F F F
85312024-02-01T17:31:09.221Z 4 080-099 76 76 76 114 114 114 F F F
85322024-02-01T17:31:09.221Z 5 100-119 71 71 71 104 104 104 F F F
85332024-02-01T17:31:09.221Z 6 120-139 74 74 74 110 110 110 F F F
85342024-02-01T17:31:09.221Z 7 140-159 75 75 75 111 111 111 T T F <---
85352024-02-01T17:31:09.221Z 8 160-179 70 70 70 102 102 102 F F F
85362024-02-01T17:31:09.221Z 9 180-199 76 76 76 115 115 115 F F F
85372024-02-01T17:31:09.221Z 10 200-219 74 74 74 110 110 110 T T F <---
85382024-02-01T17:31:09.221Z 11 220-239 73 73 73 106 106 106 T T F <---
85392024-02-01T17:31:09.221Z 12 240-259 77 77 77 116 116 116 F F F
85402024-02-01T17:31:09.221Z 13 260-279 70 70 70 103 103 103 F F F
85412024-02-01T17:31:09.221Z 14 280-299 76 76 76 114 114 114 T T F <---
85422024-02-01T17:31:09.221Z 15 300-319 78 78 76 118 118 114 F F F <---
85432024-02-01T17:31:09.221Z 16 320-339 78 78 77 118 118 116 F F F <---
85442024-02-01T17:31:09.221Z 17 340-359 74 74 74 109 109 109 F F F
85452024-02-01T17:31:09.221Z 18 360-379 76 76 76 113 113 113 T T F <---
85462024-02-01T17:31:09.221Z 19 380-399 62 62 62 87 87 87 F F F
85472024-02-01T17:31:09.222Z 20 400-419 66 66 66 95 95 95 F F F
85482024-02-01T17:31:09.222Z 21 420-439 74 74 74 108 108 108 T T F <---
85492024-02-01T17:31:09.222Z 22 440-459 76 76 76 114 114 114 T T F <---
85502024-02-01T17:31:09.222Z 23 460-479 76 76 76 114 114 114 F F F
85512024-02-01T17:31:09.222Z 24 480-499 71 71 71 104 104 104 T T F <---
85522024-02-01T17:31:09.222Z 25 500-519 75 75 75 111 111 111 F F F
85532024-02-01T17:31:09.222Z 26 520-539 68 68 68 98 98 98 T T F <---
85542024-02-01T17:31:09.222Z 27 540-559 76 76 76 114 114 114 F F F
85552024-02-01T17:31:09.222Z 28 560-579 74 74 74 108 108 108 T T F <---
85562024-02-01T17:31:09.222Z 29 580-599 78 78 76 118 118 112 T T F <---
85572024-02-01T17:31:09.222ZMax gen: 78, Max flush: 118
85582024-02-01T17:31:09.222ZError: Difference in extent metadata found!
85592024-02-01T17:31:09.226ZA Difference in extent metadata is expected here
85602024-02-01T17:31:09.226ZOn loop 39
85612024-02-01T17:31:09.226Z
85622024-02-01T17:31:09.226ZVerifying data now
85632024-02-01T17:31:14.412ZNo change after verify
85642024-02-01T17:31:14.413ZLoop: 39 Downstairs dump after verify (and repair):
85652024-02-01T17:31:14.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85662024-02-01T17:31:14.434Z 0 000-019 74 74 74 110 110 110 F F F
85672024-02-01T17:31:14.434Z 1 020-039 79 79 79 119 119 119 F F F
85682024-02-01T17:31:14.434Z 2 040-059 76 76 76 114 114 114 F F F
85692024-02-01T17:31:14.438Z 3 060-079 74 74 74 109 109 109 F F F
85702024-02-01T17:31:14.441Z 4 080-099 76 76 76 114 114 114 F F F
85712024-02-01T17:31:14.441Z 5 100-119 71 71 71 104 104 104 F F F
85722024-02-01T17:31:14.441Z 6 120-139 74 74 74 110 110 110 F F F
85732024-02-01T17:31:14.441Z 7 140-159 79 79 79 119 119 119 F F F
85742024-02-01T17:31:14.441Z 8 160-179 70 70 70 102 102 102 F F F
85752024-02-01T17:31:14.441Z 9 180-199 76 76 76 115 115 115 F F F
85762024-02-01T17:31:14.442Z 10 200-219 79 79 79 119 119 119 F F F
85772024-02-01T17:31:14.442Z 11 220-239 79 79 79 119 119 119 F F F
85782024-02-01T17:31:14.442Z 12 240-259 77 77 77 116 116 116 F F F
85792024-02-01T17:31:14.442Z 13 260-279 70 70 70 103 103 103 F F F
85802024-02-01T17:31:14.442Z 14 280-299 79 79 79 119 119 119 F F F
85812024-02-01T17:31:14.442Z 15 300-319 78 78 78 118 118 118 F F F
85822024-02-01T17:31:14.442Z 16 320-339 78 78 78 118 118 118 F F F
85832024-02-01T17:31:14.442Z 17 340-359 74 74 74 109 109 109 F F F
85842024-02-01T17:31:14.442Z 18 360-379 79 79 79 119 119 119 F F F
85852024-02-01T17:31:14.442Z 19 380-399 62 62 62 87 87 87 F F F
85862024-02-01T17:31:14.442Z 20 400-419 66 66 66 95 95 95 F F F
85872024-02-01T17:31:14.442Z 21 420-439 79 79 79 119 119 119 F F F
85882024-02-01T17:31:14.443Z 22 440-459 79 79 79 119 119 119 F F F
85892024-02-01T17:31:14.443Z 23 460-479 76 76 76 114 114 114 F F F
85902024-02-01T17:31:14.443Z 24 480-499 79 79 79 119 119 119 F F F
85912024-02-01T17:31:14.443Z 25 500-519 75 75 75 111 111 111 F F F
85922024-02-01T17:31:14.443Z 26 520-539 79 79 79 119 119 119 F F F
85932024-02-01T17:31:14.443Z 27 540-559 76 76 76 114 114 114 F F F
85942024-02-01T17:31:14.443Z 28 560-579 79 79 79 119 119 119 F F F
85952024-02-01T17:31:14.443Z 29 580-599 79 79 79 119 119 119 F F F
85962024-02-01T17:31:14.443ZMax gen: 79, Max flush: 119
85972024-02-01T17:31:14.443Z
85982024-02-01T17:31:14.443ZBegin loop 40
85992024-02-01T17:31:14.443ZDownstairs to restart: 0
86002024-02-01T17:31:14.468Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.737753983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
86012024-02-01T17:31:14.468Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738244588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
86022024-02-01T17:31:14.480Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
86032024-02-01T17:31:14.480Z,"time":"2024-02-01T17:31:13.738286933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
86042024-02-01T17:31:14.480Z{"msg":"Crucible 3431745c-de06-4d35-a375-4dde68855fdf has session id: ef191fcb-28bf-4ae5-89a5-7c10c301baaa","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738378118Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86052024-02-01T17:31:14.480Z{"msg":"Upstairs opts: Upstairs UUID: 3431745c-de06-4d35-a375-4dde68855fdf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738509337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86062024-02-01T17:31:14.480Z{"msg":"Crucible stats registered with UUID: 3431745c-de06-4d35-a375-4dde68855fdf","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738556687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86072024-02-01T17:31:14.480Z{"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.73860962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
86082024-02-01T17:31:14.480Z{"msg":"3431745c-de06-4d35-a375-4dde68855fdf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738655669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86092024-02-01T17:31:14.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738717677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86102024-02-01T17:31:14.480Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738769393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86112024-02-01T17:31:14.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738835932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86122024-02-01T17:31:14.480Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738880957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86132024-02-01T17:31:14.480Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.738947069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86142024-02-01T17:31:14.480Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739019775Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86152024-02-01T17:31:14.480Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739204168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86162024-02-01T17:31:14.480Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.73930129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86172024-02-01T17:31:14.480Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739349325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86182024-02-01T17:31:14.481Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739422863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86192024-02-01T17:31:14.481Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739499342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86202024-02-01T17:31:14.481Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.739546116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"io task","client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86212024-02-01T17:31:14.481Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.740869901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86222024-02-01T17:31:14.481Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.740909607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86232024-02-01T17:31:14.481Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.740933846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86242024-02-01T17:31:14.481Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.740956209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86252024-02-01T17:31:14.481Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.740978291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86262024-02-01T17:31:14.481Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.741016633Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86272024-02-01T17:31:14.481Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74182913Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86282024-02-01T17:31:14.481Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.742198042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86292024-02-01T17:31:14.481Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.742614386Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86302024-02-01T17:31:14.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.742847844Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86312024-02-01T17:31:14.481Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74291107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86322024-02-01T17:31:14.481Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74295449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86332024-02-01T17:31:14.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743154725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86342024-02-01T17:31:14.481Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743215943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86352024-02-01T17:31:14.481Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74324261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86362024-02-01T17:31:14.481Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743473025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86372024-02-01T17:31:14.481Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743530799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86382024-02-01T17:31:14.481Z{"msg":"[0]R flush_numbers[0..12]: [110, 119, 114, 109, 114, 104, 110, 119, 102, 115, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74355784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86392024-02-01T17:31:14.481Z{"msg":"[0]R generation[0..12]: [74, 79, 76, 74, 76, 71, 74, 79, 70, 76, 79, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743582486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86402024-02-01T17:31:14.481Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743606964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86412024-02-01T17:31:14.482Z{"msg":"[1]R flush_numbers[0..12]: [110, 119, 114, 109, 114, 104, 110, 119, 102, 115, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743648904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86422024-02-01T17:31:14.482Z{"msg":"[1]R generation[0..12]: [74, 79, 76, 74, 76, 71, 74, 79, 70, 76, 79, 79]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743674051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86432024-02-01T17:31:14.482Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.74371611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86442024-02-01T17:31:14.482Z{"msg":"[2]R flush_numbers[0..12]: [110, 119, 114, 109, 114, 104, 110, 119, 102, 115, 119, 119]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743741706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86452024-02-01T17:31:14.482Z{"msg":"[2]R generation[0..12]: [74, 79, 76, 74, 76, 71, 74, 79, 70, 76, 79, 79]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
86462024-02-01T17:31:14.482Z2024-02-01T17:31:13.743783634Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86472024-02-01T17:31:14.482Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743813552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86482024-02-01T17:31:14.482Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743857253Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86492024-02-01T17:31:14.482Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743912517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86502024-02-01T17:31:14.482Z{"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743936217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86512024-02-01T17:31:14.482Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.743976276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"":"downstairs","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86522024-02-01T17:31:14.483Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744000492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86532024-02-01T17:31:14.483Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744039419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"0","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86542024-02-01T17:31:14.483Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744063203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"1","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86552024-02-01T17:31:14.483Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744116865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"client":"2","":"downstairs","session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86562024-02-01T17:31:14.483Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744142269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86572024-02-01T17:31:14.483Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744170887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86582024-02-01T17:31:14.483Z{"msg":"3431745c-de06-4d35-a375-4dde68855fdf is now active with session: ef191fcb-28bf-4ae5-89a5-7c10c301baaa","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744213437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86592024-02-01T17:31:14.483Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744235466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449,"session_id":"ef191fcb-28bf-4ae5-89a5-7c10c301baaa"}
86602024-02-01T17:31:14.483Z{"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:13.744275846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
86612024-02-01T17:31:14.483ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86622024-02-01T17:31:14.483ZLoading write count information from file "/tmp/test_repair_verify.data"
86632024-02-01T17:31:14.483ZRun Repair workload
86642024-02-01T17:31:14.483Z01/30 Flush
86652024-02-01T17:31:14.483Z02/30 Write block 213 len 36864 data: 5 6 6 3 3 7 8 8 7
86662024-02-01T17:31:14.489Z03/30 Write block 276 len 32768 data: 9 7 9 6 7 5 4 5
86672024-02-01T17:31:14.490Z04/30 Write block 584 len 8192 data: 5 7
86682024-02-01T17:31:14.490Z05/30 Write block 203 len 12288 data: 10 10 9
86692024-02-01T17:31:14.490Z06/30 Write block 83 len 20480 data: 4 3 3 4 3
86702024-02-01T17:31:14.490Z07/30 Read block 199 len 24576
86712024-02-01T17:31:14.490Z08/30 Read block 15 len 32768
86722024-02-01T17:31:14.490Z09/30 Read block 396 len 36864
86732024-02-01T17:31:14.490Z10/30 Read block 41 len 32768
86742024-02-01T17:31:14.496Z11/30 Write block 570 len 12288 data: 5 3 2
86752024-02-01T17:31:14.496Z12/30 Read block 197 len 28672
86762024-02-01T17:31:14.496Z13/30 Write block 218 len 24576 data: 8 9 9 8 6 5
86772024-02-01T17:31:14.499Z14/30 Write block 18 len 12288 data: 7 6 7
86782024-02-01T17:31:14.499Z15/30 Read block 219 len 4096
86792024-02-01T17:31:14.499Z16/30 Write block 454 len 40960 data: 4 4 7 9 8 10 9 8 7 6
86802024-02-01T17:31:14.499Z17/30 Write block 65 len 40960 data: 6 7 7 7 9 7 9 10 10 10
86812024-02-01T17:31:14.499Z18/30 Read block 188 len 8192
86822024-02-01T17:31:14.503Z19/30 Read block 479 len 24576
86832024-02-01T17:31:14.503Z20/30 Read block 474 len 36864
86842024-02-01T17:31:14.506Z21/30 Read block 322 len 32768
86852024-02-01T17:31:14.506Z22/30 Flush
86862024-02-01T17:31:14.511Z23/30 Write block 400 len 36864 data: 2 4 4 6 5 5 5 6 7
86872024-02-01T17:31:14.514Z24/30 Read block 502 len 20480
86882024-02-01T17:31:14.517Z25/30 Read block 15 len 8192
86892024-02-01T17:31:14.520Z26/30 Write block 30 len 32768 data: 5 5 6 5 6 7 8 9
86902024-02-01T17:31:14.520Z27/30 Read block 376 len 28672
86912024-02-01T17:31:14.520Z28/30 Write block 370 len 16384 data: 6 6 6 7
86922024-02-01T17:31:14.520Z29/30 Write block 120 len 4096 data: 4
86932024-02-01T17:31:14.520Z30/30 Write block 518 len 24576 data: 4 4 5 5 5 7
86942024-02-01T17:31:14.524Z----------------------------------------------------------------
86952024-02-01T17:31:14.524Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:29
86962024-02-01T17:31:14.524ZStates: Active Active Active
86972024-02-01T17:31:14.524ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86982024-02-01T17:31:14.524Z 2 Acked 1001 Write 9 Sent Done Done false
86992024-02-01T17:31:14.524Z 3 Acked 1002 Write 8 Sent Done Done false
87002024-02-01T17:31:14.524Z 4 Acked 1003 Write 2 Sent Done Done false
87012024-02-01T17:31:14.525Z 5 Acked 1004 Write 3 Sent Done Done false
87022024-02-01T17:31:14.525Z 6 Acked 1005 Write 5 Sent Done Done false
87032024-02-01T17:31:14.525Z 7 Acked 1006 Read 6 Sent Done Done false
87042024-02-01T17:31:14.525Z 8 Acked 1007 Read 8 Sent Done Done false
87052024-02-01T17:31:14.525Z 9 Acked 1008 Read 9 Sent Done Done false
87062024-02-01T17:31:14.525Z 10 Acked 1009 Read 8 Sent Done Done false
87072024-02-01T17:31:14.525Z 11 Acked 1010 Write 3 Sent Done Done false
87082024-02-01T17:31:14.525Z 12 Acked 1011 Read 7 Sent Done Done false
87092024-02-01T17:31:14.525Z 13 Acked 1012 Write 6 Sent Done Done false
87102024-02-01T17:31:14.525Z 14 Acked 1013 Write 3 Sent Done Done false
87112024-02-01T17:31:14.525Z 15 Acked 1014 Read 1 Sent Done Done false
87122024-02-01T17:31:14.525Z 16 Acked 1015 Write 10 Sent Done Done false
87132024-02-01T17:31:14.525Z 17 Acked 1016 Write 10 Sent Done Done false
87142024-02-01T17:31:14.525Z 18 Acked 1017 Read 2 Sent Done Done false
87152024-02-01T17:31:14.525Z 19 Acked 1018 Read 6 Sent Done Done false
87162024-02-01T17:31:14.525Z 20 Acked 1019 Read 9 Sent Done Done false
87172024-02-01T17:31:14.525Z 21 Acked 1020 Read 8 Sent Done Done false
87182024-02-01T17:31:14.525Z 22 Acked 1021 Flush 0 Sent Done Done false
87192024-02-01T17:31:14.525Z 23 Acked 1022 Write 9 Sent Done Done false
87202024-02-01T17:31:14.525Z 24 Acked 1023 Read 5 Sent Done Done false
87212024-02-01T17:31:14.525Z 25 Acked 1024 Read 2 Sent Done Done false
87222024-02-01T17:31:14.526Z 26 Acked 1025 Write 8 Sent Done Done false
87232024-02-01T17:31:14.526Z 27 Acked 1026 Read 7 Sent Done Done false
87242024-02-01T17:31:14.526Z 28 Acked 1027 Write 4 Sent Done Done false
87252024-02-01T17:31:14.526Z 29 Acked 1028 Write 1 Sent Done Done false
87262024-02-01T17:31:14.526Z 30 Acked 1029 Write 6 Sent Sent Sent false
87272024-02-01T17:31:14.526Z STATES DS:0 DS:1 DS:2 TOTAL
87282024-02-01T17:31:14.526Z New 0 0 0 0
87292024-02-01T17:31:14.526Z Sent 29 1 1 31
87302024-02-01T17:31:14.526Z Done 0 28 28 56
87312024-02-01T17:31:14.526Z Skipped 0 0 0 0
87322024-02-01T17:31:14.526Z Error 0 0 0 0
87332024-02-01T17:31:14.526ZLast Flush: 1000 1021 1021
87342024-02-01T17:31:14.526ZDownstairs last five completed: 1000
87352024-02-01T17:31:14.526ZUpstairs last five completed: 30 29 28 27 26
87362024-02-01T17:31:14.526Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:13.791542316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1449}
87372024-02-01T17:31:14.526ZWrote out file "/tmp/test_repair_verify.data"
87382024-02-01T17:31:14.532Z
87392024-02-01T17:31:14.536ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87402024-02-01T17:31:14.560ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87412024-02-01T17:31:14.560Z 0 000-019 74 80 80 110 121 121 F F F <---
87422024-02-01T17:31:14.560Z 1 020-039 79 80 80 119 121 121 F T T <---
87432024-02-01T17:31:14.560Z 2 040-059 76 76 76 114 114 114 F F F
87442024-02-01T17:31:14.564Z 3 060-079 74 80 80 109 121 121 F F F <---
87452024-02-01T17:31:14.564Z 4 080-099 76 80 80 114 121 121 F F F <---
87462024-02-01T17:31:14.564Z 5 100-119 71 71 71 104 104 104 F F F
87472024-02-01T17:31:14.564Z 6 120-139 74 74 74 110 110 110 F T T <---
87482024-02-01T17:31:14.564Z 7 140-159 79 79 79 119 119 119 F F F
87492024-02-01T17:31:14.564Z 8 160-179 70 70 70 102 102 102 F F F
87502024-02-01T17:31:14.565Z 9 180-199 76 76 76 115 115 115 F F F
87512024-02-01T17:31:14.565Z 10 200-219 79 80 80 119 121 121 F F F <---
87522024-02-01T17:31:14.565Z 11 220-239 79 80 80 119 121 121 F F F <---
87532024-02-01T17:31:14.565Z 12 240-259 77 77 77 116 116 116 F F F
87542024-02-01T17:31:14.565Z 13 260-279 70 80 80 103 121 121 F F F <---
87552024-02-01T17:31:14.565Z 14 280-299 79 80 80 119 121 121 F F F <---
87562024-02-01T17:31:14.565Z 15 300-319 78 78 78 118 118 118 F F F
87572024-02-01T17:31:14.565Z 16 320-339 78 78 78 118 118 118 F F F
87582024-02-01T17:31:14.565Z 17 340-359 74 74 74 109 109 109 F F F
87592024-02-01T17:31:14.565Z 18 360-379 79 79 79 119 119 119 F T T <---
87602024-02-01T17:31:14.565Z 19 380-399 62 62 62 87 87 87 F F F
87612024-02-01T17:31:14.566Z 20 400-419 66 66 66 95 95 95 F T T <---
87622024-02-01T17:31:14.566Z 21 420-439 79 79 79 119 119 119 F F F
87632024-02-01T17:31:14.566Z 22 440-459 79 80 80 119 121 121 F F F <---
87642024-02-01T17:31:14.567Z 23 460-479 76 80 80 114 121 121 F F F <---
87652024-02-01T17:31:14.567Z 24 480-499 79 79 79 119 119 119 F F F
87662024-02-01T17:31:14.567Z 25 500-519 75 75 75 111 111 111 F T T <---
87672024-02-01T17:31:14.567Z 26 520-539 79 79 79 119 119 119 F T T <---
87682024-02-01T17:31:14.567Z 27 540-559 76 76 76 114 114 114 F F F
87692024-02-01T17:31:14.567Z 28 560-579 79 80 80 119 121 121 F F F <---
87702024-02-01T17:31:14.567Z 29 580-599 79 80 80 119 121 121 F F F <---
87712024-02-01T17:31:14.567ZMax gen: 80, Max flush: 121
87722024-02-01T17:31:14.567ZError: Difference in extent metadata found!
87732024-02-01T17:31:14.986ZA Difference in extent metadata is expected here
87742024-02-01T17:31:14.986ZOn loop 40
87752024-02-01T17:31:14.986Z
87762024-02-01T17:31:14.986ZVerifying data now
87772024-02-01T17:31:19.965ZNo change after verify
87782024-02-01T17:31:19.965ZLoop: 40 Downstairs dump after verify (and repair):
87792024-02-01T17:31:19.987ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87802024-02-01T17:31:19.987Z 0 000-019 80 80 80 121 121 121 F F F
87812024-02-01T17:31:19.987Z 1 020-039 81 81 81 122 122 122 F F F
87822024-02-01T17:31:19.987Z 2 040-059 76 76 76 114 114 114 F F F
87832024-02-01T17:31:19.987Z 3 060-079 80 80 80 121 121 121 F F F
87842024-02-01T17:31:19.987Z 4 080-099 80 80 80 121 121 121 F F F
87852024-02-01T17:31:19.987Z 5 100-119 71 71 71 104 104 104 F F F
87862024-02-01T17:31:19.987Z 6 120-139 81 81 81 122 122 122 F F F
87872024-02-01T17:31:19.987Z 7 140-159 79 79 79 119 119 119 F F F
87882024-02-01T17:31:19.987Z 8 160-179 70 70 70 102 102 102 F F F
87892024-02-01T17:31:19.991Z 9 180-199 76 76 76 115 115 115 F F F
87902024-02-01T17:31:19.991Z 10 200-219 80 80 80 121 121 121 F F F
87912024-02-01T17:31:19.991Z 11 220-239 80 80 80 121 121 121 F F F
87922024-02-01T17:31:19.991Z 12 240-259 77 77 77 116 116 116 F F F
87932024-02-01T17:31:19.991Z 13 260-279 80 80 80 121 121 121 F F F
87942024-02-01T17:31:19.991Z 14 280-299 80 80 80 121 121 121 F F F
87952024-02-01T17:31:19.991Z 15 300-319 78 78 78 118 118 118 F F F
87962024-02-01T17:31:19.991Z 16 320-339 78 78 78 118 118 118 F F F
87972024-02-01T17:31:19.991Z 17 340-359 74 74 74 109 109 109 F F F
87982024-02-01T17:31:19.991Z 18 360-379 81 81 81 122 122 122 F F F
87992024-02-01T17:31:19.991Z 19 380-399 62 62 62 87 87 87 F F F
88002024-02-01T17:31:19.991Z 20 400-419 81 81 81 122 122 122 F F F
88012024-02-01T17:31:19.991Z 21 420-439 79 79 79 119 119 119 F F F
88022024-02-01T17:31:19.991Z 22 440-459 80 80 80 121 121 121 F F F
88032024-02-01T17:31:19.991Z 23 460-479 80 80 80 121 121 121 F F F
88042024-02-01T17:31:19.991Z 24 480-499 79 79 79 119 119 119 F F F
88052024-02-01T17:31:19.991Z 25 500-519 81 81 81 122 122 122 F F F
88062024-02-01T17:31:19.991Z 26 520-539 81 81 81 122 122 122 F F F
88072024-02-01T17:31:19.991Z 27 540-559 76 76 76 114 114 114 F F F
88082024-02-01T17:31:19.991Z 28 560-579 80 80 80 121 121 121 F F F
88092024-02-01T17:31:19.991Z 29 580-599 80 80 80 121 121 121 F F F
88102024-02-01T17:31:19.992ZMax gen: 81, Max flush: 122
88112024-02-01T17:31:19.996Z
88122024-02-01T17:31:19.996ZBegin loop 41
88132024-02-01T17:31:19.996ZDownstairs to restart: 0
88142024-02-01T17:31:20.022Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.291581539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
88152024-02-01T17:31:20.029Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292089958Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
88162024-02-01T17:31:20.029Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"Crucible runtime is spawned
88172024-02-01T17:31:20.029Zname":"crucible","level":30,"time":"2024-02-01T17:31:19.292415167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
88182024-02-01T17:31:20.029Z{"msg":"Crucible 3181351b-388d-407d-b798-d4007a0aea79 has session id: c338e356-518e-4715-bd54-0533d3d509a1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292515738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88192024-02-01T17:31:20.029Z{"msg":"Upstairs opts: Upstairs UUID: 3181351b-388d-407d-b798-d4007a0aea79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.29259888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88202024-02-01T17:31:20.030Z{"msg":"Crucible stats registered with UUID: 3181351b-388d-407d-b798-d4007a0aea79","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292680333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88212024-02-01T17:31:20.030Z{"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292750882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
88222024-02-01T17:31:20.030Z{"msg":"3181351b-388d-407d-b798-d4007a0aea79 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292791715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88232024-02-01T17:31:20.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292866501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88242024-02-01T17:31:20.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292920013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88252024-02-01T17:31:20.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.292996485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88262024-02-01T17:31:20.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293100069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88272024-02-01T17:31:20.030Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293274017Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88282024-02-01T17:31:20.030Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293443138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88292024-02-01T17:31:20.030Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293539343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88302024-02-01T17:31:20.030Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293617804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88312024-02-01T17:31:20.030Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293811243Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88322024-02-01T17:31:20.030Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293886789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88332024-02-01T17:31:20.030Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.293985246Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88342024-02-01T17:31:20.030Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.294058729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"io task","client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88352024-02-01T17:31:20.030Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.294783931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88362024-02-01T17:31:20.030Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.294876911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88372024-02-01T17:31:20.031Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.295028031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88382024-02-01T17:31:20.031Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.295086386Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88392024-02-01T17:31:20.031Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.295130773Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88402024-02-01T17:31:20.031Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.295188605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88412024-02-01T17:31:20.031Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.295606601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88422024-02-01T17:31:20.036Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.296461974Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88432024-02-01T17:31:20.036Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.296816064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88442024-02-01T17:31:20.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.296878837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88452024-02-01T17:31:20.036Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.29695059Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88462024-02-01T17:31:20.036Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297021458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88472024-02-01T17:31:20.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297180432Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88482024-02-01T17:31:20.036Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.2972326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88492024-02-01T17:31:20.036Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297258332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88502024-02-01T17:31:20.036Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297610928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88512024-02-01T17:31:20.036Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297687211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88522024-02-01T17:31:20.036Z{"msg":"[0]R flush_numbers[0..12]: [121, 122, 114, 121, 121, 104, 122, 119, 102, 115, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297758875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88532024-02-01T17:31:20.036Z{"msg":"[0]R generation[0..12]: [80, 81, 76, 80, 80, 71, 81, 79, 70, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.29781981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88542024-02-01T17:31:20.037Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.297848333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88552024-02-01T17:31:20.038Z{"msg":"[1]R flush_numbers[0..12]: [121, 122, 114, 121, 121, 104, 122, 119, 102, 115, 121, 121]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88562024-02-01T17:31:20.038Z,"time":"2024-02-01T17:31:19.297940355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88572024-02-01T17:31:20.038Z{"msg":"[1]R generation[0..12]: [80, 81, 76, 80, 80, 71, 81, 79, 70, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298010147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88582024-02-01T17:31:20.038Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298076661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88592024-02-01T17:31:20.038Z{"msg":"[2]R flush_numbers[0..12]: [121, 122, 114, 121, 121, 104, 122, 119, 102, 115, 121, 121]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298147439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88602024-02-01T17:31:20.038Z{"msg":"[2]R generation[0..12]: [80, 81, 76, 80, 80, 71, 81, 79, 70, 76, 80, 80]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298174913Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88612024-02-01T17:31:20.038Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298242453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88622024-02-01T17:31:20.038Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298271997Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88632024-02-01T17:31:20.038Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298340284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88642024-02-01T17:31:20.038Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298402126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88652024-02-01T17:31:20.038Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298443119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"":"downstairs","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88662024-02-01T17:31:20.038Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298503578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88672024-02-01T17:31:20.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
88682024-02-01T17:31:20.038Z,"time":"2024-02-01T17:31:19.298572824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"0","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88692024-02-01T17:31:20.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298608825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"1","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88702024-02-01T17:31:20.038Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298668265Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"client":"2","":"downstairs","session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88712024-02-01T17:31:20.038Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298695627Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88722024-02-01T17:31:20.038Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298735587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88732024-02-01T17:31:20.039Z{"msg":"3181351b-388d-407d-b798-d4007a0aea79 is now active with session: c338e356-518e-4715-bd54-0533d3d509a1","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298758855Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88742024-02-01T17:31:20.039Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298778739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457,"session_id":"c338e356-518e-4715-bd54-0533d3d509a1"}
88752024-02-01T17:31:20.040Z{"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:19.298799288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
88762024-02-01T17:31:20.040ZLoading write count information from file "/tmp/test_repair_verify.data"
88772024-02-01T17:31:20.040ZRun Repair workload
88782024-02-01T17:31:20.040Z01/30 Write block 369 len 16384 data: 5 7 7 7
88792024-02-01T17:31:20.040Z02/30 Read block 362 len 32768
88802024-02-01T17:31:20.040Z03/30 Read block 264 len 40960
88812024-02-01T17:31:20.044Z04/30 Read block 192 len 16384
88822024-02-01T17:31:20.044Z05/30 Write block 15 len 16384 data: 4 5 7 8
88832024-02-01T17:31:20.044Z06/30 Read block 176 len 32768
88842024-02-01T17:31:20.044Z07/30 Write block 275 len 16384 data: 7 10 8 10
88852024-02-01T17:31:20.044Z08/30 Read block 213 len 20480
88862024-02-01T17:31:20.044Z09/30 Write block 458 len 20480 data: 9 11 10 9 8
88872024-02-01T17:31:20.044Z10/30 Read block 558 len 8192
88882024-02-01T17:31:20.044Z11/30 Read block 19 len 40960
88892024-02-01T17:31:20.047Z12/30 Write block 166 len 32768 data: 4 4 5 7 8 9 9 9
88902024-02-01T17:31:20.050Z13/30 Write block 430 len 36864 data: 6 6 6 7 8 9 8 9 7
88912024-02-01T17:31:20.050Z14/30 Write block 354 len 36864 data: 9 7 6 7 9 9 9 10 12
88922024-02-01T17:31:20.053Z15/30 Write block 494 len 24576 data: 6 8 8 6 7 6
88932024-02-01T17:31:20.053Z16/30 Write block 461 len 16384 data: 10 9 7 6
88942024-02-01T17:31:20.053Z17/30 Write block 282 len 12288 data: 5 6 6
88952024-02-01T17:31:20.056Z18/30 Write block 279 len 12288 data: 7 8 6
88962024-02-01T17:31:20.057Z19/30 Flush
88972024-02-01T17:31:20.059Z20/30 Write block 541 len 8192 data: 12 13
88982024-02-01T17:31:20.063Z21/30 Write block 275 len 12288 data: 8 11 9
88992024-02-01T17:31:20.063Z22/30 Write block 127 len 8192 data: 4 5
89002024-02-01T17:31:20.063Z23/30 Read block 507 len 16384
89012024-02-01T17:31:20.066Z24/30 Write block 233 len 28672 data: 4 5 4 5 4 5 6
89022024-02-01T17:31:20.066Z25/30 Write block 134 len 36864 data: 6 7 9 10 10 7 8 8 7
89032024-02-01T17:31:20.066Z26/30 Read block 513 len 20480
89042024-02-01T17:31:20.069Z27/30 Read block 549 len 40960
89052024-02-01T17:31:20.072Z28/30 Read block 60 len 20480
89062024-02-01T17:31:20.072Z29/30 Read block 214 len 40960
89072024-02-01T17:31:20.076Z30/30 Read block 389 len 40960
89082024-02-01T17:31:20.076Z----------------------------------------------------------------
89092024-02-01T17:31:20.076Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
89102024-02-01T17:31:20.076ZStates: Active Active Active
89112024-02-01T17:31:20.076ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89122024-02-01T17:31:20.076Z 1 Acked 1000 Write 4 Sent Done Done false
89132024-02-01T17:31:20.076Z 2 Acked 1001 Read 8 Sent Done Done false
89142024-02-01T17:31:20.076Z 3 Acked 1002 Read 10 Sent Done Done false
89152024-02-01T17:31:20.076Z 4 Acked 1003 Read 4 Sent Done Done false
89162024-02-01T17:31:20.076Z 5 Acked 1004 Write 4 Sent Done Done false
89172024-02-01T17:31:20.076Z 6 Acked 1005 Read 8 Sent Done Done false
89182024-02-01T17:31:20.076Z 7 Acked 1006 Write 4 Sent Done Done false
89192024-02-01T17:31:20.076Z 8 Acked 1007 Read 5 Sent Done Done false
89202024-02-01T17:31:20.076Z 9 Acked 1008 Write 5 Sent Done Done false
89212024-02-01T17:31:20.076Z 10 Acked 1009 Read 2 Sent Done Done false
89222024-02-01T17:31:20.077Z 11 Acked 1010 Read 10 Sent Done Done false
89232024-02-01T17:31:20.077Z 12 Acked 1011 Write 8 Sent Done Done false
89242024-02-01T17:31:20.077Z 13 Acked 1012 Write 9 Sent Done Done false
89252024-02-01T17:31:20.077Z 14 Acked 1013 Write 9 Sent Done Done false
89262024-02-01T17:31:20.077Z 15 Acked 1014 Write 6 Sent Done Done false
89272024-02-01T17:31:20.077Z 16 Acked 1015 Write 4 Sent Done Done false
89282024-02-01T17:31:20.077Z 17 Acked 1016 Write 3 Sent Done Done false
89292024-02-01T17:31:20.077Z 18 Acked 1017 Write 3 Sent Done Done false
89302024-02-01T17:31:20.077Z 19 Acked 1018 Flush 0 Sent Done Done false
89312024-02-01T17:31:20.077Z 20 Acked 1019 Write 2 Sent Done Done false
89322024-02-01T17:31:20.077Z 21 Acked 1020 Write 3 Sent Done Done false
89332024-02-01T17:31:20.077Z 22 Acked 1021 Write 2 Sent Done Done false
89342024-02-01T17:31:20.077Z 23 Acked 1022 Read 4 Sent Done Done false
89352024-02-01T17:31:20.077Z 24 Acked 1023 Write 7 Sent Done Done false
89362024-02-01T17:31:20.077Z 25 Acked 1024 Write 9 Sent Done Done false
89372024-02-01T17:31:20.077Z 26 Acked 1025 Read 5 Sent Done Done false
89382024-02-01T17:31:20.077Z 27 Acked 1026 Read 10 Sent Done Done false
89392024-02-01T17:31:20.077Z 28 Acked 1027 Read 5 Sent Done Done false
89402024-02-01T17:31:20.077Z 29 Acked 1028 Read 10 Sent Done Done false
89412024-02-01T17:31:20.077Z 30 Acked 1029 Read 10 Sent Done Done false
89422024-02-01T17:31:20.077Z STATES DS:0 DS:1 DS:2 TOTAL
89432024-02-01T17:31:20.077Z New 0 0 0 0
89442024-02-01T17:31:20.078Z Sent 30 0 0 30
89452024-02-01T17:31:20.078Z Done 0 30 30 60
89462024-02-01T17:31:20.078Z Skipped 0 0 0 0
89472024-02-01T17:31:20.078Z Error 0 0 0 0
89482024-02-01T17:31:20.078ZLast Flush: 0 1018 1018
89492024-02-01T17:31:20.078ZDownstairs last five completed:
89502024-02-01T17:31:20.078ZUpstairs last five completed: 30 29 28 27 26
89512024-02-01T17:31:20.078Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:19.346056181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1457}
89522024-02-01T17:31:20.078ZWrote out file "/tmp/test_repair_verify.data"
89532024-02-01T17:31:20.082Z
89542024-02-01T17:31:20.089ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89552024-02-01T17:31:20.115ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89562024-02-01T17:31:20.115Z 0 000-019 80 82 82 121 123 123 F F F <---
89572024-02-01T17:31:20.119Z 1 020-039 81 81 81 122 122 122 F F F
89582024-02-01T17:31:20.119Z 2 040-059 76 76 76 114 114 114 F F F
89592024-02-01T17:31:20.119Z 3 060-079 80 80 80 121 121 121 F F F
89602024-02-01T17:31:20.120Z 4 080-099 80 80 80 121 121 121 F F F
89612024-02-01T17:31:20.120Z 5 100-119 71 71 71 104 104 104 F F F
89622024-02-01T17:31:20.120Z 6 120-139 81 81 81 122 122 122 F T T <---
89632024-02-01T17:31:20.120Z 7 140-159 79 79 79 119 119 119 F T T <---
89642024-02-01T17:31:20.120Z 8 160-179 70 82 82 102 123 123 F F F <---
89652024-02-01T17:31:20.120Z 9 180-199 76 76 76 115 115 115 F F F
89662024-02-01T17:31:20.120Z 10 200-219 80 80 80 121 121 121 F F F
89672024-02-01T17:31:20.120Z 11 220-239 80 80 80 121 121 121 F T T <---
89682024-02-01T17:31:20.120Z 12 240-259 77 77 77 116 116 116 F F F
89692024-02-01T17:31:20.120Z 13 260-279 80 82 82 121 123 123 F T T <---
89702024-02-01T17:31:20.120Z 14 280-299 80 82 82 121 123 123 F F F <---
89712024-02-01T17:31:20.120Z 15 300-319 78 78 78 118 118 118 F F F
89722024-02-01T17:31:20.120Z 16 320-339 78 78 78 118 118 118 F F F
89732024-02-01T17:31:20.120Z 17 340-359 74 82 82 109 123 123 F F F <---
89742024-02-01T17:31:20.120Z 18 360-379 81 82 82 122 123 123 F F F <---
89752024-02-01T17:31:20.120Z 19 380-399 62 62 62 87 87 87 F F F
89762024-02-01T17:31:20.120Z 20 400-419 81 81 81 122 122 122 F F F
89772024-02-01T17:31:20.120Z 21 420-439 79 82 82 119 123 123 F F F <---
89782024-02-01T17:31:20.120Z 22 440-459 80 82 82 121 123 123 F F F <---
89792024-02-01T17:31:20.120Z 23 460-479 80 82 82 121 123 123 F F F <---
89802024-02-01T17:31:20.120Z 24 480-499 79 82 82 119 123 123 F F F <---
89812024-02-01T17:31:20.120Z 25 500-519 81 81 81 122 122 122 F F F
89822024-02-01T17:31:20.120Z 26 520-539 81 81 81 122 122 122 F F F
89832024-02-01T17:31:20.120Z 27 540-559 76 76 76 114 114 114 F T T <---
89842024-02-01T17:31:20.121Z 28 560-579 80 80 80 121 121 121 F F F
89852024-02-01T17:31:20.121Z 29 580-599 80 80 80 121 121 121 F F F
89862024-02-01T17:31:20.121ZMax gen: 82, Max flush: 123
89872024-02-01T17:31:20.121ZError: Difference in extent metadata found!
89882024-02-01T17:31:20.121ZA Difference in extent metadata is expected here
89892024-02-01T17:31:20.121ZOn loop 41
89902024-02-01T17:31:20.121Z
89912024-02-01T17:31:20.124ZVerifying data now
89922024-02-01T17:31:25.304ZNo change after verify
89932024-02-01T17:31:25.304ZLoop: 41 Downstairs dump after verify (and repair):
89942024-02-01T17:31:25.328ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89952024-02-01T17:31:25.328Z 0 000-019 82 82 82 123 123 123 F F F
89962024-02-01T17:31:25.328Z 1 020-039 81 81 81 122 122 122 F F F
89972024-02-01T17:31:25.328Z 2 040-059 76 76 76 114 114 114 F F F
89982024-02-01T17:31:25.328Z 3 060-079 80 80 80 121 121 121 F F F
89992024-02-01T17:31:25.328Z 4 080-099 80 80 80 121 121 121 F F F
90002024-02-01T17:31:25.328Z 5 100-119 71 71 71 104 104 104 F F F
90012024-02-01T17:31:25.328Z 6 120-139 83 83 83 124 124 124 F F F
90022024-02-01T17:31:25.331Z 7 140-159 83 83 83 124 124 124 F F F
90032024-02-01T17:31:25.331Z 8 160-179 82 82 82 123 123 123 F F F
90042024-02-01T17:31:25.331Z 9 180-199 76 76 76 115 115 115 F F F
90052024-02-01T17:31:25.331Z 10 200-219 80 80 80 121 121 121 F F F
90062024-02-01T17:31:25.331Z 11 220-239 83 83 83 124 124 124 F F F
90072024-02-01T17:31:25.331Z 12 240-259 77 77 77 116 116 116 F F F
90082024-02-01T17:31:25.331Z 13 260-279 83 83 83 124 124 124 F F F
90092024-02-01T17:31:25.331Z 14 280-299 82 82 82 123 123 123 F F F
90102024-02-01T17:31:25.332Z 15 300-319 78 78 78 118 118 118 F F F
90112024-02-01T17:31:25.332Z 16 320-339 78 78 78 118 118 118 F F F
90122024-02-01T17:31:25.332Z 17 340-359 82 82 82 123 123 123 F F F
90132024-02-01T17:31:25.332Z 18 360-379 82 82 82 123 123 123 F F F
90142024-02-01T17:31:25.332Z 19 380-399 62 62 62 87 87 87 F F F
90152024-02-01T17:31:25.332Z 20 400-419 81 81 81 122 122 122 F F F
90162024-02-01T17:31:25.332Z 21 420-439 82 82 82 123 123 123 F F F
90172024-02-01T17:31:25.332Z 22 440-459 82 82 82 123 123 123 F F F
90182024-02-01T17:31:25.332Z 23 460-479 82 82 82 123 123 123 F F F
90192024-02-01T17:31:25.332Z 24 480-499 82 82 82 123 123 123 F F F
90202024-02-01T17:31:25.332Z 25 500-519 81 81 81 122 122 122 F F F
90212024-02-01T17:31:25.332Z 26 520-539 81 81 81 122 122 122 F F F
90222024-02-01T17:31:25.332Z 27 540-559 83 83 83 124 124 124 F F F
90232024-02-01T17:31:25.332Z 28 560-579 80 80 80 121 121 121 F F F
90242024-02-01T17:31:25.332Z 29 580-599 80 80 80 121 121 121 F F F
90252024-02-01T17:31:25.332ZMax gen: 83, Max flush: 124
90262024-02-01T17:31:25.337Z
90272024-02-01T17:31:25.337ZBegin loop 42
90282024-02-01T17:31:25.337ZDownstairs to restart: 1
90292024-02-01T17:31:25.367Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.636931707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
90302024-02-01T17:31:25.371Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637387547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
90312024-02-01T17:31:25.371Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
90322024-02-01T17:31:25.371Z,"time":"2024-02-01T17:31:24.637460189Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
90332024-02-01T17:31:25.371Z{"msg":"Crucible b45ddda1-1e80-4657-ba74-735b282fc9b4 has session id: 87c82f82-6aff-4043-a7b8-962cd6a51f42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637696918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90342024-02-01T17:31:25.371Z{"msg":"Upstairs opts: Upstairs UUID: b45ddda1-1e80-4657-ba74-735b282fc9b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637762683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90352024-02-01T17:31:25.371Z{"msg":"Crucible stats registered with UUID: b45ddda1-1e80-4657-ba74-735b282fc9b4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637832966Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90362024-02-01T17:31:25.371Z{"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637903305Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
90372024-02-01T17:31:25.371Z{"msg":"b45ddda1-1e80-4657-ba74-735b282fc9b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637941274Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90382024-02-01T17:31:25.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.637979273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90392024-02-01T17:31:25.371Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638028559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90402024-02-01T17:31:25.371Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.63810273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90412024-02-01T17:31:25.372Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638229838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90422024-02-01T17:31:25.372Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638302248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90432024-02-01T17:31:25.373Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638349776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90442024-02-01T17:31:25.373Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638394509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90452024-02-01T17:31:25.373Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638439546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90462024-02-01T17:31:25.373Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638482129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90472024-02-01T17:31:25.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638528192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90482024-02-01T17:31:25.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638574368Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90492024-02-01T17:31:25.373Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.638629611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"io task","client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90502024-02-01T17:31:25.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.63967114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90512024-02-01T17:31:25.374Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.63973329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90522024-02-01T17:31:25.374Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.639760011Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90532024-02-01T17:31:25.375Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.6397829Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90542024-02-01T17:31:25.375Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.640720108Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90552024-02-01T17:31:25.375Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.640990907Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90562024-02-01T17:31:25.379Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.641555409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90572024-02-01T17:31:25.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.641650586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90582024-02-01T17:31:25.379Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.641699066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90592024-02-01T17:31:25.379Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.641772989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90602024-02-01T17:31:25.379Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.641820205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90612024-02-01T17:31:25.380Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.642079989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90622024-02-01T17:31:25.380Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.642142564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90632024-02-01T17:31:25.380Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.642169251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90642024-02-01T17:31:25.380Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.642945214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90652024-02-01T17:31:25.380Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643716744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90662024-02-01T17:31:25.381Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643794119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90672024-02-01T17:31:25.381Z{"msg":"[0]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 104, 124, 124, 123, 115, 121, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643831981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90682024-02-01T17:31:25.381Z{"msg":"[0]R generation[0..12]: [82, 81, 76, 80, 80, 71, 83, 83, 82, 76, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643856585Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90692024-02-01T17:31:25.381Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643901251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90702024-02-01T17:31:25.381Z{"msg":"[1]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 104, 124, 124, 123, 115, 121, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643927261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90712024-02-01T17:31:25.381Z{"msg":"[1]R generation[0..12]: [82, 81, 76, 80, 80, 71, 83, 83, 82, 76, 80, 83]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643968404Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90722024-02-01T17:31:25.381Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.643993981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90732024-02-01T17:31:25.381Z{"msg":"[2]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 104, 124, 124, 123, 115, 121, 124]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644035623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90742024-02-01T17:31:25.381Z{"msg":"[2]R generation[0..12]: [82, 81, 76, 80, 80, 71, 83, 83, 82, 76, 80, 83]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
90752024-02-01T17:31:25.381Z,"time":"2024-02-01T17:31:24.644060376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90762024-02-01T17:31:25.381Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644086672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90772024-02-01T17:31:25.381Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644108267Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90782024-02-01T17:31:25.381Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644130706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90792024-02-01T17:31:25.381Z{"msg":"Next flush: 125","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644169747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90802024-02-01T17:31:25.381Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644192875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"":"downstairs","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90812024-02-01T17:31:25.381Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.64421486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90822024-02-01T17:31:25.381Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644255727Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"0","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90832024-02-01T17:31:25.381Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644279397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"1","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90842024-02-01T17:31:25.381Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644301211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"client":"2","":"downstairs","session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90852024-02-01T17:31:25.381Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644340912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90862024-02-01T17:31:25.381Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644363413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90872024-02-01T17:31:25.381Z{"msg":"b45ddda1-1e80-4657-ba74-735b282fc9b4 is now active with session: 87c82f82-6aff-4043-a7b8-962cd6a51f42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644385631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90882024-02-01T17:31:25.382Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644423973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465,"session_id":"87c82f82-6aff-4043-a7b8-962cd6a51f42"}
90892024-02-01T17:31:25.382Z{"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:24.644446302Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
90902024-02-01T17:31:25.382ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90912024-02-01T17:31:25.382ZLoading write count information from file "/tmp/test_repair_verify.data"
90922024-02-01T17:31:25.382ZRun Repair workload
90932024-02-01T17:31:25.382Z01/30 Write block 228 len 8192 data: 4 5
90942024-02-01T17:31:25.382Z02/30 Write block 277 len 20480 data: 10 11 8 9 7
90952024-02-01T17:31:25.386Z03/30 Write block 406 len 12288 data: 6 7 8
90962024-02-01T17:31:25.387Z04/30 Write block 107 len 40960 data: 8 7 6 3 4 4 5 5 5 4
90972024-02-01T17:31:25.387Z05/30 Read block 521 len 36864
90982024-02-01T17:31:25.387Z06/30 Write block 308 len 8192 data: 5 4
90992024-02-01T17:31:25.387Z07/30 Write block 349 len 36864 data: 4 3 4 7 6 10 8 7 8
91002024-02-01T17:31:25.389Z08/30 Write block 412 len 32768 data: 5 5 7 7 9 5 6 7
91012024-02-01T17:31:25.389Z09/30 Read block 145 len 16384
91022024-02-01T17:31:25.392Z10/30 Write block 475 len 24576 data: 5 5 4 5 6 6
91032024-02-01T17:31:25.393Z11/30 Read block 246 len 8192
91042024-02-01T17:31:25.393Z12/30 Write block 486 len 28672 data: 7 8 9 9 10 10 11
91052024-02-01T17:31:25.396Z13/30 Read block 171 len 32768
91062024-02-01T17:31:25.396Z14/30 Write block 270 len 4096 data: 8
91072024-02-01T17:31:25.399Z15/30 Write block 299 len 24576 data: 10 9 7 6 6 7
91082024-02-01T17:31:25.399Z16/30 Flush
91092024-02-01T17:31:25.402Z17/30 Write block 452 len 32768 data: 5 5 5 5 8 10 10 12
91102024-02-01T17:31:25.405Z18/30 Read block 555 len 8192
91112024-02-01T17:31:25.405Z19/30 Write block 164 len 20480 data: 7 5 5 5 6
91122024-02-01T17:31:25.405Z20/30 Flush
91132024-02-01T17:31:25.409Z21/30 Write block 336 len 28672 data: 11 8 7 6 4 4 5
91142024-02-01T17:31:25.413Z22/30 Read block 137 len 12288
91152024-02-01T17:31:25.413Z23/30 Read block 538 len 20480
91162024-02-01T17:31:25.417Z24/30 Write block 385 len 40960 data: 5 4 4 5 6 7 7 8 9 7
91172024-02-01T17:31:25.417Z25/30 Read block 423 len 4096
91182024-02-01T17:31:25.420Z26/30 Write block 344 len 20480 data: 5 5 5 6 6
91192024-02-01T17:31:25.420Z27/30 Read block 242 len 16384
91202024-02-01T17:31:25.420Z28/30 Write block 594 len 8192 data: 5 5
91212024-02-01T17:31:25.420Z29/30 Read block 430 len 28672
91222024-02-01T17:31:25.423Z30/30 Write block 371 len 12288 data: 8 8 8
91232024-02-01T17:31:25.423Z----------------------------------------------------------------
91242024-02-01T17:31:25.424Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
91252024-02-01T17:31:25.424ZStates: Active Active Active
91262024-02-01T17:31:25.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91272024-02-01T17:31:25.424Z 1 Acked 1000 Write 2 Done Done Done false
91282024-02-01T17:31:25.424Z 2 Acked 1001 Write 5 Done Sent Done false
91292024-02-01T17:31:25.424Z 3 Acked 1002 Write 3 Done Sent Done false
91302024-02-01T17:31:25.424Z 4 Acked 1003 Write 10 Done Sent Done false
91312024-02-01T17:31:25.424Z 5 Acked 1004 Read 9 Done Sent Done false
91322024-02-01T17:31:25.424Z 6 Acked 1005 Write 2 Done Sent Done false
91332024-02-01T17:31:25.424Z 7 Acked 1006 Write 9 Done Sent Done false
91342024-02-01T17:31:25.424Z 8 Acked 1007 Write 8 Done Sent Done false
91352024-02-01T17:31:25.424Z 9 Acked 1008 Read 4 Done Sent Done false
91362024-02-01T17:31:25.424Z 10 Acked 1009 Write 6 Done Sent Done false
91372024-02-01T17:31:25.424Z 11 Acked 1010 Read 2 Done Sent Done false
91382024-02-01T17:31:25.424Z 12 Acked 1011 Write 7 Done Sent Done false
91392024-02-01T17:31:25.424Z 13 Acked 1012 Read 8 Done Sent Done false
91402024-02-01T17:31:25.424Z 14 Acked 1013 Write 1 Done Sent Done false
91412024-02-01T17:31:25.424Z 15 Acked 1014 Write 6 Done Sent Done false
91422024-02-01T17:31:25.424Z 16 Acked 1015 Flush 0 Done Sent Done false
91432024-02-01T17:31:25.424Z 17 Acked 1016 Write 8 Done Sent Done false
91442024-02-01T17:31:25.424Z 18 Acked 1017 Read 2 Done Sent Done false
91452024-02-01T17:31:25.424Z 19 Acked 1018 Write 5 Done Sent Done false
91462024-02-01T17:31:25.425Z 20 Acked 1019 Flush 0 Done Sent Done false
91472024-02-01T17:31:25.425Z 21 Acked 1020 Write 7 Done Sent Done false
91482024-02-01T17:31:25.425Z 22 Acked 1021 Read 3 Done Sent Done false
91492024-02-01T17:31:25.425Z 23 Acked 1022 Read 5 Done Sent Done false
91502024-02-01T17:31:25.425Z 24 Acked 1023 Write 10 Done Sent Done false
91512024-02-01T17:31:25.425Z 25 Acked 1024 Read 1 Done Sent Done false
91522024-02-01T17:31:25.426Z 26 Acked 1025 Write 5 Done Sent Done false
91532024-02-01T17:31:25.426Z 27 Acked 1026 Read 4 Done Sent Done false
91542024-02-01T17:31:25.426Z 28 Acked 1027 Write 2 Done Sent Done false
91552024-02-01T17:31:25.426Z 29 Acked 1028 Read 7 Done Sent Done false
91562024-02-01T17:31:25.426Z 30 Acked 1029 Write 3 Sent Sent Sent false
91572024-02-01T17:31:25.426Z STATES DS:0 DS:1 DS:2 TOTAL
91582024-02-01T17:31:25.426Z New 0 0 0 0
91592024-02-01T17:31:25.426Z Sent 1 29 1 31
91602024-02-01T17:31:25.426Z Done 29 1 29 59
91612024-02-01T17:31:25.427Z Skipped 0 0 0 0
91622024-02-01T17:31:25.427Z Error 0 0 0 0
91632024-02-01T17:31:25.427ZLast Flush: 1019 0 1019
91642024-02-01T17:31:25.427ZDownstairs last five completed:
91652024-02-01T17:31:25.427ZUpstairs last five completed: 30 29 28 27 26
91662024-02-01T17:31:25.430Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:24.694306133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1465}
91672024-02-01T17:31:25.431ZWrote out file "/tmp/test_repair_verify.data"
91682024-02-01T17:31:25.434Z
91692024-02-01T17:31:25.440ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91702024-02-01T17:31:25.462ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91712024-02-01T17:31:25.462Z 0 000-019 82 82 82 123 123 123 F F F
91722024-02-01T17:31:25.462Z 1 020-039 81 81 81 122 122 122 F F F
91732024-02-01T17:31:25.462Z 2 040-059 76 76 76 114 114 114 F F F
91742024-02-01T17:31:25.463Z 3 060-079 80 80 80 121 121 121 F F F
91752024-02-01T17:31:25.463Z 4 080-099 80 80 80 121 121 121 F F F
91762024-02-01T17:31:25.467Z 5 100-119 84 71 84 125 104 125 F F F <---
91772024-02-01T17:31:25.467Z 6 120-139 83 83 83 124 124 124 F F F
91782024-02-01T17:31:25.467Z 7 140-159 83 83 83 124 124 124 F F F
91792024-02-01T17:31:25.467Z 8 160-179 84 82 84 126 123 126 F F F <---
91802024-02-01T17:31:25.467Z 9 180-199 76 76 76 115 115 115 F F F
91812024-02-01T17:31:25.467Z 10 200-219 80 80 80 121 121 121 F F F
91822024-02-01T17:31:25.467Z 11 220-239 84 83 84 125 124 125 F T F <---
91832024-02-01T17:31:25.467Z 12 240-259 77 77 77 116 116 116 F F F
91842024-02-01T17:31:25.467Z 13 260-279 84 83 84 125 124 125 F F F <---
91852024-02-01T17:31:25.467Z 14 280-299 84 82 84 125 123 125 F F F <---
91862024-02-01T17:31:25.467Z 15 300-319 84 78 84 125 118 125 F F F <---
91872024-02-01T17:31:25.467Z 16 320-339 78 78 78 118 118 118 T F T <---
91882024-02-01T17:31:25.467Z 17 340-359 84 82 84 125 123 125 T F T <---
91892024-02-01T17:31:25.467Z 18 360-379 82 82 82 123 123 123 T F T <---
91902024-02-01T17:31:25.467Z 19 380-399 62 62 62 87 87 87 T F T <---
91912024-02-01T17:31:25.467Z 20 400-419 84 81 84 125 122 125 F F F <---
91922024-02-01T17:31:25.468Z 21 420-439 82 82 82 123 123 123 F F F
91932024-02-01T17:31:25.468Z 22 440-459 84 82 84 126 123 126 F F F <---
91942024-02-01T17:31:25.468Z 23 460-479 84 82 84 125 123 125 F F F <---
91952024-02-01T17:31:25.468Z 24 480-499 84 82 84 125 123 125 F F F <---
91962024-02-01T17:31:25.468Z 25 500-519 81 81 81 122 122 122 F F F
91972024-02-01T17:31:25.468Z 26 520-539 81 81 81 122 122 122 F F F
91982024-02-01T17:31:25.468Z 27 540-559 83 83 83 124 124 124 F F F
91992024-02-01T17:31:25.468Z 28 560-579 80 80 80 121 121 121 F F F
92002024-02-01T17:31:25.468Z 29 580-599 80 80 80 121 121 121 T F T <---
92012024-02-01T17:31:25.468ZMax gen: 84, Max flush: 126
92022024-02-01T17:31:25.468ZError: Difference in extent metadata found!
92032024-02-01T17:31:25.468ZA Difference in extent metadata is expected here
92042024-02-01T17:31:25.468ZOn loop 42
92052024-02-01T17:31:25.468Z
92062024-02-01T17:31:25.472ZVerifying data now
92072024-02-01T17:31:30.729ZNo change after verify
92082024-02-01T17:31:30.729ZLoop: 42 Downstairs dump after verify (and repair):
92092024-02-01T17:31:30.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92102024-02-01T17:31:30.753Z 0 000-019 82 82 82 123 123 123 F F F
92112024-02-01T17:31:30.767Z 1 020-039 81 81 81 122 122 122 F F F
92122024-02-01T17:31:30.767Z 2 040-059 76 76 76 114 114 114 F F F
92132024-02-01T17:31:30.767Z 3 060-079 80 80 80 121 121 121 F F F
92142024-02-01T17:31:30.767Z 4 080-099 80 80 80 121 121 121 F F F
92152024-02-01T17:31:30.768Z 5 100-119 84 84 84 125 125 125 F F F
92162024-02-01T17:31:30.768Z 6 120-139 83 83 83 124 124 124 F F F
92172024-02-01T17:31:30.768Z 7 140-159 83 83 83 124 124 124 F F F
92182024-02-01T17:31:30.768Z 8 160-179 84 84 84 126 126 126 F F F
92192024-02-01T17:31:30.768Z 9 180-199 76 76 76 115 115 115 F F F
92202024-02-01T17:31:30.768Z 10 200-219 80 80 80 121 121 121 F F F
92212024-02-01T17:31:30.769Z 11 220-239 84 84 84 125 125 125 F F F
92222024-02-01T17:31:30.769Z 12 240-259 77 77 77 116 116 116 F F F
92232024-02-01T17:31:30.769Z 13 260-279 84 84 84 125 125 125 F F F
92242024-02-01T17:31:30.769Z 14 280-299 84 84 84 125 125 125 F F F
92252024-02-01T17:31:30.769Z 15 300-319 84 84 84 125 125 125 F F F
92262024-02-01T17:31:30.769Z 16 320-339 85 85 85 127 127 127 F F F
92272024-02-01T17:31:30.769Z 17 340-359 85 85 85 127 127 127 F F F
92282024-02-01T17:31:30.769Z 18 360-379 85 85 85 127 127 127 F F F
92292024-02-01T17:31:30.770Z 19 380-399 85 85 85 127 127 127 F F F
92302024-02-01T17:31:30.770Z 20 400-419 84 84 84 125 125 125 F F F
92312024-02-01T17:31:30.770Z 21 420-439 82 82 82 123 123 123 F F F
92322024-02-01T17:31:30.770Z 22 440-459 84 84 84 126 126 126 F F F
92332024-02-01T17:31:30.770Z 23 460-479 84 84 84 125 125 125 F F F
92342024-02-01T17:31:30.770Z 24 480-499 84 84 84 125 125 125 F F F
92352024-02-01T17:31:30.770Z 25 500-519 81 81 81 122 122 122 F F F
92362024-02-01T17:31:30.770Z 26 520-539 81 81 81 122 122 122 F F F
92372024-02-01T17:31:30.770Z 27 540-559 83 83 83 124 124 124 F F F
92382024-02-01T17:31:30.770Z 28 560-579 80 80 80 121 121 121 F F F
92392024-02-01T17:31:30.770Z 29 580-599 85 85 85 127 127 127 F F F
92402024-02-01T17:31:30.770ZMax gen: 85, Max flush: 127
92412024-02-01T17:31:30.771Z
92422024-02-01T17:31:30.771ZBegin loop 43
92432024-02-01T17:31:30.772ZDownstairs to restart: 2
92442024-02-01T17:31:30.785Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.055607338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
92452024-02-01T17:31:30.785Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056175502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
92462024-02-01T17:31:30.798Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056306437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
92472024-02-01T17:31:30.798Z{"msg":"Crucible runtime is spawned
92482024-02-01T17:31:30.799ZCrucible 0ca20197-7202-4737-a5a5-b31dad4ad247 has session id: 5276d561-fde0-435b-aab9-d1ccc6504ca3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056405841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92492024-02-01T17:31:30.799Z{"msg":"Upstairs opts: Upstairs UUID: 0ca20197-7202-4737-a5a5-b31dad4ad247, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.05649049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92502024-02-01T17:31:30.799Z{"msg":"Crucible stats registered with UUID: 0ca20197-7202-4737-a5a5-b31dad4ad247","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056544146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92512024-02-01T17:31:30.799Z{"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056582858Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
92522024-02-01T17:31:30.799Z{"msg":"0ca20197-7202-4737-a5a5-b31dad4ad247 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056726211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92532024-02-01T17:31:30.799Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056803319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92542024-02-01T17:31:30.800Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056886294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92552024-02-01T17:31:30.800Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.056931662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92562024-02-01T17:31:30.801Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057006899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92572024-02-01T17:31:30.801Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.05705124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92582024-02-01T17:31:30.801Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057120966Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92592024-02-01T17:31:30.801Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057163504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92602024-02-01T17:31:30.801Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057202557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92612024-02-01T17:31:30.801Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057240263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92622024-02-01T17:31:30.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057282933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92632024-02-01T17:31:30.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057322627Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92642024-02-01T17:31:30.801Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.057501396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"io task","client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92652024-02-01T17:31:30.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058633387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92662024-02-01T17:31:30.801Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058702499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92672024-02-01T17:31:30.801Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058750251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92682024-02-01T17:31:30.802Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058786779Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92692024-02-01T17:31:30.802Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058826836Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92702024-02-01T17:31:30.802Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.058865048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92712024-02-01T17:31:30.802Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.05964919Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92722024-02-01T17:31:30.802Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.059739759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92732024-02-01T17:31:30.802Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.059985486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92742024-02-01T17:31:30.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.06067177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92752024-02-01T17:31:30.802Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.060766064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92762024-02-01T17:31:30.802Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.060898322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92772024-02-01T17:31:30.802Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.060984074Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92782024-02-01T17:31:30.802Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061114782Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92792024-02-01T17:31:30.803Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061190301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92802024-02-01T17:31:30.803Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061233628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92812024-02-01T17:31:30.803Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061359469Z","hostname":"Wait for a query_work_queue command to finish before sending IO
92822024-02-01T17:31:30.803Zip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92832024-02-01T17:31:30.803Z{"msg":"[0]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 125, 124, 124, 126, 115, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061484448Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92842024-02-01T17:31:30.805Z{"msg":"[0]R generation[0..12]: [82, 81, 76, 80, 80, 84, 83, 83, 84, 76, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061561724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92852024-02-01T17:31:30.805Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061635197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92862024-02-01T17:31:30.805Z{"msg":"[1]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 125, 124, 124, 126, 115, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061710256Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92872024-02-01T17:31:30.805Z{"msg":"[1]R generation[0..12]: [82, 81, 76, 80, 80, 84, 83, 83, 84, 76, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061785691Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92882024-02-01T17:31:30.805Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061863414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92892024-02-01T17:31:30.805Z{"msg":"[2]R flush_numbers[0..12]: [123, 122, 114, 121, 121, 125, 124, 124, 126, 115, 121, 125]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.061942169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92902024-02-01T17:31:30.805Z{"msg":"[2]R generation[0..12]: [82, 81, 76, 80, 80, 84, 83, 83, 84, 76, 80, 84]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062022535Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92912024-02-01T17:31:30.805Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062098739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92922024-02-01T17:31:30.805ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92932024-02-01T17:31:30.806Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062174033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92942024-02-01T17:31:30.806Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062231591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92952024-02-01T17:31:30.806Z{"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062257021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92962024-02-01T17:31:30.807Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062297458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"":"downstairs","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92972024-02-01T17:31:30.807Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.06232177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92982024-02-01T17:31:30.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062361053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"0","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
92992024-02-01T17:31:30.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062385226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"1","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93002024-02-01T17:31:30.807Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062425258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"client":"2","":"downstairs","session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93012024-02-01T17:31:30.807Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062449639Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93022024-02-01T17:31:30.807Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062471468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93032024-02-01T17:31:30.807Z{"msg":"0ca20197-7202-4737-a5a5-b31dad4ad247 is now active with session: 5276d561-fde0-435b-aab9-d1ccc6504ca3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062511842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93042024-02-01T17:31:30.807Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.06253395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473,"session_id":"5276d561-fde0-435b-aab9-d1ccc6504ca3"}
93052024-02-01T17:31:30.807Z{"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:30.062576755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
93062024-02-01T17:31:30.807ZLoading write count information from file "/tmp/test_repair_verify.data"
93072024-02-01T17:31:30.808ZRun Repair workload
93082024-02-01T17:31:30.808Z01/30 Write block 169 len 32768 data: 8 9 10 10 10 10 9 12
93092024-02-01T17:31:30.808Z02/30 Read block 68 len 16384
93102024-02-01T17:31:30.813Z03/30 Write block 44 len 28672 data: 7 7 8 6 9 8 11
93112024-02-01T17:31:30.813Z04/30 Read block 484 len 20480
93122024-02-01T17:31:30.813Z05/30 Write block 579 len 24576 data: 6 6 6 7 5 6
93132024-02-01T17:31:30.813Z06/30 Write block 432 len 40960 data: 7 8 9 10 9 10 8 8 8 9
93142024-02-01T17:31:30.813Z07/30 Write block 415 len 8192 data: 8 10
93152024-02-01T17:31:30.813Z08/30 Read block 109 len 28672
93162024-02-01T17:31:30.813Z09/30 Read block 111 len 20480
93172024-02-01T17:31:30.813Z10/30 Read block 148 len 16384
93182024-02-01T17:31:30.814Z11/30 Write block 241 len 8192 data: 4 6
93192024-02-01T17:31:30.814Z12/30 Read block 372 len 12288
93202024-02-01T17:31:30.814Z13/30 Read block 497 len 32768
93212024-02-01T17:31:30.814Z14/30 Read block 63 len 36864
93222024-02-01T17:31:30.817Z15/30 Write block 425 len 4096 data: 9
93232024-02-01T17:31:30.817Z16/30 Write block 17 len 40960 data: 8 9 7 8 8 9 8 8 9 7
93242024-02-01T17:31:30.818Z17/30 Write block 101 len 36864 data: 5 6 5 4 7 8 9 8 7
93252024-02-01T17:31:30.824Z18/30 Write block 222 len 40960 data: 7 6 5 4 4 4 5 6 5 5
93262024-02-01T17:31:30.824Z19/30 Read block 426 len 40960
93272024-02-01T17:31:30.827Z20/30 Read block 166 len 32768
93282024-02-01T17:31:30.827Z21/30 Write block 201 len 20480 data: 7 9 11 11 10
93292024-02-01T17:31:30.827Z22/30 Read block 275 len 24576
93302024-02-01T17:31:30.827Z23/30 Read block 140 len 4096
93312024-02-01T17:31:30.831Z24/30 Write block 433 len 32768 data: 9 10 11 10 11 9 9 9
93322024-02-01T17:31:30.831Z25/30 Write block 517 len 24576 data: 3 5 5 6 6 6
93332024-02-01T17:31:30.831Z26/30 Read block 68 len 4096
93342024-02-01T17:31:30.834Z27/30 Write block 43 len 28672 data: 9 8 8 9 7 10 9
93352024-02-01T17:31:30.834Z28/30 Read block 344 len 36864
93362024-02-01T17:31:30.839Z29/30 Write block 313 len 32768 data: 9 7 9 9 9 8 8 9
93372024-02-01T17:31:30.839Z30/30 Write block 92 len 40960 data: 6 5 5 5 6 6 6 6 6 6
93382024-02-01T17:31:30.840Z----------------------------------------------------------------
93392024-02-01T17:31:30.840Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
93402024-02-01T17:31:30.840ZStates: Active Active Active
93412024-02-01T17:31:30.840ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93422024-02-01T17:31:30.840Z 1 Acked 1000 Write 8 Done Done Sent false
93432024-02-01T17:31:30.840Z 2 Acked 1001 Read 4 Done Done Sent false
93442024-02-01T17:31:30.840Z 3 Acked 1002 Write 7 Done Done Sent false
93452024-02-01T17:31:30.840Z 4 Acked 1003 Read 5 Done Done Sent false
93462024-02-01T17:31:30.841Z 5 Acked 1004 Write 6 Done Done Sent false
93472024-02-01T17:31:30.841Z 6 Acked 1005 Write 10 Done Done Sent false
93482024-02-01T17:31:30.841Z 7 Acked 1006 Write 2 Done Done Sent false
93492024-02-01T17:31:30.842Z 8 Acked 1007 Read 7 Done Done Sent false
93502024-02-01T17:31:30.843Z 9 Acked 1008 Read 5 Done Done Sent false
93512024-02-01T17:31:30.843Z 10 Acked 1009 Read 4 Done Done Sent false
93522024-02-01T17:31:30.843Z 11 Acked 1010 Write 2 Done Done Sent false
93532024-02-01T17:31:30.843Z 12 Acked 1011 Read 3 Done Done Sent false
93542024-02-01T17:31:30.843Z 13 Acked 1012 Read 8 Done Done Sent false
93552024-02-01T17:31:30.843Z 14 Acked 1013 Read 9 Done Done Sent false
93562024-02-01T17:31:30.843Z 15 Acked 1014 Write 1 Done Done Sent false
93572024-02-01T17:31:30.843Z 16 Acked 1015 Write 10 Done Done Sent false
93582024-02-01T17:31:30.843Z 17 Acked 1016 Write 9 Done Done Sent false
93592024-02-01T17:31:30.843Z 18 Acked 1017 Write 10 Done Done Sent false
93602024-02-01T17:31:30.844Z 19 Acked 1018 Read 10 Done Done Sent false
93612024-02-01T17:31:30.844Z 20 Acked 1019 Read 8 Done Done Sent false
93622024-02-01T17:31:30.844Z 21 Acked 1020 Write 5 Done Done Sent false
93632024-02-01T17:31:30.844Z 22 Acked 1021 Read 6 Done Done Sent false
93642024-02-01T17:31:30.844Z 23 Acked 1022 Read 1 Done Done Sent false
93652024-02-01T17:31:30.844Z 24 Acked 1023 Write 8 Done Done Sent false
93662024-02-01T17:31:30.844Z 25 Acked 1024 Write 6 Done Done Sent false
93672024-02-01T17:31:30.844Z 26 Acked 1025 Read 1 Done Done Sent false
93682024-02-01T17:31:30.844Z 27 Acked 1026 Write 7 Done Done Sent false
93692024-02-01T17:31:30.844Z 28 Acked 1027 Read 9 Done Done Sent false
93702024-02-01T17:31:30.845Z 29 Acked 1028 Write 8 Sent Done Sent false
93712024-02-01T17:31:30.845Z 30 Acked 1029 Write 10 Sent Sent Sent false
93722024-02-01T17:31:30.846Z STATES DS:0 DS:1 DS:2 TOTAL
93732024-02-01T17:31:30.847Z New 0 0 0 0
93742024-02-01T17:31:30.847Z Sent 2 1 30 33
93752024-02-01T17:31:30.847Z Done 28 29 0 57
93762024-02-01T17:31:30.847Z Skipped 0 0 0 0
93772024-02-01T17:31:30.847Z Error 0 0 0 0
93782024-02-01T17:31:30.847ZLast Flush: 0 0 0
93792024-02-01T17:31:30.847ZDownstairs last five completed:
93802024-02-01T17:31:30.847ZUpstairs last five completed: 30 29 28 27 26
93812024-02-01T17:31:30.848Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:30.109252364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1473}
93822024-02-01T17:31:30.852ZWrote out file "/tmp/test_repair_verify.data"
93832024-02-01T17:31:30.852Z
93842024-02-01T17:31:30.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93852024-02-01T17:31:30.881ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93862024-02-01T17:31:30.881Z 0 000-019 82 82 82 123 123 123 T T F <---
93872024-02-01T17:31:30.881Z 1 020-039 81 81 81 122 122 122 T T F <---
93882024-02-01T17:31:30.881Z 2 040-059 76 76 76 114 114 114 T T F <---
93892024-02-01T17:31:30.882Z 3 060-079 80 80 80 121 121 121 F F F
93902024-02-01T17:31:30.882Z 4 080-099 80 80 80 121 121 121 T T F <---
93912024-02-01T17:31:30.882Z 5 100-119 84 84 84 125 125 125 T T F <---
93922024-02-01T17:31:30.882Z 6 120-139 83 83 83 124 124 124 F F F
93932024-02-01T17:31:30.882Z 7 140-159 83 83 83 124 124 124 F F F
93942024-02-01T17:31:30.882Z 8 160-179 84 84 84 126 126 126 T T F <---
93952024-02-01T17:31:30.888Z 9 180-199 76 76 76 115 115 115 F F F
93962024-02-01T17:31:30.888Z 10 200-219 80 80 80 121 121 121 T T F <---
93972024-02-01T17:31:30.888Z 11 220-239 84 84 84 125 125 125 T T F <---
93982024-02-01T17:31:30.888Z 12 240-259 77 77 77 116 116 116 T T F <---
93992024-02-01T17:31:30.888Z 13 260-279 84 84 84 125 125 125 F F F
94002024-02-01T17:31:30.889Z 14 280-299 84 84 84 125 125 125 F F F
94012024-02-01T17:31:30.889Z 15 300-319 84 84 84 125 125 125 T T F <---
94022024-02-01T17:31:30.889Z 16 320-339 85 85 85 127 127 127 T T F <---
94032024-02-01T17:31:30.889Z 17 340-359 85 85 85 127 127 127 F F F
94042024-02-01T17:31:30.889Z 18 360-379 85 85 85 127 127 127 F F F
94052024-02-01T17:31:30.889Z 19 380-399 85 85 85 127 127 127 F F F
94062024-02-01T17:31:30.889Z 20 400-419 84 84 84 125 125 125 T T F <---
94072024-02-01T17:31:30.889Z 21 420-439 82 82 82 123 123 123 T T F <---
94082024-02-01T17:31:30.889Z 22 440-459 84 84 84 126 126 126 T T F <---
94092024-02-01T17:31:30.889Z 23 460-479 84 84 84 125 125 125 F F F
94102024-02-01T17:31:30.890Z 24 480-499 84 84 84 125 125 125 F F F
94112024-02-01T17:31:30.890Z 25 500-519 81 81 81 122 122 122 T T F <---
94122024-02-01T17:31:30.890Z 26 520-539 81 81 81 122 122 122 T T F <---
94132024-02-01T17:31:30.890Z 27 540-559 83 83 83 124 124 124 F F F
94142024-02-01T17:31:30.890Z 28 560-579 80 80 80 121 121 121 T T F <---
94152024-02-01T17:31:30.890Z 29 580-599 85 85 85 127 127 127 T T F <---
94162024-02-01T17:31:30.890ZMax gen: 85, Max flush: 127
94172024-02-01T17:31:30.890ZError: Difference in extent metadata found!
94182024-02-01T17:31:30.890ZA Difference in extent metadata is expected here
94192024-02-01T17:31:30.890ZOn loop 43
94202024-02-01T17:31:30.890Z
94212024-02-01T17:31:30.896ZVerifying data now
94222024-02-01T17:31:36.398ZNo change after verify
94232024-02-01T17:31:36.398ZLoop: 43 Downstairs dump after verify (and repair):
94242024-02-01T17:31:36.410ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94252024-02-01T17:31:36.410Z 0 000-019 86 86 86 128 128 128 F F F
94262024-02-01T17:31:36.410Z 1 020-039 86 86 86 128 128 128 F F F
94272024-02-01T17:31:36.410Z 2 040-059 86 86 86 128 128 128 F F F
94282024-02-01T17:31:36.410Z 3 060-079 80 80 80 121 121 121 F F F
94292024-02-01T17:31:36.410Z 4 080-099 86 86 86 128 128 128 F F F
94302024-02-01T17:31:36.421Z 5 100-119 86 86 86 128 128 128 F F F
94312024-02-01T17:31:36.421Z 6 120-139 83 83 83 124 124 124 F F F
94322024-02-01T17:31:36.421Z 7 140-159 83 83 83 124 124 124 F F F
94332024-02-01T17:31:36.422Z 8 160-179 86 86 86 128 128 128 F F F
94342024-02-01T17:31:36.422Z 9 180-199 76 76 76 115 115 115 F F F
94352024-02-01T17:31:36.422Z 10 200-219 86 86 86 128 128 128 F F F
94362024-02-01T17:31:36.422Z 11 220-239 86 86 86 128 128 128 F F F
94372024-02-01T17:31:36.422Z 12 240-259 86 86 86 128 128 128 F F F
94382024-02-01T17:31:36.422Z 13 260-279 84 84 84 125 125 125 F F F
94392024-02-01T17:31:36.422Z 14 280-299 84 84 84 125 125 125 F F F
94402024-02-01T17:31:36.422Z 15 300-319 86 86 86 128 128 128 F F F
94412024-02-01T17:31:36.422Z 16 320-339 86 86 86 128 128 128 F F F
94422024-02-01T17:31:36.422Z 17 340-359 85 85 85 127 127 127 F F F
94432024-02-01T17:31:36.422Z 18 360-379 85 85 85 127 127 127 F F F
94442024-02-01T17:31:36.422Z 19 380-399 85 85 85 127 127 127 F F F
94452024-02-01T17:31:36.423Z 20 400-419 86 86 86 128 128 128 F F F
94462024-02-01T17:31:36.423Z 21 420-439 86 86 86 128 128 128 F F F
94472024-02-01T17:31:36.423Z 22 440-459 86 86 86 128 128 128 F F F
94482024-02-01T17:31:36.423Z 23 460-479 84 84 84 125 125 125 F F F
94492024-02-01T17:31:36.423Z 24 480-499 84 84 84 125 125 125 F F F
94502024-02-01T17:31:36.423Z 25 500-519 86 86 86 128 128 128 F F F
94512024-02-01T17:31:36.423Z 26 520-539 86 86 86 128 128 128 F F F
94522024-02-01T17:31:36.423Z 27 540-559 83 83 83 124 124 124 F F F
94532024-02-01T17:31:36.423Z 28 560-579 86 86 86 128 128 128 F F F
94542024-02-01T17:31:36.423Z 29 580-599 86 86 86 128 128 128 F F F
94552024-02-01T17:31:36.423ZMax gen: 86, Max flush: 128
94562024-02-01T17:31:36.423Z
94572024-02-01T17:31:36.423ZBegin loop 44
94582024-02-01T17:31:36.423ZDownstairs to restart: 0
94592024-02-01T17:31:36.447Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.716594781Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
94602024-02-01T17:31:36.469Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717073796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
94612024-02-01T17:31:36.469Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717136097Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
94622024-02-01T17:31:36.470Z{"msg":"Crucible 7ff03194-5a9f-4689-924d-0358bd443fbf has session id: c053b82f-75bf-401d-8115-cecdde169e7d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717227879Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94632024-02-01T17:31:36.470Z{"msg":"Crucible runtime is spawned
94642024-02-01T17:31:36.470ZUpstairs opts: Upstairs UUID: 7ff03194-5a9f-4689-924d-0358bd443fbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717404445Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94652024-02-01T17:31:36.470Z{"msg":"Crucible stats registered with UUID: 7ff03194-5a9f-4689-924d-0358bd443fbf","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717472007Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94662024-02-01T17:31:36.470Z{"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717512946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
94672024-02-01T17:31:36.470Z{"msg":"7ff03194-5a9f-4689-924d-0358bd443fbf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717567025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94682024-02-01T17:31:36.470Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717636474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94692024-02-01T17:31:36.471Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717701891Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94702024-02-01T17:31:36.472Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717744226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94712024-02-01T17:31:36.472Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717781841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94722024-02-01T17:31:36.472Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717883533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94732024-02-01T17:31:36.472Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.717928636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94742024-02-01T17:31:36.472Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718017611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94752024-02-01T17:31:36.472Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718065747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94762024-02-01T17:31:36.472Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718150685Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94772024-02-01T17:31:36.472Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718225975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94782024-02-01T17:31:36.472Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718271345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94792024-02-01T17:31:36.473Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.718362047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"io task","client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94802024-02-01T17:31:36.473Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.719716461Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94812024-02-01T17:31:36.473Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.719758498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94822024-02-01T17:31:36.473Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.719782624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94832024-02-01T17:31:36.473Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.719804896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94842024-02-01T17:31:36.473Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.720686677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94852024-02-01T17:31:36.474Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.720747306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94862024-02-01T17:31:36.474Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.720775129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94872024-02-01T17:31:36.474Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.720879402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94882024-02-01T17:31:36.474Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.721858665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94892024-02-01T17:31:36.474Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.721928363Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94902024-02-01T17:31:36.474Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.72197486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94912024-02-01T17:31:36.474Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722058018Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94922024-02-01T17:31:36.474Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722100036Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94932024-02-01T17:31:36.475Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722156154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94942024-02-01T17:31:36.475Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722200196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94952024-02-01T17:31:36.475Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722667767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94962024-02-01T17:31:36.475Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722758153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94972024-02-01T17:31:36.475Z{"msg":"[0]R flush_numbers[0..12]: [128, 128, 128, 121, 128, 128, 124, 124, 128, 115, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722833663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94982024-02-01T17:31:36.476Z{"msg":"[0]R generation[0..12]: [86, 86, 86, 80, 86, 86, 83, 83, 86, 76, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722906294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
94992024-02-01T17:31:36.476Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.722977992Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95002024-02-01T17:31:36.476Z{"msg":"[1]R flush_numbers[0..12]: [128, 128, 128, 121, 128, 128, 124, 124, 128, 115, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723026634Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95012024-02-01T17:31:36.476Z{"msg":"[1]R generation[0..12]: [86, 86, 86, 80, 86, 86, 83, 83, 86, 76, 86, 86]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
95022024-02-01T17:31:36.476Z,"time":"2024-02-01T17:31:35.723096765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95032024-02-01T17:31:36.476Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723176862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95042024-02-01T17:31:36.476Z{"msg":"[2]R flush_numbers[0..12]: [128, 128, 128, 121, 128, 128, 124, 124, 128, 115, 128, 128]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723250761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95052024-02-01T17:31:36.476Z{"msg":"[2]R generation[0..12]: [86, 86, 86, 80, 86, 86, 83, 83, 86, 76, 86, 86]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723324601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95062024-02-01T17:31:36.477Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723398694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95072024-02-01T17:31:36.477Z{"msg":"Max found gen is 87","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723470115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95082024-02-01T17:31:36.477Z{"msg":"Generation requested: 88 >= found:87","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723541753Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95092024-02-01T17:31:36.477Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723610957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95102024-02-01T17:31:36.477Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723683266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"":"downstairs","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95112024-02-01T17:31:36.477Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723776093Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95122024-02-01T17:31:36.477Z,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95132024-02-01T17:31:36.477Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723823871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"0","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95142024-02-01T17:31:36.478Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723860301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"1","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95152024-02-01T17:31:36.478Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723905792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"client":"2","":"downstairs","session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95162024-02-01T17:31:36.478Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.72393027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95172024-02-01T17:31:36.478Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.72397006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95182024-02-01T17:31:36.478Z{"msg":"7ff03194-5a9f-4689-924d-0358bd443fbf is now active with session: c053b82f-75bf-401d-8115-cecdde169e7d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.723993583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95192024-02-01T17:31:36.478Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.724046429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481,"session_id":"c053b82f-75bf-401d-8115-cecdde169e7d"}
95202024-02-01T17:31:36.481Z{"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:35.724070985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
95212024-02-01T17:31:36.481ZLoading write count information from file "/tmp/test_repair_verify.data"
95222024-02-01T17:31:36.481ZRun Repair workload
95232024-02-01T17:31:36.481Z01/30 Flush
95242024-02-01T17:31:36.481Z02/30 Write block 124 len 28672 data: 6 6 5 5 6 5 5
95252024-02-01T17:31:36.481Z03/30 Read block 390 len 32768
95262024-02-01T17:31:36.490Z04/30 Write block 408 len 8192 data: 9 6
95272024-02-01T17:31:36.491Z05/30 Write block 551 len 24576 data: 8 8 6 4 7 6
95282024-02-01T17:31:36.491Z06/30 Read block 359 len 4096
95292024-02-01T17:31:36.491Z07/30 Read block 547 len 4096
95302024-02-01T17:31:36.492Z08/30 Read block 295 len 36864
95312024-02-01T17:31:36.492Z09/30 Read block 475 len 40960
95322024-02-01T17:31:36.492Z10/30 Write block 308 len 20480 data: 6 5 4 4 7
95332024-02-01T17:31:36.492Z11/30 Read block 178 len 32768
95342024-02-01T17:31:36.492Z12/30 Read block 171 len 20480
95352024-02-01T17:31:36.492Z13/30 Read block 186 len 36864
95362024-02-01T17:31:36.492Z14/30 Read block 151 len 40960
95372024-02-01T17:31:36.492Z15/30 Write block 214 len 28672 data: 7 7 4 4 9 10 10
95382024-02-01T17:31:36.492Z16/30 Flush
95392024-02-01T17:31:36.493Z17/30 Write block 208 len 24576 data: 13 12 8 9 5 6
95402024-02-01T17:31:36.493Z18/30 Flush
95412024-02-01T17:31:36.493Z19/30 Write block 49 len 24576 data: 10 12 11 9 8 10
95422024-02-01T17:31:36.493Z20/30 Write block 355 len 28672 data: 9 8 9 10 10 10 11
95432024-02-01T17:31:36.493Z21/30 Read block 526 len 36864
95442024-02-01T17:31:36.514Z22/30 Write block 75 len 4096 data: 9
95452024-02-01T17:31:36.514Z23/30 Read block 60 len 24576
95462024-02-01T17:31:36.514Z24/30 Flush
95472024-02-01T17:31:36.514Z25/30 Write block 362 len 8192 data: 13 11
95482024-02-01T17:31:36.514Z26/30 Read block 552 len 16384
95492024-02-01T17:31:36.514Z27/30 Write block 1 len 40960 data: 4 4 5 6 6 7 7 7 4 6
95502024-02-01T17:31:36.514Z28/30 Read block 504 len 12288
95512024-02-01T17:31:36.514Z29/30 Read block 82 len 32768
95522024-02-01T17:31:36.514Z30/30 Read block 62 len 24576
95532024-02-01T17:31:36.514Z----------------------------------------------------------------
95542024-02-01T17:31:36.514Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
95552024-02-01T17:31:36.514ZStates: Active Active Active
95562024-02-01T17:31:36.514ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95572024-02-01T17:31:36.514Z 1 Acked 1000 Flush 0 Sent Done Done false
95582024-02-01T17:31:36.514Z 2 Acked 1001 Write 7 Sent Done Done false
95592024-02-01T17:31:36.514Z 3 Acked 1002 Read 8 Sent Done Done false
95602024-02-01T17:31:36.514Z 4 Acked 1003 Write 2 Sent Done Done false
95612024-02-01T17:31:36.514Z 5 Acked 1004 Write 6 Sent Done Done false
95622024-02-01T17:31:36.514Z 6 Acked 1005 Read 1 Sent Done Done false
95632024-02-01T17:31:36.514Z 7 Acked 1006 Read 1 Sent Done Done false
95642024-02-01T17:31:36.514Z 8 Acked 1007 Read 9 Sent Done Done false
95652024-02-01T17:31:36.514Z 9 Acked 1008 Read 10 Sent Done Done false
95662024-02-01T17:31:36.514Z 10 Acked 1009 Write 5 Sent Done Done false
95672024-02-01T17:31:36.515Z 11 Acked 1010 Read 8 Sent Done Done false
95682024-02-01T17:31:36.515Z 12 Acked 1011 Read 5 Sent Done Done false
95692024-02-01T17:31:36.515Z 13 Acked 1012 Read 9 Sent Done Done false
95702024-02-01T17:31:36.515Z 14 Acked 1013 Read 10 Sent Done Done false
95712024-02-01T17:31:36.515Z 15 Acked 1014 Write 7 Sent Done Done false
95722024-02-01T17:31:36.515Z 16 Acked 1015 Flush 0 Sent Done Done false
95732024-02-01T17:31:36.515Z 17 Acked 1016 Write 6 Sent Done Done false
95742024-02-01T17:31:36.515Z 18 Acked 1017 Flush 0 Sent Done Done false
95752024-02-01T17:31:36.515Z 19 Acked 1018 Write 6 Sent Done Done false
95762024-02-01T17:31:36.515Z 20 Acked 1019 Write 7 Sent Done Done false
95772024-02-01T17:31:36.515Z 21 Acked 1020 Read 9 Sent Done Done false
95782024-02-01T17:31:36.515Z 22 Acked 1021 Write 1 Sent Done Done false
95792024-02-01T17:31:36.515Z 23 Acked 1022 Read 6 Sent Done Done false
95802024-02-01T17:31:36.515Z 24 Acked 1023 Flush 0 Sent Done Done false
95812024-02-01T17:31:36.515Z 25 Acked 1024 Write 2 Sent Done Done false
95822024-02-01T17:31:36.516Z 26 Acked 1025 Read 4 Sent Done Done false
95832024-02-01T17:31:36.516Z 27 Acked 1026 Write 10 Sent Done Done false
95842024-02-01T17:31:36.516Z 28 Acked 1027 Read 3 Sent Done Done false
95852024-02-01T17:31:36.516Z 29 Acked 1028 Read 8 Sent Done Done false
95862024-02-01T17:31:36.516Z 30 Acked 1029 Read 6 Sent Sent Done false
95872024-02-01T17:31:36.516Z STATES DS:0 DS:1 DS:2 TOTAL
95882024-02-01T17:31:36.516Z New 0 0 0 0
95892024-02-01T17:31:36.516Z Sent 30 1 0 31
95902024-02-01T17:31:36.516Z Done 0 29 30 59
95912024-02-01T17:31:36.516Z Skipped 0 0 0 0
95922024-02-01T17:31:36.516Z Error 0 0 0 0
95932024-02-01T17:31:36.516ZLast Flush: 0 1023 1023
95942024-02-01T17:31:36.516ZDownstairs last five completed:
95952024-02-01T17:31:36.516ZUpstairs last five completed: 30 29 28 27 26
95962024-02-01T17:31:36.516Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:35.77444549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1481}
95972024-02-01T17:31:36.516ZWrote out file "/tmp/test_repair_verify.data"
95982024-02-01T17:31:36.516Z
95992024-02-01T17:31:36.523ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96002024-02-01T17:31:36.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96012024-02-01T17:31:36.540Z 0 000-019 86 86 86 128 128 128 F T T <---
96022024-02-01T17:31:36.545Z 1 020-039 86 86 86 128 128 128 F F F
96032024-02-01T17:31:36.545Z 2 040-059 86 88 88 128 132 132 F F F <---
96042024-02-01T17:31:36.545Z 3 060-079 80 88 88 121 132 132 F F F <---
96052024-02-01T17:31:36.545Z 4 080-099 86 86 86 128 128 128 F F F
96062024-02-01T17:31:36.545Z 5 100-119 86 86 86 128 128 128 F F F
96072024-02-01T17:31:36.545Z 6 120-139 83 88 88 124 130 130 F F F <---
96082024-02-01T17:31:36.545Z 7 140-159 83 83 83 124 124 124 F F F
96092024-02-01T17:31:36.545Z 8 160-179 86 86 86 128 128 128 F F F
96102024-02-01T17:31:36.545Z 9 180-199 76 76 76 115 115 115 F F F
96112024-02-01T17:31:36.546Z 10 200-219 86 88 88 128 131 131 F F F <---
96122024-02-01T17:31:36.546Z 11 220-239 86 88 88 128 130 130 F F F <---
96132024-02-01T17:31:36.546Z 12 240-259 86 86 86 128 128 128 F F F
96142024-02-01T17:31:36.546Z 13 260-279 84 84 84 125 125 125 F F F
96152024-02-01T17:31:36.546Z 14 280-299 84 84 84 125 125 125 F F F
96162024-02-01T17:31:36.546Z 15 300-319 86 88 88 128 130 130 F F F <---
96172024-02-01T17:31:36.546Z 16 320-339 86 86 86 128 128 128 F F F
96182024-02-01T17:31:36.546Z 17 340-359 85 88 88 127 132 132 F F F <---
96192024-02-01T17:31:36.546Z 18 360-379 85 88 88 127 132 132 F T T <---
96202024-02-01T17:31:36.546Z 19 380-399 85 85 85 127 127 127 F F F
96212024-02-01T17:31:36.546Z 20 400-419 86 88 88 128 130 130 F F F <---
96222024-02-01T17:31:36.546Z 21 420-439 86 86 86 128 128 128 F F F
96232024-02-01T17:31:36.547Z 22 440-459 86 86 86 128 128 128 F F F
96242024-02-01T17:31:36.547Z 23 460-479 84 84 84 125 125 125 F F F
96252024-02-01T17:31:36.547Z 24 480-499 84 84 84 125 125 125 F F F
96262024-02-01T17:31:36.547Z 25 500-519 86 86 86 128 128 128 F F F
96272024-02-01T17:31:36.547Z 26 520-539 86 86 86 128 128 128 F F F
96282024-02-01T17:31:36.547Z 27 540-559 83 88 88 124 130 130 F F F <---
96292024-02-01T17:31:36.547Z 28 560-579 86 86 86 128 128 128 F F F
96302024-02-01T17:31:36.547Z 29 580-599 86 86 86 128 128 128 F F F
96312024-02-01T17:31:36.547ZMax gen: 88, Max flush: 132
96322024-02-01T17:31:36.547ZError: Difference in extent metadata found!
96332024-02-01T17:31:36.547ZA Difference in extent metadata is expected here
96342024-02-01T17:31:36.547ZOn loop 44
96352024-02-01T17:31:36.547Z
96362024-02-01T17:31:36.552ZVerifying data now
96372024-02-01T17:31:41.521ZNo change after verify
96382024-02-01T17:31:41.521ZLoop: 44 Downstairs dump after verify (and repair):
96392024-02-01T17:31:41.544ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96402024-02-01T17:31:41.544Z 0 000-019 89 89 89 133 133 133 F F F
96412024-02-01T17:31:41.548Z 1 020-039 86 86 86 128 128 128 F F F
96422024-02-01T17:31:41.548Z 2 040-059 88 88 88 132 132 132 F F F
96432024-02-01T17:31:41.548Z 3 060-079 88 88 88 132 132 132 F F F
96442024-02-01T17:31:41.548Z 4 080-099 86 86 86 128 128 128 F F F
96452024-02-01T17:31:41.548Z 5 100-119 86 86 86 128 128 128 F F F
96462024-02-01T17:31:41.548Z 6 120-139 88 88 88 130 130 130 F F F
96472024-02-01T17:31:41.548Z 7 140-159 83 83 83 124 124 124 F F F
96482024-02-01T17:31:41.548Z 8 160-179 86 86 86 128 128 128 F F F
96492024-02-01T17:31:41.548Z 9 180-199 76 76 76 115 115 115 F F F
96502024-02-01T17:31:41.548Z 10 200-219 88 88 88 131 131 131 F F F
96512024-02-01T17:31:41.548Z 11 220-239 88 88 88 130 130 130 F F F
96522024-02-01T17:31:41.548Z 12 240-259 86 86 86 128 128 128 F F F
96532024-02-01T17:31:41.549Z 13 260-279 84 84 84 125 125 125 F F F
96542024-02-01T17:31:41.549Z 14 280-299 84 84 84 125 125 125 F F F
96552024-02-01T17:31:41.549Z 15 300-319 88 88 88 130 130 130 F F F
96562024-02-01T17:31:41.549Z 16 320-339 86 86 86 128 128 128 F F F
96572024-02-01T17:31:41.549Z 17 340-359 88 88 88 132 132 132 F F F
96582024-02-01T17:31:41.549Z 18 360-379 89 89 89 133 133 133 F F F
96592024-02-01T17:31:41.549Z 19 380-399 85 85 85 127 127 127 F F F
96602024-02-01T17:31:41.549Z 20 400-419 88 88 88 130 130 130 F F F
96612024-02-01T17:31:41.549Z 21 420-439 86 86 86 128 128 128 F F F
96622024-02-01T17:31:41.549Z 22 440-459 86 86 86 128 128 128 F F F
96632024-02-01T17:31:41.549Z 23 460-479 84 84 84 125 125 125 F F F
96642024-02-01T17:31:41.549Z 24 480-499 84 84 84 125 125 125 F F F
96652024-02-01T17:31:41.549Z 25 500-519 86 86 86 128 128 128 F F F
96662024-02-01T17:31:41.550Z 26 520-539 86 86 86 128 128 128 F F F
96672024-02-01T17:31:41.550Z 27 540-559 88 88 88 130 130 130 F F F
96682024-02-01T17:31:41.550Z 28 560-579 86 86 86 128 128 128 F F F
96692024-02-01T17:31:41.550Z 29 580-599 86 86 86 128 128 128 F F F
96702024-02-01T17:31:41.551ZMax gen: 89, Max flush: 133
96712024-02-01T17:31:41.554Z
96722024-02-01T17:31:41.554ZBegin loop 45
96732024-02-01T17:31:41.554ZDownstairs to restart: 2
96742024-02-01T17:31:41.579Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.850562258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
96752024-02-01T17:31:41.585Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851089878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
96762024-02-01T17:31:41.585Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
96772024-02-01T17:31:41.585Z,"time":"2024-02-01T17:31:40.851251697Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
96782024-02-01T17:31:41.585Z{"msg":"Crucible 5b4cb383-a151-48fc-adfc-3586edb4ad5e has session id: 88d99522-42e1-4907-8dd6-4c8e7223f0c7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851294627Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96792024-02-01T17:31:41.585Z{"msg":"Upstairs opts: Upstairs UUID: 5b4cb383-a151-48fc-adfc-3586edb4ad5e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851372455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96802024-02-01T17:31:41.585Z{"msg":"Crucible stats registered with UUID: 5b4cb383-a151-48fc-adfc-3586edb4ad5e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851443682Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96812024-02-01T17:31:41.585Z{"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851484316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
96822024-02-01T17:31:41.585Z{"msg":"5b4cb383-a151-48fc-adfc-3586edb4ad5e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851618013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96832024-02-01T17:31:41.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851732959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96842024-02-01T17:31:41.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851936551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96852024-02-01T17:31:41.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.851984555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96862024-02-01T17:31:41.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.85202559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96872024-02-01T17:31:41.586Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852167235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96882024-02-01T17:31:41.586Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.85224179Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96892024-02-01T17:31:41.586Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852345846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96902024-02-01T17:31:41.586Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852392353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96912024-02-01T17:31:41.586Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852496365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96922024-02-01T17:31:41.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852544161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96932024-02-01T17:31:41.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852590851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96942024-02-01T17:31:41.587Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.852631135Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"io task","client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96952024-02-01T17:31:41.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.853879719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96962024-02-01T17:31:41.587Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.853949646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96972024-02-01T17:31:41.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.853977335Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96982024-02-01T17:31:41.587Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.854053606Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
96992024-02-01T17:31:41.587Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.85410073Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97002024-02-01T17:31:41.587Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.854146915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97012024-02-01T17:31:41.587Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.854604505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97022024-02-01T17:31:41.587Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.854695412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97032024-02-01T17:31:41.588Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855006067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97042024-02-01T17:31:41.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855579769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97052024-02-01T17:31:41.593Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855667569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97062024-02-01T17:31:41.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855713259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97072024-02-01T17:31:41.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855780801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97082024-02-01T17:31:41.594Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855828276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97092024-02-01T17:31:41.594Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855909941Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97102024-02-01T17:31:41.594Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855957537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97112024-02-01T17:31:41.594Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.855985064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97122024-02-01T17:31:41.594Z{"msg":"[0]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 128, 115, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856030944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97132024-02-01T17:31:41.594Z{"msg":"[0]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 86, 76, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856056777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97142024-02-01T17:31:41.594Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856081784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97152024-02-01T17:31:41.594Z{"msg":"[1]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 128, 115, 131, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856153585Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97162024-02-01T17:31:41.594Z{"msg":"[1]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 86, 76, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856182183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97172024-02-01T17:31:41.594Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856205791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97182024-02-01T17:31:41.594Z{"msg":"[2]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 128, 115, 131, 130]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
97192024-02-01T17:31:41.595Z,"time":"2024-02-01T17:31:40.856230183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97202024-02-01T17:31:41.595Z{"msg":"[2]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 86, 76, 88, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856253762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97212024-02-01T17:31:41.595Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.85629668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97222024-02-01T17:31:41.596Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856320896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97232024-02-01T17:31:41.596Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856343605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97242024-02-01T17:31:41.596Z{"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856364754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97252024-02-01T17:31:41.596Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856386321Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"":"downstairs","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97262024-02-01T17:31:41.596Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856433765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97272024-02-01T17:31:41.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856455792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"0","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97282024-02-01T17:31:41.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856477387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"1","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97292024-02-01T17:31:41.597Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856519963Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"client":"2","":"downstairs","session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97302024-02-01T17:31:41.597Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856543912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97312024-02-01T17:31:41.597Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856564399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97322024-02-01T17:31:41.597Z{"msg":"5b4cb383-a151-48fc-adfc-3586edb4ad5e is now active with session: 88d99522-42e1-4907-8dd6-4c8e7223f0c7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856612626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97332024-02-01T17:31:41.597Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856634171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489,"session_id":"88d99522-42e1-4907-8dd6-4c8e7223f0c7"}
97342024-02-01T17:31:41.597Z{"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:40.856654534Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
97352024-02-01T17:31:41.597ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97362024-02-01T17:31:41.597ZLoading write count information from file "/tmp/test_repair_verify.data"
97372024-02-01T17:31:41.597ZRun Repair workload
97382024-02-01T17:31:41.597Z01/30 Write block 484 len 4096 data: 9
97392024-02-01T17:31:41.598Z02/30 Read block 590 len 8192
97402024-02-01T17:31:41.598Z03/30 Read block 254 len 16384
97412024-02-01T17:31:41.598Z04/30 Write block 566 len 12288 data: 5 4 4
97422024-02-01T17:31:41.598Z05/30 Read block 3 len 20480
97432024-02-01T17:31:41.602Z06/30 Write block 496 len 28672 data: 9 7 8 7 5 5 5
97442024-02-01T17:31:41.602Z07/30 Write block 168 len 24576 data: 7 9 10 11 11 11
97452024-02-01T17:31:41.602Z08/30 Read block 349 len 24576
97462024-02-01T17:31:41.602Z09/30 Read block 259 len 32768
97472024-02-01T17:31:41.602Z10/30 Read block 578 len 16384
97482024-02-01T17:31:41.602Z11/30 Write block 547 len 40960 data: 11 8 7 7 9 9 7 5 8 7
97492024-02-01T17:31:41.602Z12/30 Read block 547 len 8192
97502024-02-01T17:31:41.606Z13/30 Read block 367 len 4096
97512024-02-01T17:31:41.606Z14/30 Read block 54 len 8192
97522024-02-01T17:31:41.606Z15/30 Write block 471 len 28672 data: 8 8 6 4 6 6 5
97532024-02-01T17:31:41.619Z16/30 Write block 200 len 28672 data: 7 8 10 12 12 11 10
97542024-02-01T17:31:41.619Z17/30 Read block 377 len 28672
97552024-02-01T17:31:41.619Z18/30 Read block 546 len 40960
97562024-02-01T17:31:41.624Z19/30 Read block 433 len 36864
97572024-02-01T17:31:41.624Z20/30 Read block 125 len 12288
97582024-02-01T17:31:41.624Z21/30 Read block 286 len 12288
97592024-02-01T17:31:41.624Z22/30 Read block 99 len 12288
97602024-02-01T17:31:41.624Z23/30 Read block 299 len 28672
97612024-02-01T17:31:41.624Z24/30 Read block 78 len 24576
97622024-02-01T17:31:41.624Z25/30 Write block 492 len 16384 data: 12 9 7 9
97632024-02-01T17:31:41.624Z26/30 Write block 425 len 36864 data: 10 11 7 6 5 7 7 8 10
97642024-02-01T17:31:41.624Z27/30 Read block 441 len 12288
97652024-02-01T17:31:41.628Z28/30 Write block 263 len 28672 data: 4 4 6 7 7 7 7
97662024-02-01T17:31:41.628Z29/30 Read block 210 len 8192
97672024-02-01T17:31:41.628Z30/30 Read block 105 len 12288
97682024-02-01T17:31:41.631Z----------------------------------------------------------------
97692024-02-01T17:31:41.631Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
97702024-02-01T17:31:41.632ZStates: Active Active Active
97712024-02-01T17:31:41.632ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
97722024-02-01T17:31:41.632Z 1 Acked 1000 Write 1 Done Done Done false
97732024-02-01T17:31:41.632Z 2 Acked 1001 Read 2 Done Done Done false
97742024-02-01T17:31:41.632Z 3 Acked 1002 Read 4 Done Done Sent false
97752024-02-01T17:31:41.632Z 4 Acked 1003 Write 3 Done Done Sent false
97762024-02-01T17:31:41.632Z 5 Acked 1004 Read 5 Done Done Sent false
97772024-02-01T17:31:41.632Z 6 Acked 1005 Write 7 Done Done Sent false
97782024-02-01T17:31:41.632Z 7 Acked 1006 Write 6 Done Done Sent false
97792024-02-01T17:31:41.632Z 8 Acked 1007 Read 6 Done Done Sent false
97802024-02-01T17:31:41.632Z 9 Acked 1008 Read 8 Done Done Sent false
97812024-02-01T17:31:41.632Z 10 Acked 1009 Read 4 Done Done Sent false
97822024-02-01T17:31:41.632Z 11 Acked 1010 Write 10 Done Done Sent false
97832024-02-01T17:31:41.632Z 12 Acked 1011 Read 2 Done Done Sent false
97842024-02-01T17:31:41.632Z 13 Acked 1012 Read 1 Done Done Sent false
97852024-02-01T17:31:41.632Z 14 Acked 1013 Read 2 Done Done Sent false
97862024-02-01T17:31:41.632Z 15 Acked 1014 Write 7 Done Done Sent false
97872024-02-01T17:31:41.632Z 16 Acked 1015 Write 7 Done Done Sent false
97882024-02-01T17:31:41.632Z 17 Acked 1016 Read 7 Done Done Sent false
97892024-02-01T17:31:41.632Z 18 Acked 1017 Read 10 Done Done Sent false
97902024-02-01T17:31:41.632Z 19 Acked 1018 Read 9 Done Done Sent false
97912024-02-01T17:31:41.632Z 20 Acked 1019 Read 3 Done Done Sent false
97922024-02-01T17:31:41.633Z 21 Acked 1020 Read 3 Done Done Sent false
97932024-02-01T17:31:41.633Z 22 Acked 1021 Read 3 Done Done Sent false
97942024-02-01T17:31:41.633Z 23 Acked 1022 Read 7 Done Done Sent false
97952024-02-01T17:31:41.633Z 24 Acked 1023 Read 6 Done Done Sent false
97962024-02-01T17:31:41.633Z 25 Acked 1024 Write 4 Done Done Sent false
97972024-02-01T17:31:41.633Z 26 Acked 1025 Write 9 Done Done Sent false
97982024-02-01T17:31:41.633Z 27 Acked 1026 Read 3 Done Done Sent false
97992024-02-01T17:31:41.633Z 28 Acked 1027 Write 7 Done Done Sent false
98002024-02-01T17:31:41.633Z 29 Acked 1028 Read 2 Done Done Sent false
98012024-02-01T17:31:41.634Z 30 Acked 1029 Read 3 Done Done Sent false
98022024-02-01T17:31:41.634Z STATES DS:0 DS:1 DS:2 TOTAL
98032024-02-01T17:31:41.634Z New 0 0 0 0
98042024-02-01T17:31:41.634Z Sent 0 0 28 28
98052024-02-01T17:31:41.634Z Done 30 30 2 62
98062024-02-01T17:31:41.634Z Skipped 0 0 0 0
98072024-02-01T17:31:41.634Z Error 0 0 0 0
98082024-02-01T17:31:41.634ZLast Flush: 0 0 0
98092024-02-01T17:31:41.634ZDownstairs last five completed:
98102024-02-01T17:31:41.634ZUpstairs last five completed: 30 29 28 27 26
98112024-02-01T17:31:41.634Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:40.900248481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1489}
98122024-02-01T17:31:41.634ZWrote out file "/tmp/test_repair_verify.data"
98132024-02-01T17:31:41.637Z
98142024-02-01T17:31:41.642ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98152024-02-01T17:31:41.669ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98162024-02-01T17:31:41.670Z 0 000-019 89 89 89 133 133 133 F F F
98172024-02-01T17:31:41.670Z 1 020-039 86 86 86 128 128 128 F F F
98182024-02-01T17:31:41.670Z 2 040-059 88 88 88 132 132 132 F F F
98192024-02-01T17:31:41.670Z 3 060-079 88 88 88 132 132 132 F F F
98202024-02-01T17:31:41.673Z 4 080-099 86 86 86 128 128 128 F F F
98212024-02-01T17:31:41.673Z 5 100-119 86 86 86 128 128 128 F F F
98222024-02-01T17:31:41.673Z 6 120-139 88 88 88 130 130 130 F F F
98232024-02-01T17:31:41.673Z 7 140-159 83 83 83 124 124 124 F F F
98242024-02-01T17:31:41.673Z 8 160-179 86 86 86 128 128 128 T T F <---
98252024-02-01T17:31:41.673Z 9 180-199 76 76 76 115 115 115 F F F
98262024-02-01T17:31:41.673Z 10 200-219 88 88 88 131 131 131 T T F <---
98272024-02-01T17:31:41.673Z 11 220-239 88 88 88 130 130 130 F F F
98282024-02-01T17:31:41.673Z 12 240-259 86 86 86 128 128 128 F F F
98292024-02-01T17:31:41.673Z 13 260-279 84 84 84 125 125 125 T T F <---
98302024-02-01T17:31:41.673Z 14 280-299 84 84 84 125 125 125 F F F
98312024-02-01T17:31:41.674Z 15 300-319 88 88 88 130 130 130 F F F
98322024-02-01T17:31:41.675Z 16 320-339 86 86 86 128 128 128 F F F
98332024-02-01T17:31:41.675Z 17 340-359 88 88 88 132 132 132 F F F
98342024-02-01T17:31:41.675Z 18 360-379 89 89 89 133 133 133 F F F
98352024-02-01T17:31:41.675Z 19 380-399 85 85 85 127 127 127 F F F
98362024-02-01T17:31:41.675Z 20 400-419 88 88 88 130 130 130 F F F
98372024-02-01T17:31:41.675Z 21 420-439 86 86 86 128 128 128 T T F <---
98382024-02-01T17:31:41.675Z 22 440-459 86 86 86 128 128 128 F F F
98392024-02-01T17:31:41.675Z 23 460-479 84 84 84 125 125 125 T T F <---
98402024-02-01T17:31:41.675Z 24 480-499 84 84 84 125 125 125 T T T
98412024-02-01T17:31:41.675Z 25 500-519 86 86 86 128 128 128 T T F <---
98422024-02-01T17:31:41.676Z 26 520-539 86 86 86 128 128 128 F F F
98432024-02-01T17:31:41.676Z 27 540-559 88 88 88 130 130 130 T T F <---
98442024-02-01T17:31:41.676Z 28 560-579 86 86 86 128 128 128 T T F <---
98452024-02-01T17:31:41.676Z 29 580-599 86 86 86 128 128 128 F F F
98462024-02-01T17:31:41.676ZMax gen: 89, Max flush: 133
98472024-02-01T17:31:41.676ZError: Difference in extent metadata found!
98482024-02-01T17:31:41.682ZA Difference in extent metadata is expected here
98492024-02-01T17:31:41.682ZOn loop 45
98502024-02-01T17:31:41.682Z
98512024-02-01T17:31:41.682ZVerifying data now
98522024-02-01T17:31:46.518ZNo change after verify
98532024-02-01T17:31:46.519ZLoop: 45 Downstairs dump after verify (and repair):
98542024-02-01T17:31:46.541ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98552024-02-01T17:31:46.541Z 0 000-019 89 89 89 133 133 133 F F F
98562024-02-01T17:31:46.542Z 1 020-039 86 86 86 128 128 128 F F F
98572024-02-01T17:31:46.542Z 2 040-059 88 88 88 132 132 132 F F F
98582024-02-01T17:31:46.542Z 3 060-079 88 88 88 132 132 132 F F F
98592024-02-01T17:31:46.546Z 4 080-099 86 86 86 128 128 128 F F F
98602024-02-01T17:31:46.546Z 5 100-119 86 86 86 128 128 128 F F F
98612024-02-01T17:31:46.546Z 6 120-139 88 88 88 130 130 130 F F F
98622024-02-01T17:31:46.547Z 7 140-159 83 83 83 124 124 124 F F F
98632024-02-01T17:31:46.547Z 8 160-179 90 90 90 134 134 134 F F F
98642024-02-01T17:31:46.547Z 9 180-199 76 76 76 115 115 115 F F F
98652024-02-01T17:31:46.547Z 10 200-219 90 90 90 134 134 134 F F F
98662024-02-01T17:31:46.547Z 11 220-239 88 88 88 130 130 130 F F F
98672024-02-01T17:31:46.547Z 12 240-259 86 86 86 128 128 128 F F F
98682024-02-01T17:31:46.548Z 13 260-279 90 90 90 134 134 134 F F F
98692024-02-01T17:31:46.548Z 14 280-299 84 84 84 125 125 125 F F F
98702024-02-01T17:31:46.548Z 15 300-319 88 88 88 130 130 130 F F F
98712024-02-01T17:31:46.548Z 16 320-339 86 86 86 128 128 128 F F F
98722024-02-01T17:31:46.548Z 17 340-359 88 88 88 132 132 132 F F F
98732024-02-01T17:31:46.548Z 18 360-379 89 89 89 133 133 133 F F F
98742024-02-01T17:31:46.548Z 19 380-399 85 85 85 127 127 127 F F F
98752024-02-01T17:31:46.548Z 20 400-419 88 88 88 130 130 130 F F F
98762024-02-01T17:31:46.548Z 21 420-439 90 90 90 134 134 134 F F F
98772024-02-01T17:31:46.548Z 22 440-459 86 86 86 128 128 128 F F F
98782024-02-01T17:31:46.548Z 23 460-479 90 90 90 134 134 134 F F F
98792024-02-01T17:31:46.548Z 24 480-499 90 90 90 134 134 134 F F F
98802024-02-01T17:31:46.548Z 25 500-519 90 90 90 134 134 134 F F F
98812024-02-01T17:31:46.549Z 26 520-539 86 86 86 128 128 128 F F F
98822024-02-01T17:31:46.549Z 27 540-559 90 90 90 134 134 134 F F F
98832024-02-01T17:31:46.549Z 28 560-579 90 90 90 134 134 134 F F F
98842024-02-01T17:31:46.549Z 29 580-599 86 86 86 128 128 128 F F F
98852024-02-01T17:31:46.549ZMax gen: 90, Max flush: 134
98862024-02-01T17:31:46.549Z
98872024-02-01T17:31:46.550ZBegin loop 46
98882024-02-01T17:31:46.550ZDownstairs to restart: 0
98892024-02-01T17:31:46.576Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.84754685Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
98902024-02-01T17:31:46.583Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848100473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
98912024-02-01T17:31:46.583Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848199998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
98922024-02-01T17:31:46.583ZCrucible runtime is spawned
98932024-02-01T17:31:46.583Z{"msg":"Crucible a839b54d-0e4d-4c08-9104-d6931c8a8741 has session id: 05a816c0-4070-42c5-a2af-b3ffe4ddd8f7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848523765Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
98942024-02-01T17:31:46.583Z{"msg":"Upstairs opts: Upstairs UUID: a839b54d-0e4d-4c08-9104-d6931c8a8741, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848618112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
98952024-02-01T17:31:46.583Z{"msg":"Crucible stats registered with UUID: a839b54d-0e4d-4c08-9104-d6931c8a8741","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848705605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
98962024-02-01T17:31:46.583Z{"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.84877495Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
98972024-02-01T17:31:46.583Z{"msg":"a839b54d-0e4d-4c08-9104-d6931c8a8741 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848817292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
98982024-02-01T17:31:46.584Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.848915851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
98992024-02-01T17:31:46.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849111519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99002024-02-01T17:31:46.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849189742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99012024-02-01T17:31:46.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849220675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99022024-02-01T17:31:46.585Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849497114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99032024-02-01T17:31:46.585Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849602946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99042024-02-01T17:31:46.586Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849670009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99052024-02-01T17:31:46.586Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849715712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99062024-02-01T17:31:46.586Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849780564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99072024-02-01T17:31:46.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849870747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99082024-02-01T17:31:46.586Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.849916719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99092024-02-01T17:31:46.591Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850068129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"io task","client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99102024-02-01T17:31:46.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850577446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99112024-02-01T17:31:46.591Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.85064588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99122024-02-01T17:31:46.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850673534Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99132024-02-01T17:31:46.591Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850697572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99142024-02-01T17:31:46.591Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850757414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99152024-02-01T17:31:46.592Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.850871167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99162024-02-01T17:31:46.592Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.851558565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99172024-02-01T17:31:46.592Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.851627878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99182024-02-01T17:31:46.592Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.851697957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99192024-02-01T17:31:46.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852495732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99202024-02-01T17:31:46.593Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.85257455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99212024-02-01T17:31:46.593Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852603616Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99222024-02-01T17:31:46.593Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852660254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99232024-02-01T17:31:46.593Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852688684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99242024-02-01T17:31:46.599Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852747519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99252024-02-01T17:31:46.599Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852777077Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99262024-02-01T17:31:46.599Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852813163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99272024-02-01T17:31:46.599Z{"msg":"[0]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 134, 115, 134, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852844556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99282024-02-01T17:31:46.599Z{"msg":"[0]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 90, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852900942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99292024-02-01T17:31:46.600Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.85293544Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99302024-02-01T17:31:46.600Z{"msg":"[1]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 134, 115, 134, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.852995044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99312024-02-01T17:31:46.601Z{"msg":"[1]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 90, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853025728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99322024-02-01T17:31:46.601Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853092045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99332024-02-01T17:31:46.601Z{"msg":"[2]R flush_numbers[0..12]: [133, 128, 132, 132, 128, 128, 130, 124, 134, 115, 134, 130]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
99342024-02-01T17:31:46.601Z,"time":"2024-02-01T17:31:45.853120388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99352024-02-01T17:31:46.601Z{"msg":"[2]R generation[0..12]: [89, 86, 88, 88, 86, 86, 88, 83, 90, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853244112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99362024-02-01T17:31:46.601Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853280796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99372024-02-01T17:31:46.601Z{"msg":"Max found gen is 91","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853343234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99382024-02-01T17:31:46.601Z{"msg":"Generation requested: 92 >= found:91","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853369734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99392024-02-01T17:31:46.602Z{"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853394436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99402024-02-01T17:31:46.602Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853451214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"":"downstairs","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99412024-02-01T17:31:46.602Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853482052Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99422024-02-01T17:31:46.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.85353636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"0","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99432024-02-01T17:31:46.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853562905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"1","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99442024-02-01T17:31:46.602Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853623341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"client":"2","":"downstairs","session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99452024-02-01T17:31:46.602Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853650748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99462024-02-01T17:31:46.602Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853706116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99472024-02-01T17:31:46.602Z{"msg":"a839b54d-0e4d-4c08-9104-d6931c8a8741 is now active with session: 05a816c0-4070-42c5-a2af-b3ffe4ddd8f7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853734853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99482024-02-01T17:31:46.603Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853796493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497,"session_id":"05a816c0-4070-42c5-a2af-b3ffe4ddd8f7"}
99492024-02-01T17:31:46.603Z{"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:45.853821374Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
99502024-02-01T17:31:46.603ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
99512024-02-01T17:31:46.603ZLoading write count information from file "/tmp/test_repair_verify.data"
99522024-02-01T17:31:46.603ZRun Repair workload
99532024-02-01T17:31:46.603Z01/30 Write block 300 len 4096 data: 10
99542024-02-01T17:31:46.603Z02/30 Read block 193 len 16384
99552024-02-01T17:31:46.603Z03/30 Read block 30 len 8192
99562024-02-01T17:31:46.603Z04/30 Write block 75 len 40960 data: 10 5 4 3 3 4 3 3 5 4
99572024-02-01T17:31:46.603Z05/30 Write block 377 len 36864 data: 12 10 11 11 12 8 8 7 6
99582024-02-01T17:31:46.603Z06/30 Write block 432 len 8192 data: 9 11
99592024-02-01T17:31:46.603Z07/30 Read block 228 len 28672
99602024-02-01T17:31:46.603Z08/30 Read block 217 len 40960
99612024-02-01T17:31:46.604Z09/30 Read block 526 len 32768
99622024-02-01T17:31:46.608Z10/30 Read block 141 len 36864
99632024-02-01T17:31:46.608Z11/30 Read block 16 len 24576
99642024-02-01T17:31:46.608Z12/30 Write block 110 len 32768 data: 4 5 5 6 6 6 5 4
99652024-02-01T17:31:46.608Z13/30 Read block 412 len 36864
99662024-02-01T17:31:46.608Z14/30 Write block 508 len 40960 data: 5 6 5 6 6 6 4 4 3 4
99672024-02-01T17:31:46.608Z15/30 Read block 235 len 4096
99682024-02-01T17:31:46.608Z16/30 Write block 469 len 4096 data: 6
99692024-02-01T17:31:46.611Z17/30 Read block 424 len 20480
99702024-02-01T17:31:46.611Z18/30 Read block 210 len 32768
99712024-02-01T17:31:46.611Z19/30 Flush
99722024-02-01T17:31:46.616Z20/30 Write block 69 len 24576 data: 10 8 10 11 11 11
99732024-02-01T17:31:46.619Z21/30 Read block 256 len 32768
99742024-02-01T17:31:46.619Z22/30 Read block 305 len 12288
99752024-02-01T17:31:46.622Z23/30 Write block 29 len 20480 data: 5 6 6 7 6
99762024-02-01T17:31:46.622Z24/30 Write block 445 len 8192 data: 10 8
99772024-02-01T17:31:46.625Z25/30 Write block 154 len 40960 data: 7 9 7 6 8 7 8 10 9 7
99782024-02-01T17:31:46.625Z26/30 Read block 444 len 8192
99792024-02-01T17:31:46.625Z27/30 Read block 323 len 8192
99802024-02-01T17:31:46.629Z28/30 Read block 20 len 16384
99812024-02-01T17:31:46.629Z29/30 Write block 322 len 40960 data: 8 5 6 3 5 6 5 5 6 8
99822024-02-01T17:31:46.629Z30/30 Read block 58 len 20480
99832024-02-01T17:31:46.634Z----------------------------------------------------------------
99842024-02-01T17:31:46.634Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
99852024-02-01T17:31:46.634ZStates: Active Active Active
99862024-02-01T17:31:46.634ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99872024-02-01T17:31:46.634Z 1 Acked 1000 Write 1 Done Done Done false
99882024-02-01T17:31:46.634Z 2 Acked 1001 Read 4 Done Done Done false
99892024-02-01T17:31:46.634Z 3 Acked 1002 Read 2 Done Done Done false
99902024-02-01T17:31:46.634Z 4 Acked 1003 Write 10 Done Done Done false
99912024-02-01T17:31:46.634Z 5 Acked 1004 Write 9 Done Done Done false
99922024-02-01T17:31:46.634Z 6 Acked 1005 Write 2 Done Done Done false
99932024-02-01T17:31:46.635Z 7 Acked 1006 Read 7 Done Done Done false
99942024-02-01T17:31:46.635Z 8 Acked 1007 Read 10 Done Done Done false
99952024-02-01T17:31:46.635Z 9 Acked 1008 Read 8 Done Done Done false
99962024-02-01T17:31:46.636Z 10 Acked 1009 Read 9 Done Done Done false
99972024-02-01T17:31:46.636Z 11 Acked 1010 Read 6 Done Done Done false
99982024-02-01T17:31:46.636Z 12 Acked 1011 Write 8 Done Done Done false
99992024-02-01T17:31:46.636Z 13 Acked 1012 Read 9 Done Done Done false
100002024-02-01T17:31:46.636Z 14 Acked 1013 Write 10 Sent Done Done false
100012024-02-01T17:31:46.636Z 15 Acked 1014 Read 1 Sent Done Done false
100022024-02-01T17:31:46.636Z 16 Acked 1015 Write 1 Sent Done Done false
100032024-02-01T17:31:46.636Z 17 Acked 1016 Read 5 Sent Done Done false
100042024-02-01T17:31:46.636Z 18 Acked 1017 Read 8 Sent Done Done false
100052024-02-01T17:31:46.636Z 19 Acked 1018 Flush 0 Sent Done Done false
100062024-02-01T17:31:46.637Z 20 Acked 1019 Write 6 Sent Done Done false
100072024-02-01T17:31:46.637Z 21 Acked 1020 Read 8 Sent Done Done false
100082024-02-01T17:31:46.637Z 22 Acked 1021 Read 3 Sent Done Done false
100092024-02-01T17:31:46.637Z 23 Acked 1022 Write 5 Sent Done Done false
100102024-02-01T17:31:46.637Z 24 Acked 1023 Write 2 Sent Done Done false
100112024-02-01T17:31:46.637Z 25 Acked 1024 Write 10 Sent Done Done false
100122024-02-01T17:31:46.637Z 26 Acked 1025 Read 2 Sent Done Done false
100132024-02-01T17:31:46.637Z 27 Acked 1026 Read 2 Sent Done Done false
100142024-02-01T17:31:46.637Z 28 Acked 1027 Read 4 Sent Done Done false
100152024-02-01T17:31:46.637Z 29 Acked 1028 Write 10 Sent Done Done false
100162024-02-01T17:31:46.637Z 30 Acked 1029 Read 5 Sent Done Done false
100172024-02-01T17:31:46.637Z STATES DS:0 DS:1 DS:2 TOTAL
100182024-02-01T17:31:46.637Z New 0 0 0 0
100192024-02-01T17:31:46.638Z Sent 17 0 0 17
100202024-02-01T17:31:46.638Z Done 13 30 30 73
100212024-02-01T17:31:46.638Z Skipped 0 0 0 0
100222024-02-01T17:31:46.638Z Error 0 0 0 0
100232024-02-01T17:31:46.638ZLast Flush: 0 1018 1018
100242024-02-01T17:31:46.638ZDownstairs last five completed:
100252024-02-01T17:31:46.638ZUpstairs last five completed: 30 29 28 27 26
100262024-02-01T17:31:46.638Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:45.90349712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1497}
100272024-02-01T17:31:46.638ZWrote out file "/tmp/test_repair_verify.data"
100282024-02-01T17:31:46.643Z
100292024-02-01T17:31:46.646ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
100302024-02-01T17:31:46.673ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100312024-02-01T17:31:46.673Z 0 000-019 89 89 89 133 133 133 F F F
100322024-02-01T17:31:46.673Z 1 020-039 86 86 86 128 128 128 F T T <---
100332024-02-01T17:31:46.673Z 2 040-059 88 88 88 132 132 132 F F F
100342024-02-01T17:31:46.673Z 3 060-079 88 92 92 132 135 135 T T T <---
100352024-02-01T17:31:46.676Z 4 080-099 86 92 92 128 135 135 T F F <---
100362024-02-01T17:31:46.676Z 5 100-119 86 92 92 128 135 135 T F F <---
100372024-02-01T17:31:46.676Z 6 120-139 88 88 88 130 130 130 F F F
100382024-02-01T17:31:46.676Z 7 140-159 83 83 83 124 124 124 F T T <---
100392024-02-01T17:31:46.677Z 8 160-179 90 90 90 134 134 134 F T T <---
100402024-02-01T17:31:46.677Z 9 180-199 76 76 76 115 115 115 F F F
100412024-02-01T17:31:46.678Z 10 200-219 90 90 90 134 134 134 F F F
100422024-02-01T17:31:46.678Z 11 220-239 88 88 88 130 130 130 F F F
100432024-02-01T17:31:46.678Z 12 240-259 86 86 86 128 128 128 F F F
100442024-02-01T17:31:46.678Z 13 260-279 90 90 90 134 134 134 F F F
100452024-02-01T17:31:46.678Z 14 280-299 84 84 84 125 125 125 F F F
100462024-02-01T17:31:46.678Z 15 300-319 88 92 92 130 135 135 T F F <---
100472024-02-01T17:31:46.678Z 16 320-339 86 86 86 128 128 128 F T T <---
100482024-02-01T17:31:46.678Z 17 340-359 88 88 88 132 132 132 F F F
100492024-02-01T17:31:46.678Z 18 360-379 89 92 92 133 135 135 T F F <---
100502024-02-01T17:31:46.678Z 19 380-399 85 92 92 127 135 135 T F F <---
100512024-02-01T17:31:46.678Z 20 400-419 88 88 88 130 130 130 F F F
100522024-02-01T17:31:46.679Z 21 420-439 90 92 92 134 135 135 T F F <---
100532024-02-01T17:31:46.679Z 22 440-459 86 86 86 128 128 128 F T T <---
100542024-02-01T17:31:46.679Z 23 460-479 90 92 92 134 135 135 F F F <---
100552024-02-01T17:31:46.679Z 24 480-499 90 90 90 134 134 134 F F F
100562024-02-01T17:31:46.679Z 25 500-519 90 92 92 134 135 135 F F F <---
100572024-02-01T17:31:46.679Z 26 520-539 86 86 86 128 128 128 F F F
100582024-02-01T17:31:46.679Z 27 540-559 90 90 90 134 134 134 F F F
100592024-02-01T17:31:46.679Z 28 560-579 90 90 90 134 134 134 F F F
100602024-02-01T17:31:46.679Z 29 580-599 86 86 86 128 128 128 F F F
100612024-02-01T17:31:46.679ZMax gen: 92, Max flush: 135
100622024-02-01T17:31:46.679ZError: Difference in extent metadata found!
100632024-02-01T17:31:46.683ZA Difference in extent metadata is expected here
100642024-02-01T17:31:46.684ZOn loop 46
100652024-02-01T17:31:46.684Z
100662024-02-01T17:31:46.684ZVerifying data now
100672024-02-01T17:31:53.011ZNo change after verify
100682024-02-01T17:31:53.012ZLoop: 46 Downstairs dump after verify (and repair):
100692024-02-01T17:31:53.777ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100702024-02-01T17:31:53.777Z 0 000-019 89 89 89 133 133 133 F F F
100712024-02-01T17:31:53.778Z 1 020-039 93 93 93 136 136 136 F F F
100722024-02-01T17:31:53.779Z 2 040-059 88 88 88 132 132 132 F F F
100732024-02-01T17:31:53.779Z 3 060-079 93 93 93 136 136 136 F F F
100742024-02-01T17:31:53.779Z 4 080-099 92 92 92 135 135 135 F F F
100752024-02-01T17:31:53.780Z 5 100-119 92 92 92 135 135 135 F F F
100762024-02-01T17:31:53.780Z 6 120-139 88 88 88 130 130 130 F F F
100772024-02-01T17:31:53.780Z 7 140-159 93 93 93 136 136 136 F F F
100782024-02-01T17:31:53.780Z 8 160-179 93 93 93 136 136 136 F F F
100792024-02-01T17:31:53.780Z 9 180-199 76 76 76 115 115 115 F F F
100802024-02-01T17:31:53.781Z 10 200-219 90 90 90 134 134 134 F F F
100812024-02-01T17:31:53.827Z 11 220-239 88 88 88 130 130 130 F F F
100822024-02-01T17:31:53.828Z 12 240-259 86 86 86 128 128 128 F F F
100832024-02-01T17:31:53.828Z 13 260-279 90 90 90 134 134 134 F F F
100842024-02-01T17:31:53.828Z 14 280-299 84 84 84 125 125 125 F F F
100852024-02-01T17:31:53.828Z 15 300-319 92 92 92 135 135 135 F F F
100862024-02-01T17:31:53.828Z 16 320-339 93 93 93 136 136 136 F F F
100872024-02-01T17:31:53.828Z 17 340-359 88 88 88 132 132 132 F F F
100882024-02-01T17:31:53.828Z 18 360-379 92 92 92 135 135 135 F F F
100892024-02-01T17:31:53.828Z 19 380-399 92 92 92 135 135 135 F F F
100902024-02-01T17:31:53.828Z 20 400-419 88 88 88 130 130 130 F F F
100912024-02-01T17:31:53.828Z 21 420-439 92 92 92 135 135 135 F F F
100922024-02-01T17:31:53.828Z 22 440-459 93 93 93 136 136 136 F F F
100932024-02-01T17:31:53.828Z 23 460-479 92 92 92 135 135 135 F F F
100942024-02-01T17:31:53.828Z 24 480-499 90 90 90 134 134 134 F F F
100952024-02-01T17:31:53.828Z 25 500-519 92 92 92 135 135 135 F F F
100962024-02-01T17:31:53.828Z 26 520-539 86 86 86 128 128 128 F F F
100972024-02-01T17:31:53.828Z 27 540-559 90 90 90 134 134 134 F F F
100982024-02-01T17:31:53.829Z 28 560-579 90 90 90 134 134 134 F F F
100992024-02-01T17:31:53.829Z 29 580-599 86 86 86 128 128 128 F F F
101002024-02-01T17:31:53.829ZMax gen: 93, Max flush: 136
101012024-02-01T17:31:53.829Z
101022024-02-01T17:31:53.829ZBegin loop 47
101032024-02-01T17:31:53.829ZDownstairs to restart: 2
101042024-02-01T17:31:53.829Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.195233557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
101052024-02-01T17:31:53.829Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.195741614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
101062024-02-01T17:31:53.849Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.19581507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
101072024-02-01T17:31:53.849Z{"msg":"Crucible e36f0790-34a0-4bf2-b12b-77335ac0293c has session id: debfdc6c-5160-4f6a-b669-5a166cc2d516","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.195858552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"Crucible runtime is spawned
101082024-02-01T17:31:53.849Zsession_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101092024-02-01T17:31:53.849Z{"msg":"Upstairs opts: Upstairs UUID: e36f0790-34a0-4bf2-b12b-77335ac0293c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.195995236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101102024-02-01T17:31:53.849Z{"msg":"Crucible stats registered with UUID: e36f0790-34a0-4bf2-b12b-77335ac0293c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196075497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101112024-02-01T17:31:53.849Z{"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196143543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
101122024-02-01T17:31:53.849Z{"msg":"e36f0790-34a0-4bf2-b12b-77335ac0293c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196365743Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101132024-02-01T17:31:53.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196444396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101142024-02-01T17:31:53.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196526993Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101152024-02-01T17:31:53.849Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196576453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101162024-02-01T17:31:53.849Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196620642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101172024-02-01T17:31:53.850Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196694533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101182024-02-01T17:31:53.850Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196738192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101192024-02-01T17:31:53.850Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196826185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101202024-02-01T17:31:53.850Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196875737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101212024-02-01T17:31:53.850Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.196966403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101222024-02-01T17:31:53.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.197204812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101232024-02-01T17:31:53.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.197307485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101242024-02-01T17:31:53.850Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.197355729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"io task","client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101252024-02-01T17:31:53.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198455668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101262024-02-01T17:31:53.850Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198527091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101272024-02-01T17:31:53.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198687116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101282024-02-01T17:31:53.850Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198756809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101292024-02-01T17:31:53.850Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198897373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101302024-02-01T17:31:53.850Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.198958327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101312024-02-01T17:31:53.851Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.199493066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101322024-02-01T17:31:53.851Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.199795316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101332024-02-01T17:31:53.851Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.19986377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101342024-02-01T17:31:53.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200364196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101352024-02-01T17:31:53.851Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200422447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101362024-02-01T17:31:53.851Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200448565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101372024-02-01T17:31:53.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200779318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101382024-02-01T17:31:53.851Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200835832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101392024-02-01T17:31:53.851Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.20086175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101402024-02-01T17:31:53.851Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200882964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101412024-02-01T17:31:53.851Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200923952Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101422024-02-01T17:31:53.851Z{"msg":"[0]R flush_numbers[0..12]: [133, 136, 132, 136, 135, 135, 130, 136, 136, 115, 134, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200949599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101432024-02-01T17:31:53.851Z{"msg":"[0]R generation[0..12]: [89, 93, 88, 93, 92, 92, 88, 93, 93, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.200991154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101442024-02-01T17:31:53.851Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201073744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101452024-02-01T17:31:53.851Z{"msg":"[1]R flush_numbers[0..12]: [133, 136, 132, 136, 135, 135, 130, 136, 136, 115, 134, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201100762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101462024-02-01T17:31:53.867Z{"msg":"[1]R generation[0..12]: [89, 93, 88, 93, 92, 92, 88, 93, 93, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201146337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101472024-02-01T17:31:53.867Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
101482024-02-01T17:31:53.867Z,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201179029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101492024-02-01T17:31:53.867Z{"msg":"[2]R flush_numbers[0..12]: [133, 136, 132, 136, 135, 135, 130, 136, 136, 115, 134, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201203148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101502024-02-01T17:31:53.867Z{"msg":"[2]R generation[0..12]: [89, 93, 88, 93, 92, 92, 88, 93, 93, 76, 90, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201244116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101512024-02-01T17:31:53.867Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201269528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101522024-02-01T17:31:53.867Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201293353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101532024-02-01T17:31:53.868Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201315521Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101542024-02-01T17:31:53.868Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201354882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101552024-02-01T17:31:53.868Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201378394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"":"downstairs","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101562024-02-01T17:31:53.868Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.20141917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101572024-02-01T17:31:53.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201441067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"0","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101582024-02-01T17:31:53.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201481039Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"1","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101592024-02-01T17:31:53.868Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201504359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"client":"2","":"downstairs","session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101602024-02-01T17:31:53.868Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201528112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101612024-02-01T17:31:53.868Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201548728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101622024-02-01T17:31:53.868Z{"msg":"e36f0790-34a0-4bf2-b12b-77335ac0293c is now active with session: debfdc6c-5160-4f6a-b669-5a166cc2d516","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201570037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101632024-02-01T17:31:53.868Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201607875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505,"session_id":"debfdc6c-5160-4f6a-b669-5a166cc2d516"}
101642024-02-01T17:31:53.868Z{"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:52.201629705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
101652024-02-01T17:31:53.868ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101662024-02-01T17:31:53.868ZLoading write count information from file "/tmp/test_repair_verify.data"
101672024-02-01T17:31:53.868ZRun Repair workload
101682024-02-01T17:31:53.869Z01/30 Write block 163 len 40960 data: 8 8 6 6 6 8 10 11 12 12
101692024-02-01T17:31:54.730Z02/30 Write block 178 len 12288 data: 8 8 7
101702024-02-01T17:31:54.730Z03/30 Flush
101712024-02-01T17:31:54.730Z04/30 Write block 116 len 20480 data: 6 5 4 5 5
101722024-02-01T17:31:54.730Z05/30 Flush
101732024-02-01T17:31:54.730Z06/30 Write block 163 len 12288 data: 9 9 7
101742024-02-01T17:31:54.731Z07/30 Read block 542 len 4096
101752024-02-01T17:31:54.731Z08/30 Write block 18 len 12288 data: 10 8 9
101762024-02-01T17:31:54.731Z09/30 Flush
101772024-02-01T17:31:54.731Z10/30 Write block 206 len 8192 data: 11 13
101782024-02-01T17:31:54.731Z11/30 Write block 432 len 8192 data: 10 12
101792024-02-01T17:31:54.731Z12/30 Write block 537 len 16384 data: 9 7 6 9
101802024-02-01T17:31:54.731Z13/30 Read block 422 len 28672
101812024-02-01T17:31:54.731Z14/30 Write block 348 len 28672 data: 7 5 4 5 8 7 11
101822024-02-01T17:31:54.731Z15/30 Read block 549 len 28672
101832024-02-01T17:31:54.731Z16/30 Read block 337 len 32768
101842024-02-01T17:31:54.731Z17/30 Write block 258 len 28672 data: 9 7 5 4 3 5 5
101852024-02-01T17:31:54.731Z18/30 Read block 309 len 4096
101862024-02-01T17:31:54.731Z19/30 Read block 187 len 24576
101872024-02-01T17:31:54.731Z20/30 Read block 328 len 16384
101882024-02-01T17:31:54.731Z21/30 Write block 456 len 36864 data: 9 11 11 13 11 11 10 8 7
101892024-02-01T17:31:54.732Z22/30 Read block 403 len 8192
101902024-02-01T17:31:54.732Z23/30 Read block 368 len 4096
101912024-02-01T17:31:54.732Z24/30 Write block 102 len 28672 data: 7 6 5 8 9 10 9
101922024-02-01T17:31:54.732Z25/30 Read block 313 len 8192
101932024-02-01T17:31:54.732Z26/30 Write block 66 len 28672 data: 8 8 8 11 9 11 12
101942024-02-01T17:31:54.732Z27/30 Read block 451 len 36864
101952024-02-01T17:31:54.732Z28/30 Write block 68 len 8192 data: 9 12
101962024-02-01T17:31:54.732Z29/30 Write block 53 len 8192 data: 9 11
101972024-02-01T17:31:54.732Z30/30 Read block 70 len 4096
101982024-02-01T17:31:54.732Z----------------------------------------------------------------
101992024-02-01T17:31:54.748Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
102002024-02-01T17:31:54.748ZStates: Active Active Active
102012024-02-01T17:31:54.748ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
102022024-02-01T17:31:54.748Z 1 Acked 1000 Write 10 Done Done Done false
102032024-02-01T17:31:54.748Z 2 Acked 1001 Write 3 Done Done Done false
102042024-02-01T17:31:54.748Z 3 Acked 1002 Flush 0 Done Done Sent false
102052024-02-01T17:31:54.748Z 4 Acked 1003 Write 5 Done Done Sent false
102062024-02-01T17:31:54.789Z 5 Acked 1004 Flush 0 Done Done Sent false
102072024-02-01T17:31:54.789Z 6 Acked 1005 Write 3 Done Done Sent false
102082024-02-01T17:31:54.789Z 7 Acked 1006 Read 1 Done Done Sent false
102092024-02-01T17:31:54.789Z 8 Acked 1007 Write 3 Done Done Sent false
102102024-02-01T17:31:54.789Z 9 Acked 1008 Flush 0 Done Done Sent false
102112024-02-01T17:31:54.789Z 10 Acked 1009 Write 2 Done Done Sent false
102122024-02-01T17:31:54.789Z 11 Acked 1010 Write 2 Done Done Sent false
102132024-02-01T17:31:54.790Z 12 Acked 1011 Write 4 Done Done Sent false
102142024-02-01T17:31:54.790Z 13 Acked 1012 Read 7 Done Done Sent false
102152024-02-01T17:31:54.790Z 14 Acked 1013 Write 7 Done Done Sent false
102162024-02-01T17:31:54.790Z 15 Acked 1014 Read 7 Done Done Sent false
102172024-02-01T17:31:54.790Z 16 Acked 1015 Read 8 Done Done Sent false
102182024-02-01T17:31:54.790Z 17 Acked 1016 Write 7 Done Done Sent false
102192024-02-01T17:31:54.790Z 18 Acked 1017 Read 1 Done Done Sent false
102202024-02-01T17:31:54.790Z 19 Acked 1018 Read 6 Done Done Sent false
102212024-02-01T17:31:54.790Z 20 Acked 1019 Read 4 Done Done Sent false
102222024-02-01T17:31:54.790Z 21 Acked 1020 Write 9 Done Done Sent false
102232024-02-01T17:31:54.790Z 22 Acked 1021 Read 2 Done Done Sent false
102242024-02-01T17:31:54.790Z 23 Acked 1022 Read 1 Done Done Sent false
102252024-02-01T17:31:54.790Z 24 Acked 1023 Write 7 Done Done Sent false
102262024-02-01T17:31:54.790Z 25 Acked 1024 Read 2 Done Done Sent false
102272024-02-01T17:31:54.790Z 26 Acked 1025 Write 7 Done Done Sent false
102282024-02-01T17:31:54.790Z 27 Acked 1026 Read 9 Done Done Sent false
102292024-02-01T17:31:54.791Z 28 Acked 1027 Write 2 Done Done Sent false
102302024-02-01T17:31:54.791Z 29 Acked 1028 Write 2 Done Done Sent false
102312024-02-01T17:31:54.791Z 30 Acked 1029 Read 1 Done Sent Sent false
102322024-02-01T17:31:54.791Z STATES DS:0 DS:1 DS:2 TOTAL
102332024-02-01T17:31:54.791Z New 0 0 0 0
102342024-02-01T17:31:54.791Z Sent 0 1 28 29
102352024-02-01T17:31:54.791Z Done 30 29 2 61
102362024-02-01T17:31:54.791Z Skipped 0 0 0 0
102372024-02-01T17:31:54.791Z Error 0 0 0 0
102382024-02-01T17:31:54.791ZLast Flush: 1008 1008 0
102392024-02-01T17:31:54.791ZDownstairs last five completed:
102402024-02-01T17:31:54.791ZUpstairs last five completed: 30 29 28 27 26
102412024-02-01T17:31:54.792Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:52.247232818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1505}
102422024-02-01T17:31:54.792ZWrote out file "/tmp/test_repair_verify.data"
102432024-02-01T17:31:54.792Z
102442024-02-01T17:31:54.792ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102452024-02-01T17:31:54.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102462024-02-01T17:31:54.792Z 0 000-019 94 94 89 139 139 133 F F F <---
102472024-02-01T17:31:54.792Z 1 020-039 94 94 93 139 139 136 F F F <---
102482024-02-01T17:31:54.792Z 2 040-059 88 88 88 132 132 132 T T F <---
102492024-02-01T17:31:54.792Z 3 060-079 93 93 93 136 136 136 T T F <---
102502024-02-01T17:31:54.792Z 4 080-099 92 92 92 135 135 135 F F F
102512024-02-01T17:31:54.792Z 5 100-119 94 94 92 138 138 135 T T F <---
102522024-02-01T17:31:54.792Z 6 120-139 94 94 88 138 138 130 F F F <---
102532024-02-01T17:31:54.792Z 7 140-159 93 93 93 136 136 136 F F F
102542024-02-01T17:31:54.792Z 8 160-179 94 94 93 139 139 136 F F T <---
102552024-02-01T17:31:54.792Z 9 180-199 94 94 76 137 137 115 F F T <---
102562024-02-01T17:31:54.792Z 10 200-219 90 90 90 134 134 134 T T F <---
102572024-02-01T17:31:54.792Z 11 220-239 88 88 88 130 130 130 F F F
102582024-02-01T17:31:54.792Z 12 240-259 86 86 86 128 128 128 T T F <---
102592024-02-01T17:31:54.793Z 13 260-279 90 90 90 134 134 134 T T F <---
102602024-02-01T17:31:54.793Z 14 280-299 84 84 84 125 125 125 F F F
102612024-02-01T17:31:54.793Z 15 300-319 92 92 92 135 135 135 F F F
102622024-02-01T17:31:54.793Z 16 320-339 93 93 93 136 136 136 F F F
102632024-02-01T17:31:54.793Z 17 340-359 88 88 88 132 132 132 T T F <---
102642024-02-01T17:31:54.793Z 18 360-379 92 92 92 135 135 135 F F F
102652024-02-01T17:31:54.793Z 19 380-399 92 92 92 135 135 135 F F F
102662024-02-01T17:31:54.793Z 20 400-419 88 88 88 130 130 130 F F F
102672024-02-01T17:31:54.793Z 21 420-439 92 92 92 135 135 135 T T F <---
102682024-02-01T17:31:54.793Z 22 440-459 93 93 93 136 136 136 T T F <---
102692024-02-01T17:31:55.428Z 23 460-479 92 92 92 135 135 135 T T F <---
102702024-02-01T17:31:55.428Z 24 480-499 90 90 90 134 134 134 F F F
102712024-02-01T17:31:55.428Z 25 500-519 92 92 92 135 135 135 F F F
102722024-02-01T17:31:55.428Z 26 520-539 86 86 86 128 128 128 T T F <---
102732024-02-01T17:31:55.451Z 27 540-559 90 90 90 134 134 134 T T F <---
102742024-02-01T17:31:55.451Z 28 560-579 90 90 90 134 134 134 F F F
102752024-02-01T17:31:55.451Z 29 580-599 86 86 86 128 128 128 F F F
102762024-02-01T17:31:55.451ZMax gen: 94, Max flush: 139
102772024-02-01T17:31:55.452ZError: Difference in extent metadata found!
102782024-02-01T17:31:55.452ZA Difference in extent metadata is expected here
102792024-02-01T17:31:55.452ZOn loop 47
102802024-02-01T17:31:55.452Z
102812024-02-01T17:31:55.452ZVerifying data now
102822024-02-01T17:31:58.588ZNo change after verify
102832024-02-01T17:31:58.588ZLoop: 47 Downstairs dump after verify (and repair):
102842024-02-01T17:32:20.058ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102852024-02-01T17:32:20.058Z 0 000-019 94 94 94 139 139 139 F F F
102862024-02-01T17:32:20.058Z 1 020-039 94 94 94 139 139 139 F F F
102872024-02-01T17:32:20.059Z 2 040-059 95 95 95 140 140 140 F F F
102882024-02-01T17:32:20.059Z 3 060-079 95 95 95 140 140 140 F F F
102892024-02-01T17:32:20.059Z 4 080-099 92 92 92 135 135 135 F F F
102902024-02-01T17:32:20.059Z 5 100-119 95 95 95 140 140 140 F F F
102912024-02-01T17:32:20.059Z 6 120-139 94 94 94 138 138 138 F F F
102922024-02-01T17:32:20.059Z 7 140-159 93 93 93 136 136 136 F F F
102932024-02-01T17:32:20.059Z 8 160-179 94 94 94 139 139 139 F F F
102942024-02-01T17:32:20.059Z 9 180-199 94 94 94 137 137 137 F F F
102952024-02-01T17:32:20.059Z 10 200-219 95 95 95 140 140 140 F F F
102962024-02-01T17:32:20.059Z 11 220-239 88 88 88 130 130 130 F F F
102972024-02-01T17:32:20.059Z 12 240-259 95 95 95 140 140 140 F F F
102982024-02-01T17:32:20.059Z 13 260-279 95 95 95 140 140 140 F F F
102992024-02-01T17:32:20.060Z 14 280-299 84 84 84 125 125 125 F F F
103002024-02-01T17:32:20.060Z 15 300-319 92 92 92 135 135 135 F F F
103012024-02-01T17:32:20.060Z 16 320-339 93 93 93 136 136 136 F F F
103022024-02-01T17:32:20.060Z 17 340-359 95 95 95 140 140 140 F F F
103032024-02-01T17:32:20.060Z 18 360-379 92 92 92 135 135 135 F F F
103042024-02-01T17:32:20.060Z 19 380-399 92 92 92 135 135 135 F F F
103052024-02-01T17:32:20.060Z 20 400-419 88 88 88 130 130 130 F F F
103062024-02-01T17:32:20.060Z 21 420-439 95 95 95 140 140 140 F F F
103072024-02-01T17:32:20.060Z 22 440-459 95 95 95 140 140 140 F F F
103082024-02-01T17:32:20.060Z 23 460-479 95 95 95 140 140 140 F F F
103092024-02-01T17:32:20.060Z 24 480-499 90 90 90 134 134 134 F F F
103102024-02-01T17:32:20.060Z 25 500-519 92 92 92 135 135 135 F F F
103112024-02-01T17:32:20.060Z 26 520-539 95 95 95 140 140 140 F F F
103122024-02-01T17:32:20.060Z 27 540-559 95 95 95 140 140 140 F F F
103132024-02-01T17:32:20.061Z 28 560-579 90 90 90 134 134 134 F F F
103142024-02-01T17:32:20.061Z 29 580-599 86 86 86 128 128 128 F F F
103152024-02-01T17:32:20.061ZMax gen: 95, Max flush: 140
103162024-02-01T17:32:20.061Z
103172024-02-01T17:32:20.061ZBegin loop 48
103182024-02-01T17:32:20.061ZDownstairs to restart: 2
103192024-02-01T17:32:20.061Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.777228591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
103202024-02-01T17:32:20.061Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.777710529Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
103212024-02-01T17:32:20.061Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.777777842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
103222024-02-01T17:32:20.061Z{"msg":"Crucible db5cd9f7-da17-43dd-9151-9cbaf49ae9b0 has session id: 957d562b-c848-4152-ad22-329e7c10c403","v":0,"name":"crucible","level":30Crucible runtime is spawned
103232024-02-01T17:32:20.061Z,"time":"2024-02-01T17:31:57.777818564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103242024-02-01T17:32:20.062Z{"msg":"Upstairs opts: Upstairs UUID: db5cd9f7-da17-43dd-9151-9cbaf49ae9b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778028226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103252024-02-01T17:32:20.062Z{"msg":"Crucible stats registered with UUID: db5cd9f7-da17-43dd-9151-9cbaf49ae9b0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778075797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103262024-02-01T17:32:20.062Z{"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.77812799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
103272024-02-01T17:32:20.062Z{"msg":"db5cd9f7-da17-43dd-9151-9cbaf49ae9b0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778173538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103282024-02-01T17:32:20.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778210686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103292024-02-01T17:32:20.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778257785Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103302024-02-01T17:32:20.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778299169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103312024-02-01T17:32:20.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778363283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103322024-02-01T17:32:20.063Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.77843155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103332024-02-01T17:32:20.063Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778472479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103342024-02-01T17:32:20.064Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778513508Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103352024-02-01T17:32:20.064Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778651572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103362024-02-01T17:32:20.064Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778743384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103372024-02-01T17:32:20.064Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778889161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103382024-02-01T17:32:20.065Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.778984672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103392024-02-01T17:32:20.065Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.779133715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"io task","client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103402024-02-01T17:32:20.065Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.780045973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103412024-02-01T17:32:20.066Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.780108734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103422024-02-01T17:32:20.066Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.780135759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103432024-02-01T17:32:20.066Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.780158786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103442024-02-01T17:32:20.066Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.78027647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103452024-02-01T17:32:20.066Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.780368533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103462024-02-01T17:32:20.066Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.781122744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103472024-02-01T17:32:20.066Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.78121443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103482024-02-01T17:32:20.066Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.781267383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103492024-02-01T17:32:20.066Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.781943281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103502024-02-01T17:32:20.066Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782001113Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103512024-02-01T17:32:20.066Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782034606Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103522024-02-01T17:32:20.066Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782264614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103532024-02-01T17:32:20.066Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782333079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103542024-02-01T17:32:20.066Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782440147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103552024-02-01T17:32:20.066Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782502071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103562024-02-01T17:32:20.066Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782530349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103572024-02-01T17:32:20.066Z{"msg":"[0]R flush_numbers[0..12]: [139, 139, 140, 140, 135, 140, 138, 136, 139, 137, 140, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782576492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103582024-02-01T17:32:20.067Z{"msg":"[0]R generation[0..12]: [94, 94, 95, 95, 92, 95, 94, 93, 94, 94, 95, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782602365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO
103592024-02-01T17:32:20.067Zsession_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103602024-02-01T17:32:20.067Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782804217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103612024-02-01T17:32:20.067Z{"msg":"[1]R flush_numbers[0..12]: [139, 139, 140, 140, 135, 140, 138, 136, 139, 137, 140, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782835141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103622024-02-01T17:32:20.067Z{"msg":"[1]R generation[0..12]: [94, 94, 95, 95, 92, 95, 94, 93, 94, 94, 95, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782858905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103632024-02-01T17:32:20.067Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782884182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103642024-02-01T17:32:20.067Z{"msg":"[2]R flush_numbers[0..12]: [139, 139, 140, 140, 135, 140, 138, 136, 139, 137, 140, 130]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782915231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103652024-02-01T17:32:20.067Z{"msg":"[2]R generation[0..12]: [94, 94, 95, 95, 92, 95, 94, 93, 94, 94, 95, 88]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782939096Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103662024-02-01T17:32:20.067Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782962575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103672024-02-01T17:32:20.067Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.782984234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103682024-02-01T17:32:20.067Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783026758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103692024-02-01T17:32:20.067Z{"msg":"Next flush: 141","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783050591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103702024-02-01T17:32:20.067Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783072221Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"":"downstairs","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103712024-02-01T17:32:20.067Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783116393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103722024-02-01T17:32:20.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783138807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"0","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103732024-02-01T17:32:20.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783162732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"1","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103742024-02-01T17:32:20.067Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783184483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"client":"2","":"downstairs","session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103752024-02-01T17:32:20.067Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783206287Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103762024-02-01T17:32:20.067Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783253096Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103772024-02-01T17:32:20.068Z{"msg":"db5cd9f7-da17-43dd-9151-9cbaf49ae9b0 is now active with session: 957d562b-c848-4152-ad22-329e7c10c403","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783276827Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103782024-02-01T17:32:20.068Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.78329857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513,"session_id":"957d562b-c848-4152-ad22-329e7c10c403"}
103792024-02-01T17:32:20.068Z{"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:31:57.783340785Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
103802024-02-01T17:32:20.068ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103812024-02-01T17:32:20.068ZLoading write count information from file "/tmp/test_repair_verify.data"
103822024-02-01T17:32:20.068ZRun Repair workload
103832024-02-01T17:32:20.068Z01/30 Write block 525 len 8192 data: 6 6
103842024-02-01T17:32:20.477Z02/30 Read block 260 len 24576
103852024-02-01T17:32:20.477Z03/30 Flush
103862024-02-01T17:32:20.477Z04/30 Write block 411 len 40960 data: 4 6 6 8 9 11 6 7 8 6
103872024-02-01T17:32:20.478Z05/30 Write block 2 len 20480 data: 5 6 7 7 8
103882024-02-01T17:32:20.478Z06/30 Read block 526 len 24576
103892024-02-01T17:32:20.478Z07/30 Write block 36 len 24576 data: 9 10 10 9 5 8
103902024-02-01T17:32:20.478Z08/30 Write block 346 len 40960 data: 6 7 8 6 5 6 9 8 12 10
103912024-02-01T17:32:20.479Z09/30 Read block 200 len 36864
103922024-02-01T17:32:20.479Z10/30 Flush
103932024-02-01T17:32:20.479Z11/30 Write block 518 len 8192 data: 6 6
103942024-02-01T17:32:20.479Z12/30 Write block 3 len 28672 data: 7 8 8 9 8 8 5
103952024-02-01T17:32:20.479Z13/30 Flush
103962024-02-01T17:32:20.479Z14/30 Write block 219 len 28672 data: 11 11 9 8 7 6 5
103972024-02-01T17:32:20.479Z15/30 Write block 280 len 24576 data: 10 8 6 7 7 8
103982024-02-01T17:32:20.480Z16/30 Flush
103992024-02-01T17:32:20.480Z17/30 Write block 113 len 36864 data: 7 7 7 7 6 5 6 6 4
104002024-02-01T17:32:20.480Z18/30 Read block 147 len 28672
104012024-02-01T17:32:20.480Z19/30 Read block 127 len 28672
104022024-02-01T17:32:20.480Z20/30 Write block 129 len 32768 data: 6 6 4 2 5 7 8 10
104032024-02-01T17:32:20.480Z21/30 Write block 551 len 36864 data: 10 10 8 6 9 8 9 10 10
104042024-02-01T17:32:20.480Z22/30 Write block 485 len 12288 data: 8 8 9
104052024-02-01T17:32:20.480Z23/30 Write block 296 len 24576 data: 10 9 12 11 11 8
104062024-02-01T17:32:20.480Z24/30 Read block 145 len 40960
104072024-02-01T17:32:20.482Z25/30 Write block 514 len 40960 data: 5 5 4 5 7 7 7 7 7 8
104082024-02-01T17:32:20.482Z26/30 Flush
104092024-02-01T17:32:20.483Z27/30 Write block 291 len 4096 data: 8
104102024-02-01T17:32:20.483Z28/30 Read block 427 len 16384
104112024-02-01T17:32:20.483Z29/30 Read block 145 len 40960
104122024-02-01T17:32:20.483Z30/30 Read block 418 len 8192
104132024-02-01T17:32:20.483Z----------------------------------------------------------------
104142024-02-01T17:32:20.483Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:27
104152024-02-01T17:32:20.483ZStates: Active Active Active
104162024-02-01T17:32:20.483ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104172024-02-01T17:32:20.483Z 4 Acked 1003 Write 10 Done Done Done false
104182024-02-01T17:32:20.483Z 5 Acked 1004 Write 5 Done Done Done false
104192024-02-01T17:32:20.483Z 6 Acked 1005 Read 6 Done Done Done false
104202024-02-01T17:32:20.483Z 7 Acked 1006 Write 6 Done Done Sent false
104212024-02-01T17:32:20.483Z 8 Acked 1007 Write 10 Done Done Sent false
104222024-02-01T17:32:20.484Z 9 Acked 1008 Read 9 Done Done Sent false
104232024-02-01T17:32:20.484Z 10 Acked 1009 Flush 0 Done Done Sent false
104242024-02-01T17:32:20.484Z 11 Acked 1010 Write 2 Done Done Sent false
104252024-02-01T17:32:20.484Z 12 Acked 1011 Write 7 Done Done Sent false
104262024-02-01T17:32:20.484Z 13 Acked 1012 Flush 0 Done Done Sent false
104272024-02-01T17:32:20.484Z 14 Acked 1013 Write 7 Done Done Sent false
104282024-02-01T17:32:20.484Z 15 Acked 1014 Write 6 Done Done Sent false
104292024-02-01T17:32:20.485Z 16 Acked 1015 Flush 0 Done Done Sent false
104302024-02-01T17:32:20.485Z 17 Acked 1016 Write 9 Done Done Sent false
104312024-02-01T17:32:20.486Z 18 Acked 1017 Read 7 Done Done Sent false
104322024-02-01T17:32:20.486Z 19 Acked 1018 Read 7 Done Done Sent false
104332024-02-01T17:32:20.486Z 20 Acked 1019 Write 8 Done Done Sent false
104342024-02-01T17:32:20.486Z 21 Acked 1020 Write 9 Done Done Sent false
104352024-02-01T17:32:20.486Z 22 Acked 1021 Write 3 Done Done Sent false
104362024-02-01T17:32:20.486Z 23 Acked 1022 Write 6 Done Done Sent false
104372024-02-01T17:32:20.486Z 24 Acked 1023 Read 10 Done Done Sent false
104382024-02-01T17:32:20.486Z 25 Acked 1024 Write 10 Done Done Sent false
104392024-02-01T17:32:20.486Z 26 Acked 1025 Flush 0 Done Done Sent false
104402024-02-01T17:32:20.486Z 27 Acked 1026 Write 1 Done Done Sent false
104412024-02-01T17:32:20.486Z 28 Acked 1027 Read 4 Done Done Sent false
104422024-02-01T17:32:20.486Z 29 Acked 1028 Read 10 Done Done Sent false
104432024-02-01T17:32:20.486Z 30 Acked 1029 Read 2 Done Done Sent false
104442024-02-01T17:32:20.486Z STATES DS:0 DS:1 DS:2 TOTAL
104452024-02-01T17:32:20.486Z New 0 0 0 0
104462024-02-01T17:32:20.486Z Sent 0 0 24 24
104472024-02-01T17:32:20.486Z Done 27 27 3 57
104482024-02-01T17:32:20.486Z Skipped 0 0 0 0
104492024-02-01T17:32:20.486Z Error 0 0 0 0
104502024-02-01T17:32:20.487ZLast Flush: 1025 1025 1002
104512024-02-01T17:32:20.487ZDownstairs last five completed: 1002 1001 1000
104522024-02-01T17:32:20.487ZUpstairs last five completed: 30 29 28 27 26
104532024-02-01T17:32:20.487Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:31:57.836027226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1513}
104542024-02-01T17:32:20.487ZWrote out file "/tmp/test_repair_verify.data"
104552024-02-01T17:32:20.487Z
104562024-02-01T17:32:20.487ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104572024-02-01T17:32:20.487ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104582024-02-01T17:32:20.487Z 0 000-019 96 96 94 143 143 139 F F T <---
104592024-02-01T17:32:20.487Z 1 020-039 96 96 94 142 142 139 F F F <---
104602024-02-01T17:32:20.487Z 2 040-059 96 96 95 142 142 140 F F F <---
104612024-02-01T17:32:20.487Z 3 060-079 95 95 95 140 140 140 F F F
104622024-02-01T17:32:20.487Z 4 080-099 92 92 92 135 135 135 F F F
104632024-02-01T17:32:20.487Z 5 100-119 96 96 95 145 145 140 F F F <---
104642024-02-01T17:32:20.487Z 6 120-139 96 96 94 145 145 138 F F F <---
104652024-02-01T17:32:20.487Z 7 140-159 93 93 93 136 136 136 F F F
104662024-02-01T17:32:20.487Z 8 160-179 94 94 94 139 139 139 F F F
104672024-02-01T17:32:20.487Z 9 180-199 94 94 94 137 137 137 F F F
104682024-02-01T17:32:20.487Z 10 200-219 96 96 95 144 144 140 F F F <---
104692024-02-01T17:32:20.487Z 11 220-239 96 96 88 144 144 130 F F F <---
104702024-02-01T17:32:20.488Z 12 240-259 95 95 95 140 140 140 F F F
104712024-02-01T17:32:20.488Z 13 260-279 95 95 95 140 140 140 F F F
104722024-02-01T17:32:20.488Z 14 280-299 96 96 84 145 145 125 T T F <---
104732024-02-01T17:32:20.488Z 15 300-319 96 96 92 145 145 135 F F F <---
104742024-02-01T17:32:20.488Z 16 320-339 93 93 93 136 136 136 F F F
104752024-02-01T17:32:20.488Z 17 340-359 96 96 95 142 142 140 F F F <---
104762024-02-01T17:32:20.488Z 18 360-379 92 92 92 135 135 135 F F F
104772024-02-01T17:32:20.488Z 19 380-399 92 92 92 135 135 135 F F F
104782024-02-01T17:32:20.488Z 20 400-419 96 96 88 142 142 130 F F T <---
104792024-02-01T17:32:20.488Z 21 420-439 96 96 95 142 142 140 F F T <---
104802024-02-01T17:32:20.488Z 22 440-459 95 95 95 140 140 140 F F F
104812024-02-01T17:32:20.488Z 23 460-479 95 95 95 140 140 140 F F F
104822024-02-01T17:32:20.488Z 24 480-499 96 96 90 145 145 134 F F F <---
104832024-02-01T17:32:20.488Z 25 500-519 96 96 92 145 145 135 F F F <---
104842024-02-01T17:32:20.659Z 26 520-539 96 96 96 145 145 141 F F F <---
104852024-02-01T17:32:20.660Z 27 540-559 96 96 95 145 145 140 F F F <---
104862024-02-01T17:32:20.660Z 28 560-579 90 90 90 134 134 134 F F F
104872024-02-01T17:32:20.660Z 29 580-599 86 86 86 128 128 128 F F F
104882024-02-01T17:32:20.660ZMax gen: 96, Max flush: 145
104892024-02-01T17:32:20.660ZError: Difference in extent metadata found!
104902024-02-01T17:32:20.660ZA Difference in extent metadata is expected here
104912024-02-01T17:32:20.660ZOn loop 48
104922024-02-01T17:32:20.660Z
104932024-02-01T17:32:20.660ZVerifying data now
104942024-02-01T17:32:20.660ZNo change after verify
104952024-02-01T17:32:20.661ZLoop: 48 Downstairs dump after verify (and repair):
104962024-02-01T17:32:20.661ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104972024-02-01T17:32:20.661Z 0 000-019 96 96 96 143 143 143 F F F
104982024-02-01T17:32:20.661Z 1 020-039 96 96 96 142 142 142 F F F
104992024-02-01T17:32:20.661Z 2 040-059 96 96 96 142 142 142 F F F
105002024-02-01T17:32:20.661Z 3 060-079 95 95 95 140 140 140 F F F
105012024-02-01T17:32:20.661Z 4 080-099 92 92 92 135 135 135 F F F
105022024-02-01T17:32:20.661Z 5 100-119 96 96 96 145 145 145 F F F
105032024-02-01T17:32:20.662Z 6 120-139 96 96 96 145 145 145 F F F
105042024-02-01T17:32:20.662Z 7 140-159 93 93 93 136 136 136 F F F
105052024-02-01T17:32:20.662Z 8 160-179 94 94 94 139 139 139 F F F
105062024-02-01T17:32:20.662Z 9 180-199 94 94 94 137 137 137 F F F
105072024-02-01T17:32:20.662Z 10 200-219 96 96 96 144 144 144 F F F
105082024-02-01T17:32:20.662Z 11 220-239 96 96 96 144 144 144 F F F
105092024-02-01T17:32:20.662Z 12 240-259 95 95 95 140 140 140 F F F
105102024-02-01T17:32:20.662Z 13 260-279 95 95 95 140 140 140 F F F
105112024-02-01T17:32:20.662Z 14 280-299 97 97 97 146 146 146 F F F
105122024-02-01T17:32:20.662Z 15 300-319 96 96 96 145 145 145 F F F
105132024-02-01T17:32:20.662Z 16 320-339 93 93 93 136 136 136 F F F
105142024-02-01T17:32:20.662Z 17 340-359 96 96 96 142 142 142 F F F
105152024-02-01T17:32:20.663Z 18 360-379 92 92 92 135 135 135 F F F
105162024-02-01T17:32:20.663Z 19 380-399 92 92 92 135 135 135 F F F
105172024-02-01T17:32:20.663Z 20 400-419 96 96 96 142 142 142 F F F
105182024-02-01T17:32:20.663Z 21 420-439 96 96 96 142 142 142 F F F
105192024-02-01T17:32:20.663Z 22 440-459 95 95 95 140 140 140 F F F
105202024-02-01T17:32:20.663Z 23 460-479 95 95 95 140 140 140 F F F
105212024-02-01T17:32:20.663Z 24 480-499 96 96 96 145 145 145 F F F
105222024-02-01T17:32:20.663Z 25 500-519 96 96 96 145 145 145 F F F
105232024-02-01T17:32:20.663Z 26 520-539 96 96 96 145 145 145 F F F
105242024-02-01T17:32:20.663Z 27 540-559 96 96 96 145 145 145 F F F
105252024-02-01T17:32:20.663Z 28 560-579 90 90 90 134 134 134 F F F
105262024-02-01T17:32:20.663Z 29 580-599 86 86 86 128 128 128 F F F
105272024-02-01T17:32:20.663ZMax gen: 97, Max flush: 146
105282024-02-01T17:32:20.663Z
105292024-02-01T17:32:20.663ZBegin loop 49
105302024-02-01T17:32:20.663ZDownstairs to restart: 0
105312024-02-01T17:32:20.663Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.246769112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
105322024-02-01T17:32:20.663Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247318166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
105332024-02-01T17:32:20.663Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247378771Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
105342024-02-01T17:32:20.663Z{"msg":"Crucible runtime is spawned
105352024-02-01T17:32:20.664ZCrucible 9154783e-5a96-4dff-8ed0-de99b1352c98 has session id: 6cd1113c-ce91-4e88-867d-7ce4b22a255e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247561671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105362024-02-01T17:32:20.664Z{"msg":"Upstairs opts: Upstairs UUID: 9154783e-5a96-4dff-8ed0-de99b1352c98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247628019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105372024-02-01T17:32:20.664Z{"msg":"Crucible stats registered with UUID: 9154783e-5a96-4dff-8ed0-de99b1352c98","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247674054Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105382024-02-01T17:32:20.664Z{"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.24771561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
105392024-02-01T17:32:20.664Z{"msg":"9154783e-5a96-4dff-8ed0-de99b1352c98 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247836571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105402024-02-01T17:32:20.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247895677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105412024-02-01T17:32:20.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247951517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105422024-02-01T17:32:20.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.247977932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105432024-02-01T17:32:20.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248042174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105442024-02-01T17:32:20.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248089985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105452024-02-01T17:32:20.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248191191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105462024-02-01T17:32:20.664Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248345744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105472024-02-01T17:32:20.664Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.24850643Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105482024-02-01T17:32:20.664Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248592987Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105492024-02-01T17:32:20.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248785527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105502024-02-01T17:32:20.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248864688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105512024-02-01T17:32:20.665Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.248938547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"io task","client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105522024-02-01T17:32:20.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.249763813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105532024-02-01T17:32:20.665Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.249891382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105542024-02-01T17:32:20.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.249980486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105552024-02-01T17:32:20.665Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.250028618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105562024-02-01T17:32:20.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.250595079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105572024-02-01T17:32:20.665Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.250688678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105582024-02-01T17:32:20.665Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.250881449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105592024-02-01T17:32:20.665Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.251123339Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105602024-02-01T17:32:20.666Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.251763034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105612024-02-01T17:32:20.666Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252021845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105622024-02-01T17:32:20.666Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252088536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105632024-02-01T17:32:20.666Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252150947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105642024-02-01T17:32:20.666Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252207812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105652024-02-01T17:32:20.666Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.25226535Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105662024-02-01T17:32:20.666Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252322671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105672024-02-01T17:32:20.666Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252586359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105682024-02-01T17:32:20.666Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252652531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105692024-02-01T17:32:20.666Z{"msg":"[0]R flush_numbers[0..12]: [143, 142, 142, 140, 135, 145, 145, 136, 139, 137, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252699854Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105702024-02-01T17:32:20.667Z{"msg":"[0]R generation[0..12]: [96, 96, 96, 95, 92, 96, 96, 93, 94, 94, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.25274435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105712024-02-01T17:32:20.667Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252788908Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105722024-02-01T17:32:20.667Z{"msg":"[1]R flush_numbers[0..12]: [143, 142, 142, 140, 135, 145, 145, 136, 139, 137, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252832827Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105732024-02-01T17:32:20.667Z{"msg":"[1]R generation[0..12]: [96, 96, 96, 95, 92, 96, 96, 93, 94, 94, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252876037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
105742024-02-01T17:32:20.667Z,"pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105752024-02-01T17:32:20.667Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.252988029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105762024-02-01T17:32:20.667Z{"msg":"[2]R flush_numbers[0..12]: [143, 142, 142, 140, 135, 145, 145, 136, 139, 137, 144, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253020163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105772024-02-01T17:32:20.667Z{"msg":"[2]R generation[0..12]: [96, 96, 96, 95, 92, 96, 96, 93, 94, 94, 96, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253073789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105782024-02-01T17:32:20.667Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253121332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105792024-02-01T17:32:20.668Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.2531638Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105802024-02-01T17:32:20.668Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253207478Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105812024-02-01T17:32:20.668Z{"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253249417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105822024-02-01T17:32:20.668Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253291573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"":"downstairs","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105832024-02-01T17:32:20.669Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253343912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105842024-02-01T17:32:20.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.2533973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"0","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105852024-02-01T17:32:20.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253441675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"1","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105862024-02-01T17:32:20.764Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105872024-02-01T17:32:20.764Z30,"time":"2024-02-01T17:32:03.253492092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"client":"2","":"downstairs","session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105882024-02-01T17:32:20.765Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253535282Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105892024-02-01T17:32:20.765Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253575686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105902024-02-01T17:32:20.766Z{"msg":"9154783e-5a96-4dff-8ed0-de99b1352c98 is now active with session: 6cd1113c-ce91-4e88-867d-7ce4b22a255e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253617068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105912024-02-01T17:32:20.766Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253656652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521,"session_id":"6cd1113c-ce91-4e88-867d-7ce4b22a255e"}
105922024-02-01T17:32:20.766Z{"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:03.253696966Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
105932024-02-01T17:32:20.766ZLoading write count information from file "/tmp/test_repair_verify.data"
105942024-02-01T17:32:20.766ZRun Repair workload
105952024-02-01T17:32:20.766ZError: Difference in extent metadata found!
105962024-02-01T17:32:20.766Z01/30 Write block 498 len 20480 data: 9 8 6 6 6
105972024-02-01T17:32:20.766Z02/30 Read block 548 len 36864
105982024-02-01T17:32:20.766Z03/30 Read block 418 len 16384
105992024-02-01T17:32:20.766Z04/30 Read block 544 len 20480
106002024-02-01T17:32:20.766Z05/30 Write block 212 len 8192 data: 6 7
106012024-02-01T17:32:20.766Z06/30 Write block 168 len 16384 data: 9 11 12 13
106022024-02-01T17:32:20.766Z07/30 Write block 138 len 24576 data: 11 8 9 9 8 7
106032024-02-01T17:32:20.766Z08/30 Write block 508 len 24576 data: 6 7 6 7 7 7
106042024-02-01T17:32:20.766Z09/30 Read block 42 len 32768
106052024-02-01T17:32:20.766Z10/30 Read block 516 len 20480
106062024-02-01T17:32:20.766Z11/30 Read block 180 len 24576
106072024-02-01T17:32:20.766Z12/30 Read block 314 len 40960
106082024-02-01T17:32:20.766Z13/30 Read block 309 len 24576
106092024-02-01T17:32:20.766Z14/30 Write block 433 len 8192 data: 13 11
106102024-02-01T17:32:20.766Z15/30 Read block 520 len 20480
106112024-02-01T17:32:20.766Z16/30 Read block 363 len 28672
106122024-02-01T17:32:20.766Z17/30 Write block 193 len 16384 data: 5 6 6 8
106132024-02-01T17:32:20.767Z18/30 Read block 558 len 24576
106142024-02-01T17:32:20.767Z19/30 Write block 396 len 36864 data: 5 5 4 2 3 5 5 7 6
106152024-02-01T17:32:20.767Z20/30 Write block 148 len 8192 data: 9 9
106162024-02-01T17:32:20.767Z21/30 Read block 313 len 32768
106172024-02-01T17:32:20.767Z22/30 Read block 242 len 36864
106182024-02-01T17:32:20.767Z23/30 Read block 188 len 24576
106192024-02-01T17:32:20.767Z24/30 Write block 44 len 20480 data: 9 9 10 8 11
106202024-02-01T17:32:20.767Z25/30 Read block 360 len 20480
106212024-02-01T17:32:20.767Z26/30 Write block 533 len 20480 data: 9 12 11 8 10
106222024-02-01T17:32:20.767Z27/30 Read block 382 len 4096
106232024-02-01T17:32:20.767Z28/30 Read block 195 len 40960
106242024-02-01T17:32:20.767Z29/30 Read block 423 len 28672
106252024-02-01T17:32:20.767Z30/30 Write block 277 len 20480 data: 11 12 9 11 9
106262024-02-01T17:32:20.767Z----------------------------------------------------------------
106272024-02-01T17:32:20.767Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
106282024-02-01T17:32:20.767ZStates: Active Active Active
106292024-02-01T17:32:20.767ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106302024-02-01T17:32:20.767Z 1 Acked 1000 Write 5 Sent Done Done false
106312024-02-01T17:32:20.767Z 2 Acked 1001 Read 9 Sent Done Done false
106322024-02-01T17:32:20.768Z 3 Acked 1002 Read 4 Sent Done Done false
106332024-02-01T17:32:20.768Z 4 Acked 1003 Read 5 Sent Done Done false
106342024-02-01T17:32:20.768Z 5 Acked 1004 Write 2 Sent Done Done false
106352024-02-01T17:32:20.768Z 6 Acked 1005 Write 4 Sent Done Done false
106362024-02-01T17:32:20.768Z 7 Acked 1006 Write 6 Sent Done Done false
106372024-02-01T17:32:20.768Z 8 Acked 1007 Write 6 Sent Done Done false
106382024-02-01T17:32:20.768Z 9 Acked 1008 Read 8 Sent Done Done false
106392024-02-01T17:32:20.768Z 10 Acked 1009 Read 5 Sent Done Done false
106402024-02-01T17:32:20.768Z 11 Acked 1010 Read 6 Sent Done Done false
106412024-02-01T17:32:20.768Z 12 Acked 1011 Read 10 Sent Done Done false
106422024-02-01T17:32:20.768Z 13 Acked 1012 Read 6 Sent Done Done false
106432024-02-01T17:32:20.768Z 14 Acked 1013 Write 2 Sent Done Done false
106442024-02-01T17:32:20.768Z 15 Acked 1014 Read 5 Sent Done Done false
106452024-02-01T17:32:20.768Z 16 Acked 1015 Read 7 Sent Done Done false
106462024-02-01T17:32:20.768Z 17 Acked 1016 Write 4 Sent Done Done false
106472024-02-01T17:32:20.769Z 18 Acked 1017 Read 6 Sent Done Done false
106482024-02-01T17:32:20.769Z 19 Acked 1018 Write 9 Sent Done Done false
106492024-02-01T17:32:20.769Z 20 Acked 1019 Write 2 Sent Done Done false
106502024-02-01T17:32:20.769Z 21 Acked 1020 Read 8 Sent Done Done false
106512024-02-01T17:32:20.769Z 22 Acked 1021 Read 9 Sent Done Done false
106522024-02-01T17:32:20.769Z 23 Acked 1022 Read 6 Sent Done Done false
106532024-02-01T17:32:20.769Z 24 Acked 1023 Write 5 Sent Done Done false
106542024-02-01T17:32:20.769Z 25 Acked 1024 Read 5 Sent Done Done false
106552024-02-01T17:32:20.770Z 26 Acked 1025 Write 5 Sent Done Done false
106562024-02-01T17:32:20.770Z 27 Acked 1026 Read 1 Sent Done Done false
106572024-02-01T17:32:20.770Z 28 Acked 1027 Read 10 Sent Done Done false
106582024-02-01T17:32:20.770Z 29 Acked 1028 Read 7 Sent Done Done false
106592024-02-01T17:32:20.770Z 30 Acked 1029 Write 5 Sent Sent Sent false
106602024-02-01T17:32:20.770Z STATES DS:0 DS:1 DS:2 TOTAL
106612024-02-01T17:32:20.770Z New 0 0 0 0
106622024-02-01T17:32:20.770Z Sent 30 1 1 32
106632024-02-01T17:32:20.770Z Done 0 29 29 58
106642024-02-01T17:32:20.770Z Skipped 0 0 0 0
106652024-02-01T17:32:20.770Z Error 0 0 0 0
106662024-02-01T17:32:20.770ZLast Flush: 0 0 0
106672024-02-01T17:32:20.770ZDownstairs last five completed:
106682024-02-01T17:32:20.770ZUpstairs last five completed: 30 29 28 27 26
106692024-02-01T17:32:20.770Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:03.296389849Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1521}
106702024-02-01T17:32:20.770ZWrote out file "/tmp/test_repair_verify.data"
106712024-02-01T17:32:20.770Z
106722024-02-01T17:32:20.770ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106732024-02-01T17:32:20.770ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106742024-02-01T17:32:20.771Z 0 000-019 96 96 96 143 143 143 F F F
106752024-02-01T17:32:20.771Z 1 020-039 96 96 96 142 142 142 F F F
106762024-02-01T17:32:20.771Z 2 040-059 96 96 96 142 142 142 F T T <---
106772024-02-01T17:32:20.771Z 3 060-079 95 95 95 140 140 140 F F F
106782024-02-01T17:32:20.771Z 4 080-099 92 92 92 135 135 135 F F F
106792024-02-01T17:32:20.771Z 5 100-119 96 96 96 145 145 145 F F F
106802024-02-01T17:32:20.771Z 6 120-139 96 96 96 145 145 145 F T T <---
106812024-02-01T17:32:20.771Z 7 140-159 93 93 93 136 136 136 F T T <---
106822024-02-01T17:32:20.771Z 8 160-179 94 94 94 139 139 139 F T T <---
106832024-02-01T17:32:20.771Z 9 180-199 94 94 94 137 137 137 F T T <---
106842024-02-01T17:32:20.827Z 10 200-219 96 96 96 144 144 144 F T T <---
106852024-02-01T17:32:20.827Z 11 220-239 96 96 96 144 144 144 F F F
106862024-02-01T17:32:20.828Z 12 240-259 95 95 95 140 140 140 F F F
106872024-02-01T17:32:20.828Z 13 260-279 95 95 95 140 140 140 F T T <---
106882024-02-01T17:32:20.828Z 14 280-299 97 97 97 146 146 146 F T T <---
106892024-02-01T17:32:20.828Z 15 300-319 96 96 96 145 145 145 F F F
106902024-02-01T17:32:20.828Z 16 320-339 93 93 93 136 136 136 F F F
106912024-02-01T17:32:20.828Z 17 340-359 96 96 96 142 142 142 F F F
106922024-02-01T17:32:20.828Z 18 360-379 92 92 92 135 135 135 F F F
106932024-02-01T17:32:20.828Z 19 380-399 92 92 92 135 135 135 F T T <---
106942024-02-01T17:32:20.828Z 20 400-419 96 96 96 142 142 142 F T T <---
106952024-02-01T17:32:20.828Z 21 420-439 96 96 96 142 142 142 F T T <---
106962024-02-01T17:32:20.828Z 22 440-459 95 95 95 140 140 140 F F F
106972024-02-01T17:32:20.829Z 23 460-479 95 95 95 140 140 140 F F F
106982024-02-01T17:32:20.829Z 24 480-499 96 96 96 145 145 145 F T T <---
106992024-02-01T17:32:20.829Z 25 500-519 96 96 96 145 145 145 F T T <---
107002024-02-01T17:32:20.829Z 26 520-539 96 96 96 145 145 145 F T T <---
107012024-02-01T17:32:20.829Z 27 540-559 96 96 96 145 145 145 F F F
107022024-02-01T17:32:20.829Z 28 560-579 90 90 90 134 134 134 F F F
107032024-02-01T17:32:20.829Z 29 580-599 86 86 86 128 128 128 F F F
107042024-02-01T17:32:20.829ZMax gen: 97, Max flush: 146
107052024-02-01T17:32:20.829ZA Difference in extent metadata is expected here
107062024-02-01T17:32:20.829ZOn loop 49
107072024-02-01T17:32:20.829Z
107082024-02-01T17:32:20.829ZVerifying data now
107092024-02-01T17:32:20.829ZNo change after verify
107102024-02-01T17:32:20.831ZLoop: 49 Downstairs dump after verify (and repair):
107112024-02-01T17:32:20.831ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107122024-02-01T17:32:20.831Z 0 000-019 96 96 96 143 143 143 F F F
107132024-02-01T17:32:20.831Z 1 020-039 96 96 96 142 142 142 F F F
107142024-02-01T17:32:20.831Z 2 040-059 98 98 98 147 147 147 F F F
107152024-02-01T17:32:20.831Z 3 060-079 95 95 95 140 140 140 F F F
107162024-02-01T17:32:20.831Z 4 080-099 92 92 92 135 135 135 F F F
107172024-02-01T17:32:20.831Z 5 100-119 96 96 96 145 145 145 F F F
107182024-02-01T17:32:20.831Z 6 120-139 98 98 98 147 147 147 F F F
107192024-02-01T17:32:20.831Z 7 140-159 98 98 98 147 147 147 F F F
107202024-02-01T17:32:20.831Z 8 160-179 98 98 98 147 147 147 F F F
107212024-02-01T17:32:20.831Z 9 180-199 98 98 98 147 147 147 F F F
107222024-02-01T17:32:20.831Z 10 200-219 98 98 98 147 147 147 F F F
107232024-02-01T17:32:20.832Z 11 220-239 96 96 96 144 144 144 F F F
107242024-02-01T17:32:20.832Z 12 240-259 95 95 95 140 140 140 F F F
107252024-02-01T17:32:20.832Z 13 260-279 98 98 98 147 147 147 F F F
107262024-02-01T17:32:20.832Z 14 280-299 98 98 98 147 147 147 F F F
107272024-02-01T17:32:20.832Z 15 300-319 96 96 96 145 145 145 F F F
107282024-02-01T17:32:20.832Z 16 320-339 93 93 93 136 136 136 F F F
107292024-02-01T17:32:20.832Z 17 340-359 96 96 96 142 142 142 F F F
107302024-02-01T17:32:20.832Z 18 360-379 92 92 92 135 135 135 F F F
107312024-02-01T17:32:20.832Z 19 380-399 98 98 98 147 147 147 F F F
107322024-02-01T17:32:20.832Z 20 400-419 98 98 98 147 147 147 F F F
107332024-02-01T17:32:20.832Z 21 420-439 98 98 98 147 147 147 F F F
107342024-02-01T17:32:20.832Z 22 440-459 95 95 95 140 140 140 F F F
107352024-02-01T17:32:20.833Z 23 460-479 95 95 95 140 140 140 F F F
107362024-02-01T17:32:20.833Z 24 480-499 98 98 98 147 147 147 F F F
107372024-02-01T17:32:20.833Z 25 500-519 98 98 98 147 147 147 F F F
107382024-02-01T17:32:20.833Z 26 520-539 98 98 98 147 147 147 F F F
107392024-02-01T17:32:20.833Z 27 540-559 96 96 96 145 145 145 F F F
107402024-02-01T17:32:20.833Z 28 560-579 90 90 90 134 134 134 F F F
107412024-02-01T17:32:20.833Z 29 580-599 86 86 86 128 128 128 F F F
107422024-02-01T17:32:20.833ZMax gen: 98, Max flush: 147
107432024-02-01T17:32:20.833Z
107442024-02-01T17:32:20.833ZBegin loop 50
107452024-02-01T17:32:20.834ZDownstairs to restart: 0
107462024-02-01T17:32:20.834Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614203749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
107472024-02-01T17:32:20.834Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614698041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
107482024-02-01T17:32:20.834Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614726188Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
107492024-02-01T17:32:20.834Z{"msg":"Crucible cd181edd-36e7-46c0-b041-95a7a568b6c6 has session id: b8eb8199-026f-4182-b287-5d5febde014a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614746999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107502024-02-01T17:32:20.834Z{"msg":"Upstairs opts: Upstairs UUID: cd181edd-36e7-46c0-b041-95a7a568b6c6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614784115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107512024-02-01T17:32:20.835Z{"msg":"Crucible stats registered with UUID: cd181edd-36e7-46c0-b041-95a7a568b6c6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.614809569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107522024-02-01T17:32:20.835ZCrucible runtime is spawned
107532024-02-01T17:32:20.835Z{"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615056505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
107542024-02-01T17:32:20.835Z{"msg":"cd181edd-36e7-46c0-b041-95a7a568b6c6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61521094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107552024-02-01T17:32:20.835Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615257818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107562024-02-01T17:32:20.835Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615289572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107572024-02-01T17:32:20.835Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615313215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107582024-02-01T17:32:20.835Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615336264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107592024-02-01T17:32:20.836Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615358383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107602024-02-01T17:32:20.836Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615381661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107612024-02-01T17:32:20.836Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615405329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107622024-02-01T17:32:20.837Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615429214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107632024-02-01T17:32:20.837Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61545229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107642024-02-01T17:32:20.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.615974063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107652024-02-01T17:32:20.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.616033831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107662024-02-01T17:32:20.837Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.616075391Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"io task","client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107672024-02-01T17:32:20.837Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61730395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107682024-02-01T17:32:20.838Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.617360177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107692024-02-01T17:32:20.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.617402035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107702024-02-01T17:32:20.838Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61743687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107712024-02-01T17:32:20.838Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.618158827Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107722024-02-01T17:32:20.838Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61824147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107732024-02-01T17:32:20.838Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.61827232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107742024-02-01T17:32:20.838Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.618298037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107752024-02-01T17:32:20.839Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.619111356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107762024-02-01T17:32:20.839Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.619154204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107772024-02-01T17:32:20.839Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:08.619179069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107782024-02-01T17:32:20.839Z{Wait for a query_work_queue command to finish before sending IO
107792024-02-01T17:32:20.839Z"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165772474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107802024-02-01T17:32:20.839Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165837465Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107812024-02-01T17:32:20.840Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165861945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107822024-02-01T17:32:20.840Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165884486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107832024-02-01T17:32:20.840Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165906948Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107842024-02-01T17:32:20.905Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165929718Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107852024-02-01T17:32:20.909Z{"msg":"[0]R flush_numbers[0..12]: [143, 142, 147, 140, 135, 145, 147, 147, 147, 147, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165953348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107862024-02-01T17:32:20.910Z{"msg":"[0]R generation[0..12]: [96, 96, 98, 95, 92, 96, 98, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.165976638Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107872024-02-01T17:32:20.910Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166018868Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107882024-02-01T17:32:20.910Z{"msg":"[1]R flush_numbers[0..12]: [143, 142, 147, 140, 135, 145, 147, 147, 147, 147, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166046236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107892024-02-01T17:32:20.910Z{"msg":"[1]R generation[0..12]: [96, 96, 98, 95, 92, 96, 98, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166069794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107902024-02-01T17:32:20.911Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166093863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107912024-02-01T17:32:20.912Z{"msg":"[2]R flush_numbers[0..12]: [143, 142, 147, 140, 135, 145, 147, 147, 147, 147, 147, 144]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166117511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107922024-02-01T17:32:20.912Z{"msg":"[2]R generation[0..12]: [96, 96, 98, 95, 92, 96, 98, 98, 98, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166151432Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107932024-02-01T17:32:20.912Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166176474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107942024-02-01T17:32:20.912Z{"msg":"Max found gen is 99","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166198323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107952024-02-01T17:32:20.912Z{"msg":"Generation requested: 100 >= found:99","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166220444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107962024-02-01T17:32:20.913Z{"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166241757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107972024-02-01T17:32:20.913Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166263477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"":"downstairs","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107982024-02-01T17:32:20.913Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166285641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
107992024-02-01T17:32:20.913Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166305484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"0","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108002024-02-01T17:32:20.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166327671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"1","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108012024-02-01T17:32:20.914Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166349544Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"client":"2","":"downstairs","session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108022024-02-01T17:32:20.915Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166371562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108032024-02-01T17:32:20.915Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166392197Z"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108042024-02-01T17:32:20.915Z,"hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108052024-02-01T17:32:20.915Z{"msg":"cd181edd-36e7-46c0-b041-95a7a568b6c6 is now active with session: b8eb8199-026f-4182-b287-5d5febde014a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.1664192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108062024-02-01T17:32:20.915Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166439724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529,"session_id":"b8eb8199-026f-4182-b287-5d5febde014a"}
108072024-02-01T17:32:20.915Z{"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:19.166460347Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
108082024-02-01T17:32:20.916ZLoading write count information from file "/tmp/test_repair_verify.data"
108092024-02-01T17:32:20.916ZRun Repair workload
108102024-02-01T17:32:20.916Z01/30 Write block 117 len 24576 data: 7 6 7 7 5 6
108112024-02-01T17:32:20.916Z02/30 Read block 178 len 28672
108122024-02-01T17:32:20.916Z03/30 Write block 322 len 16384 data: 9 6 7 4
108132024-02-01T17:32:20.917Z04/30 Read block 104 len 36864
108142024-02-01T17:32:20.918Z05/30 Read block 135 len 32768
108152024-02-01T17:32:20.918Z06/30 Write block 502 len 8192 data: 7 5
108162024-02-01T17:32:20.919Z07/30 Flush
108172024-02-01T17:32:20.919Z08/30 Write block 516 len 36864 data: 5 6 8 8 8 8 8 9 7
108182024-02-01T17:32:20.919Z09/30 Read block 250 len 4096
108192024-02-01T17:32:20.919Z10/30 Read block 451 len 40960
108202024-02-01T17:32:20.920Z11/30 Read block 207 len 24576
108212024-02-01T17:32:20.920Z12/30 Read block 28 len 4096
108222024-02-01T17:32:20.920Z13/30 Read block 6 len 20480
108232024-02-01T17:32:20.920Z14/30 Read block 327 len 28672
108242024-02-01T17:32:20.921Z15/30 Write block 131 len 28672 data: 5 3 6 8 9 11 11
108252024-02-01T17:32:20.921Z16/30 Write block 134 len 32768 data: 9 10 12 12 12 9 10 10
108262024-02-01T17:32:20.921Z17/30 Write block 432 len 12288 data: 11 14 12
108272024-02-01T17:32:20.921Z18/30 Read block 56 len 16384
108282024-02-01T17:32:20.921Z19/30 Read block 367 len 28672
108292024-02-01T17:32:20.922Z20/30 Read block 84 len 16384
108302024-02-01T17:32:20.922Z21/30 Write block 285 len 40960 data: 9 10 8 8 8 6 9 8 9 9
108312024-02-01T17:32:20.922Z22/30 Read block 409 len 40960
108322024-02-01T17:32:20.922Z23/30 Read block 110 len 36864
108332024-02-01T17:32:20.922Z24/30 Flush
108342024-02-01T17:32:20.922Z25/30 Write block 237 len 40960 data: 5 6 7 6 5 7 6 6 6 7
108352024-02-01T17:32:20.922Z26/30 Read block 250 len 40960
108362024-02-01T17:32:20.922Z27/30 Write block 259 len 40960 data: 8 6 5 4 6 6 7 8 8 8
108372024-02-01T17:32:20.922Z28/30 Write block 317 len 32768 data: 10 9 9 10 8 10 7 8
108382024-02-01T17:32:20.922Z29/30 Write block 370 len 8192 data: 8 9
108392024-02-01T17:32:20.922Z30/30 Write block 53 len 4096 data: 10
108402024-02-01T17:32:20.923Z----------------------------------------------------------------
108412024-02-01T17:32:20.923Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
108422024-02-01T17:32:20.923ZError: Difference in extent metadata found!
108432024-02-01T17:32:20.923ZStates: Active Active Active
108442024-02-01T17:32:20.923ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
108452024-02-01T17:32:20.924Z 1 Acked 1000 Write 6 Done Done Done false
108462024-02-01T17:32:20.924Z 2 Acked 1001 Read 7 Done Done Done false
108472024-02-01T17:32:20.924Z 3 Acked 1002 Write 4 Done Done Done false
108482024-02-01T17:32:20.924Z 4 Acked 1003 Read 9 Done Done Done false
108492024-02-01T17:32:20.924Z 5 Acked 1004 Read 8 Done Done Done false
108502024-02-01T17:32:20.924Z 6 Acked 1005 Write 2 Done Done Done false
108512024-02-01T17:32:20.924Z 7 Acked 1006 Flush 0 Sent Done Done false
108522024-02-01T17:32:20.924Z 8 Acked 1007 Write 9 Sent Done Done false
108532024-02-01T17:32:20.924Z 9 Acked 1008 Read 1 Sent Done Done false
108542024-02-01T17:32:20.924Z 10 Acked 1009 Read 10 Sent Done Done false
108552024-02-01T17:32:20.924Z 11 Acked 1010 Read 6 Sent Done Done false
108562024-02-01T17:32:20.924Z 12 Acked 1011 Read 1 Sent Done Done false
108572024-02-01T17:32:20.925Z 13 Acked 1012 Read 5 Sent Done Done false
108582024-02-01T17:32:20.925Z 14 Acked 1013 Read 7 Sent Done Done false
108592024-02-01T17:32:20.925Z 15 Acked 1014 Write 7 Sent Done Done false
108602024-02-01T17:32:20.925Z 16 Acked 1015 Write 8 Sent Done Done false
108612024-02-01T17:32:20.925Z 17 Acked 1016 Write 3 Sent Done Done false
108622024-02-01T17:32:20.925Z 18 Acked 1017 Read 4 Sent Done Done false
108632024-02-01T17:32:20.926Z 19 Acked 1018 Read 7 Sent Done Done false
108642024-02-01T17:32:20.926Z 20 Acked 1019 Read 4 Sent Done Done false
108652024-02-01T17:32:20.926Z 21 Acked 1020 Write 10 Sent Done Done false
108662024-02-01T17:32:20.926Z 22 Acked 1021 Read 10 Sent Done Done false
108672024-02-01T17:32:20.926Z 23 Acked 1022 Read 9 Sent Done Done false
108682024-02-01T17:32:20.926Z 24 Acked 1023 Flush 0 Sent Done Done false
108692024-02-01T17:32:20.926Z 25 Acked 1024 Write 10 Sent Done Done false
108702024-02-01T17:32:20.927Z 26 Acked 1025 Read 10 Sent Done Done false
108712024-02-01T17:32:20.927Z 27 Acked 1026 Write 10 Sent Done Done false
108722024-02-01T17:32:20.927Z 28 Acked 1027 Write 8 Sent Done Sent false
108732024-02-01T17:32:20.927Z 29 Acked 1028 Write 2 Sent Sent Sent false
108742024-02-01T17:32:20.927Z 30 Acked 1029 Write 1 Sent Sent Sent false
108752024-02-01T17:32:20.927Z STATES DS:0 DS:1 DS:2 TOTAL
108762024-02-01T17:32:20.927Z New 0 0 0 0
108772024-02-01T17:32:20.927Z Sent 24 2 3 29
108782024-02-01T17:32:20.927Z Done 6 28 27 61
108792024-02-01T17:32:20.927Z Skipped 0 0 0 0
108802024-02-01T17:32:20.927Z Error 0 0 0 0
108812024-02-01T17:32:20.928ZLast Flush: 0 1023 1023
108822024-02-01T17:32:20.928ZDownstairs last five completed:
108832024-02-01T17:32:20.928ZUpstairs last five completed: 30 29 28 27 26
108842024-02-01T17:32:20.971Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:19.576707616Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1529}
108852024-02-01T17:32:20.971ZWrote out file "/tmp/test_repair_verify.data"
108862024-02-01T17:32:20.971Z
108872024-02-01T17:32:20.971ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
108882024-02-01T17:32:20.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
108892024-02-01T17:32:20.971Z 0 000-019 96 96 96 143 143 143 F F F
108902024-02-01T17:32:20.971Z 1 020-039 96 96 96 142 142 142 F F F
108912024-02-01T17:32:20.971Z 2 040-059 98 98 98 147 147 147 F T T <---
108922024-02-01T17:32:20.971Z 3 060-079 95 95 95 140 140 140 F F F
108932024-02-01T17:32:20.971Z 4 080-099 92 92 92 135 135 135 F F F
108942024-02-01T17:32:20.972Z 5 100-119 96 100 100 145 148 148 T F F <---
108952024-02-01T17:32:20.972Z 6 120-139 98 100 100 147 149 149 T F F <---
108962024-02-01T17:32:20.972Z 7 140-159 98 100 100 147 149 149 F F F <---
108972024-02-01T17:32:20.972Z 8 160-179 98 98 98 147 147 147 F F F
108982024-02-01T17:32:20.972Z 9 180-199 98 98 98 147 147 147 F F F
108992024-02-01T17:32:20.972Z 10 200-219 98 98 98 147 147 147 F F F
109002024-02-01T17:32:20.972Z 11 220-239 96 96 96 144 144 144 F T T <---
109012024-02-01T17:32:20.972Z 12 240-259 95 95 95 140 140 140 F T T <---
109022024-02-01T17:32:20.972Z 13 260-279 98 98 98 147 147 147 F T T <---
109032024-02-01T17:32:20.973Z 14 280-299 98 100 100 147 149 149 F F F <---
109042024-02-01T17:32:20.973Z 15 300-319 96 96 96 145 145 145 F T T <---
109052024-02-01T17:32:20.973Z 16 320-339 93 100 100 136 148 148 T T T <---
109062024-02-01T17:32:20.973Z 17 340-359 96 96 96 142 142 142 F F F
109072024-02-01T17:32:20.973Z 18 360-379 92 92 92 135 135 135 F T T <---
109082024-02-01T17:32:20.973Z 19 380-399 98 98 98 147 147 147 F F F
109092024-02-01T17:32:20.973Z 20 400-419 98 98 98 147 147 147 F F F
109102024-02-01T17:32:20.973Z 21 420-439 98 100 100 147 149 149 F F F <---
109112024-02-01T17:32:20.973Z 22 440-459 95 95 95 140 140 140 F F F
109122024-02-01T17:32:20.973Z 23 460-479 95 95 95 140 140 140 F F F
109132024-02-01T17:32:20.973Z 24 480-499 98 98 98 147 147 147 F F F
109142024-02-01T17:32:20.973Z 25 500-519 98 100 100 147 149 149 T F F <---
109152024-02-01T17:32:20.973Z 26 520-539 98 100 100 147 149 149 F F F <---
109162024-02-01T17:32:20.973Z 27 540-559 96 96 96 145 145 145 F F F
109172024-02-01T17:32:20.973Z 28 560-579 90 90 90 134 134 134 F F F
109182024-02-01T17:32:20.973Z 29 580-599 86 86 86 128 128 128 F F F
109192024-02-01T17:32:20.973ZMax gen: 100, Max flush: 149
109202024-02-01T17:32:20.973ZA Difference in extent metadata is expected here
109212024-02-01T17:32:20.973ZOn loop 50
109222024-02-01T17:32:20.973Z
109232024-02-01T17:32:20.973ZVerifying data now
109242024-02-01T17:32:26.545ZNo change after verify
109252024-02-01T17:32:26.545ZLoop: 50 Downstairs dump after verify (and repair):
109262024-02-01T17:32:26.567ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109272024-02-01T17:32:26.567Z 0 000-019 96 96 96 143 143 143 F F F
109282024-02-01T17:32:26.570Z 1 020-039 96 96 96 142 142 142 F F F
109292024-02-01T17:32:26.570Z 2 040-059 101 101 101 150 150 150 F F F
109302024-02-01T17:32:26.570Z 3 060-079 95 95 95 140 140 140 F F F
109312024-02-01T17:32:26.570Z 4 080-099 92 92 92 135 135 135 F F F
109322024-02-01T17:32:26.571Z 5 100-119 100 100 100 148 148 148 F F F
109332024-02-01T17:32:26.571Z 6 120-139 100 100 100 149 149 149 F F F
109342024-02-01T17:32:26.571Z 7 140-159 100 100 100 149 149 149 F F F
109352024-02-01T17:32:26.571Z 8 160-179 98 98 98 147 147 147 F F F
109362024-02-01T17:32:26.571Z 9 180-199 98 98 98 147 147 147 F F F
109372024-02-01T17:32:26.571Z 10 200-219 98 98 98 147 147 147 F F F
109382024-02-01T17:32:26.571Z 11 220-239 101 101 101 150 150 150 F F F
109392024-02-01T17:32:26.571Z 12 240-259 101 101 101 150 150 150 F F F
109402024-02-01T17:32:26.571Z 13 260-279 101 101 101 150 150 150 F F F
109412024-02-01T17:32:26.571Z 14 280-299 100 100 100 149 149 149 F F F
109422024-02-01T17:32:26.571Z 15 300-319 101 101 101 150 150 150 F F F
109432024-02-01T17:32:26.571Z 16 320-339 101 101 101 150 150 150 F F F
109442024-02-01T17:32:26.571Z 17 340-359 96 96 96 142 142 142 F F F
109452024-02-01T17:32:26.571Z 18 360-379 101 101 101 150 150 150 F F F
109462024-02-01T17:32:26.571Z 19 380-399 98 98 98 147 147 147 F F F
109472024-02-01T17:32:26.571Z 20 400-419 98 98 98 147 147 147 F F F
109482024-02-01T17:32:26.571Z 21 420-439 100 100 100 149 149 149 F F F
109492024-02-01T17:32:26.571Z 22 440-459 95 95 95 140 140 140 F F F
109502024-02-01T17:32:26.573Z 23 460-479 95 95 95 140 140 140 F F F
109512024-02-01T17:32:26.573Z 24 480-499 98 98 98 147 147 147 F F F
109522024-02-01T17:32:26.573Z 25 500-519 100 100 100 149 149 149 F F F
109532024-02-01T17:32:26.573Z 26 520-539 100 100 100 149 149 149 F F F
109542024-02-01T17:32:26.573Z 27 540-559 96 96 96 145 145 145 F F F
109552024-02-01T17:32:26.573Z 28 560-579 90 90 90 134 134 134 F F F
109562024-02-01T17:32:26.573Z 29 580-599 86 86 86 128 128 128 F F F
109572024-02-01T17:32:26.573ZMax gen: 101, Max flush: 150
109582024-02-01T17:32:26.576Z
109592024-02-01T17:32:26.576ZBegin loop 51
109602024-02-01T17:32:26.576ZDownstairs to restart: 2
109612024-02-01T17:32:26.603Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.876583677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
109622024-02-01T17:32:26.607Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877079543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
109632024-02-01T17:32:26.607Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877170209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
109642024-02-01T17:32:26.608Z{"msg":"Crucible 534c8750-5fd0-4ac7-94dd-b4b95be26aec has session id: 0524e113-4d72-4c69-9427-3333d2f981fc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877241545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"Crucible runtime is spawned
109652024-02-01T17:32:26.608Z0524e113-4d72-4c69-9427-3333d2f981fc"}
109662024-02-01T17:32:26.608Z{"msg":"Upstairs opts: Upstairs UUID: 534c8750-5fd0-4ac7-94dd-b4b95be26aec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877382421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109672024-02-01T17:32:26.608Z{"msg":"Crucible stats registered with UUID: 534c8750-5fd0-4ac7-94dd-b4b95be26aec","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877459098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109682024-02-01T17:32:26.608Z{"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877528926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
109692024-02-01T17:32:26.608Z{"msg":"534c8750-5fd0-4ac7-94dd-b4b95be26aec active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877594751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109702024-02-01T17:32:26.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877661021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109712024-02-01T17:32:26.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877740562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109722024-02-01T17:32:26.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877784924Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109732024-02-01T17:32:26.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877860132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109742024-02-01T17:32:26.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.87790224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109752024-02-01T17:32:26.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877949234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109762024-02-01T17:32:26.608Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.877991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109772024-02-01T17:32:26.609Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.878211458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109782024-02-01T17:32:26.609Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.878301327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109792024-02-01T17:32:26.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.878347264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109802024-02-01T17:32:26.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.878424496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109812024-02-01T17:32:26.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.878535253Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"io task","client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109822024-02-01T17:32:26.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.879523078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109832024-02-01T17:32:26.609Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.879628772Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109842024-02-01T17:32:26.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.87976658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109852024-02-01T17:32:26.609Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.87984439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109862024-02-01T17:32:26.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.880073484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109872024-02-01T17:32:26.618Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.880158832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109882024-02-01T17:32:26.626Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.880586713Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109892024-02-01T17:32:26.626Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.880839023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109902024-02-01T17:32:26.626Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881387865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109912024-02-01T17:32:26.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881462095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109922024-02-01T17:32:26.626Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881510374Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109932024-02-01T17:32:26.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881575477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109942024-02-01T17:32:26.626Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.88161927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109952024-02-01T17:32:26.626Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881694287Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109962024-02-01T17:32:26.626Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.881738608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109972024-02-01T17:32:26.627Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882172842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109982024-02-01T17:32:26.627Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882235581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
109992024-02-01T17:32:26.627Z{"msg":"[0]R flush_numbers[0..12]: [143, 142, 150, 140, 135, 148, 149, 149, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882264205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110002024-02-01T17:32:26.627Z{"msg":"[0]R generation[0..12]: [96, 96, 101, 95, 92, 100, 100, 100, 98, 98, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882306528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110012024-02-01T17:32:26.627Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882332961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110022024-02-01T17:32:26.627Z{"msg":"[1]R flush_numbers[0..12]: [143, 142, 150, 140, 135, 148, 149, 149, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882374866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110032024-02-01T17:32:26.627Z{"msg":"[1]R generation[0..12]: [96, 96, 101, 95, 92, 100, 100, 100, 98, 98, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882400108Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110042024-02-01T17:32:26.627Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882442126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":Wait for a query_work_queue command to finish before sending IO
110052024-02-01T17:32:26.627Z1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110062024-02-01T17:32:26.628Z{"msg":"[2]R flush_numbers[0..12]: [143, 142, 150, 140, 135, 148, 149, 149, 147, 147, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882472144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110072024-02-01T17:32:26.628Z{"msg":"[2]R generation[0..12]: [96, 96, 101, 95, 92, 100, 100, 100, 98, 98, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882526861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110082024-02-01T17:32:26.628Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882554646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110092024-02-01T17:32:26.628Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882614615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110102024-02-01T17:32:26.628Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882657184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110112024-02-01T17:32:26.628Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882713833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110122024-02-01T17:32:26.628Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.88275611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"":"downstairs","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110132024-02-01T17:32:26.628Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882819437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110142024-02-01T17:32:26.628Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882858659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"0","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110152024-02-01T17:32:26.628Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.882914218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"1","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110162024-02-01T17:32:26.628Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.8829576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"client":"2","":"downstairs","session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110172024-02-01T17:32:26.628Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
110182024-02-01T17:32:26.628Z"time":"2024-02-01T17:32:25.883027206Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110192024-02-01T17:32:26.628Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.883073672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110202024-02-01T17:32:26.628Z{"msg":"534c8750-5fd0-4ac7-94dd-b4b95be26aec is now active with session: 0524e113-4d72-4c69-9427-3333d2f981fc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.883135624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110212024-02-01T17:32:26.628Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.883176299Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537,"session_id":"0524e113-4d72-4c69-9427-3333d2f981fc"}
110222024-02-01T17:32:26.628Z{"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:25.883227068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
110232024-02-01T17:32:26.628ZLoading write count information from file "/tmp/test_repair_verify.data"
110242024-02-01T17:32:26.628ZRun Repair workload
110252024-02-01T17:32:26.628Z01/30 Write block 365 len 40960 data: 8 9 8 6 6 9 10 9 9 8
110262024-02-01T17:32:26.628Z02/30 Write block 44 len 20480 data: 10 10 11 9 12
110272024-02-01T17:32:26.628Z03/30 Write block 480 len 12288 data: 7 8 8
110282024-02-01T17:32:26.628Z04/30 Write block 531 len 32768 data: 7 9 10 13 12 9 11 8
110292024-02-01T17:32:26.629Z05/30 Write block 416 len 28672 data: 12 7 8 9 7 6 8
110302024-02-01T17:32:26.629Z06/30 Flush
110312024-02-01T17:32:26.629Z07/30 Write block 137 len 12288 data: 13 13 10
110322024-02-01T17:32:26.629Z08/30 Read block 119 len 12288
110332024-02-01T17:32:26.634Z09/30 Read block 460 len 24576
110342024-02-01T17:32:26.637Z10/30 Flush
110352024-02-01T17:32:26.637Z11/30 Write block 461 len 4096 data: 12
110362024-02-01T17:32:26.637Z12/30 Read block 288 len 20480
110372024-02-01T17:32:26.637Z13/30 Read block 480 len 16384
110382024-02-01T17:32:26.637Z14/30 Read block 9 len 8192
110392024-02-01T17:32:26.637Z15/30 Write block 541 len 28672 data: 13 14 11 13 11 12 12
110402024-02-01T17:32:26.637Z16/30 Read block 535 len 16384
110412024-02-01T17:32:26.641Z17/30 Read block 577 len 12288
110422024-02-01T17:32:26.641Z18/30 Read block 380 len 20480
110432024-02-01T17:32:26.641Z19/30 Flush
110442024-02-01T17:32:26.645Z20/30 Write block 190 len 40960 data: 9 8 5 6 7 7 9 8 5 6
110452024-02-01T17:32:26.645Z21/30 Read block 486 len 16384
110462024-02-01T17:32:26.650Z22/30 Write block 421 len 32768 data: 7 9 8 8 11 12 8 7
110472024-02-01T17:32:26.650Z23/30 Write block 285 len 28672 data: 10 11 9 9 9 7 10
110482024-02-01T17:32:26.650Z24/30 Write block 371 len 28672 data: 11 10 10 9 10 12 13
110492024-02-01T17:32:26.650Z25/30 Read block 428 len 40960
110502024-02-01T17:32:26.657Z26/30 Write block 132 len 16384 data: 4 7 10 11
110512024-02-01T17:32:26.657Z27/30 Write block 64 len 32768 data: 7 7 9 9 10 13 10 12
110522024-02-01T17:32:26.657Z28/30 Write block 510 len 36864 data: 7 8 8 8 6 6 6 7 9
110532024-02-01T17:32:26.657Z29/30 Read block 94 len 28672
110542024-02-01T17:32:26.663Z30/30 Read block 52 len 24576
110552024-02-01T17:32:26.663Z----------------------------------------------------------------
110562024-02-01T17:32:26.663Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30
110572024-02-01T17:32:26.663ZStates: Active Active Active
110582024-02-01T17:32:26.663ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
110592024-02-01T17:32:26.664Z 1 Acked 1000 Write 10 Done Done Done false
110602024-02-01T17:32:26.664Z 2 Acked 1001 Write 5 Done Done Done false
110612024-02-01T17:32:26.664Z 3 Acked 1002 Write 3 Done Done Sent false
110622024-02-01T17:32:26.664Z 4 Acked 1003 Write 8 Done Done Sent false
110632024-02-01T17:32:26.664Z 5 Acked 1004 Write 7 Done Done Sent false
110642024-02-01T17:32:26.664Z 6 Acked 1005 Flush 0 Done Done Sent false
110652024-02-01T17:32:26.664Z 7 Acked 1006 Write 3 Done Done Sent false
110662024-02-01T17:32:26.665Z 8 Acked 1007 Read 3 Done Done Sent false
110672024-02-01T17:32:26.665Z 9 Acked 1008 Read 6 Done Done Sent false
110682024-02-01T17:32:26.665Z 10 Acked 1009 Flush 0 Done Done Sent false
110692024-02-01T17:32:26.665Z 11 Acked 1010 Write 1 Done Done Sent false
110702024-02-01T17:32:26.665Z 12 Acked 1011 Read 5 Done Done Sent false
110712024-02-01T17:32:26.665Z 13 Acked 1012 Read 4 Done Done Sent false
110722024-02-01T17:32:26.665Z 14 Acked 1013 Read 2 Done Done Sent false
110732024-02-01T17:32:26.665Z 15 Acked 1014 Write 7 Done Done Sent false
110742024-02-01T17:32:26.665Z 16 Acked 1015 Read 4 Done Done Sent false
110752024-02-01T17:32:26.665Z 17 Acked 1016 Read 3 Done Done Sent false
110762024-02-01T17:32:26.665Z 18 Acked 1017 Read 5 Done Done Sent false
110772024-02-01T17:32:26.665Z 19 Acked 1018 Flush 0 Done Done Sent false
110782024-02-01T17:32:26.665Z 20 Acked 1019 Write 10 Done Done Sent false
110792024-02-01T17:32:26.665Z 21 Acked 1020 Read 4 Done Done Sent false
110802024-02-01T17:32:26.665Z 22 Acked 1021 Write 8 Done Done Sent false
110812024-02-01T17:32:26.665Z 23 Acked 1022 Write 7 Done Done Sent false
110822024-02-01T17:32:26.665Z 24 Acked 1023 Write 7 Done Done Sent false
110832024-02-01T17:32:26.665Z 25 Acked 1024 Read 10 Done Done Sent false
110842024-02-01T17:32:26.665Z 26 Acked 1025 Write 4 Done Done Sent false
110852024-02-01T17:32:26.665Z 27 Acked 1026 Write 8 Done Done Sent false
110862024-02-01T17:32:26.665Z 28 Acked 1027 Write 9 Done Done Sent false
110872024-02-01T17:32:26.665Z 29 Acked 1028 Read 7 Done Done Sent false
110882024-02-01T17:32:26.665Z 30 Acked 1029 Read 6 Done Done Sent false
110892024-02-01T17:32:26.665Z STATES DS:0 DS:1 DS:2 TOTAL
110902024-02-01T17:32:26.666Z New 0 0 0 0
110912024-02-01T17:32:26.666Z Sent 0 0 28 28
110922024-02-01T17:32:26.666Z Done 30 30 2 62
110932024-02-01T17:32:26.666Z Skipped 0 0 0 0
110942024-02-01T17:32:26.666Z Error 0 0 0 0
110952024-02-01T17:32:26.666ZLast Flush: 1018 1018 0
110962024-02-01T17:32:26.666ZDownstairs last five completed:
110972024-02-01T17:32:26.666ZUpstairs last five completed: 30 29 28 27 26
110982024-02-01T17:32:26.666Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:25.932361832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1537}
110992024-02-01T17:32:26.666ZWrote out file "/tmp/test_repair_verify.data"
111002024-02-01T17:32:26.669Z
111012024-02-01T17:32:26.672ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111022024-02-01T17:32:26.696ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111032024-02-01T17:32:26.697Z 0 000-019 96 96 96 143 143 143 F F F
111042024-02-01T17:32:26.700Z 1 020-039 96 96 96 142 142 142 F F F
111052024-02-01T17:32:26.700Z 2 040-059 102 102 101 151 151 150 F F T <---
111062024-02-01T17:32:26.701Z 3 060-079 95 95 95 140 140 140 T T F <---
111072024-02-01T17:32:26.701Z 4 080-099 92 92 92 135 135 135 F F F
111082024-02-01T17:32:26.701Z 5 100-119 100 100 100 148 148 148 F F F
111092024-02-01T17:32:26.701Z 6 120-139 102 102 100 152 152 149 T T F <---
111102024-02-01T17:32:26.701Z 7 140-159 100 100 100 149 149 149 F F F
111112024-02-01T17:32:26.701Z 8 160-179 98 98 98 147 147 147 F F F
111122024-02-01T17:32:26.701Z 9 180-199 98 98 98 147 147 147 T T F <---
111132024-02-01T17:32:26.701Z 10 200-219 98 98 98 147 147 147 F F F
111142024-02-01T17:32:26.701Z 11 220-239 101 101 101 150 150 150 F F F
111152024-02-01T17:32:26.701Z 12 240-259 101 101 101 150 150 150 F F F
111162024-02-01T17:32:26.701Z 13 260-279 101 101 101 150 150 150 F F F
111172024-02-01T17:32:26.701Z 14 280-299 100 100 100 149 149 149 T T F <---
111182024-02-01T17:32:26.701Z 15 300-319 101 101 101 150 150 150 F F F
111192024-02-01T17:32:26.701Z 16 320-339 101 101 101 150 150 150 F F F
111202024-02-01T17:32:26.702Z 17 340-359 96 96 96 142 142 142 F F F
111212024-02-01T17:32:26.702Z 18 360-379 102 102 101 151 151 150 T T T <---
111222024-02-01T17:32:26.702Z 19 380-399 98 98 98 147 147 147 F F F
111232024-02-01T17:32:26.702Z 20 400-419 102 102 98 151 151 147 F F F <---
111242024-02-01T17:32:26.702Z 21 420-439 102 102 100 151 151 149 T T F <---
111252024-02-01T17:32:26.702Z 22 440-459 95 95 95 140 140 140 F F F
111262024-02-01T17:32:26.702Z 23 460-479 102 102 95 153 153 140 F F F <---
111272024-02-01T17:32:26.702Z 24 480-499 102 102 98 151 151 147 F F F <---
111282024-02-01T17:32:26.702Z 25 500-519 100 100 100 149 149 149 T T F <---
111292024-02-01T17:32:26.702Z 26 520-539 102 102 100 151 151 149 F F F <---
111302024-02-01T17:32:26.702Z 27 540-559 102 102 96 153 153 145 F F F <---
111312024-02-01T17:32:26.702Z 28 560-579 90 90 90 134 134 134 F F F
111322024-02-01T17:32:26.702Z 29 580-599 86 86 86 128 128 128 F F F
111332024-02-01T17:32:26.702ZMax gen: 102, Max flush: 153
111342024-02-01T17:32:26.702ZError: Difference in extent metadata found!
111352024-02-01T17:32:26.708ZA Difference in extent metadata is expected here
111362024-02-01T17:32:26.708ZOn loop 51
111372024-02-01T17:32:26.708Z
111382024-02-01T17:32:26.708ZVerifying data now
111392024-02-01T17:32:32.010ZNo change after verify
111402024-02-01T17:32:32.010ZLoop: 51 Downstairs dump after verify (and repair):
111412024-02-01T17:32:32.684ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111422024-02-01T17:32:32.684Z 0 000-019 96 96 96 143 143 143 F F F
111432024-02-01T17:32:32.684Z 1 020-039 96 96 96 142 142 142 F F F
111442024-02-01T17:32:32.684Z 2 040-059 102 102 102 151 151 151 F F F
111452024-02-01T17:32:32.684Z 3 060-079 103 103 103 154 154 154 F F F
111462024-02-01T17:32:32.685Z 4 080-099 92 92 92 135 135 135 F F F
111472024-02-01T17:32:32.685Z 5 100-119 100 100 100 148 148 148 F F F
111482024-02-01T17:32:32.685Z 6 120-139 103 103 103 154 154 154 F F F
111492024-02-01T17:32:32.685Z 7 140-159 100 100 100 149 149 149 F F F
111502024-02-01T17:32:32.685Z 8 160-179 98 98 98 147 147 147 F F F
111512024-02-01T17:32:32.685Z 9 180-199 103 103 103 154 154 154 F F F
111522024-02-01T17:32:32.685Z 10 200-219 98 98 98 147 147 147 F F F
111532024-02-01T17:32:32.685Z 11 220-239 101 101 101 150 150 150 F F F
111542024-02-01T17:32:32.685Z 12 240-259 101 101 101 150 150 150 F F F
111552024-02-01T17:32:32.685Z 13 260-279 101 101 101 150 150 150 F F F
111562024-02-01T17:32:32.686Z 14 280-299 103 103 103 154 154 154 F F F
111572024-02-01T17:32:32.686Z 15 300-319 101 101 101 150 150 150 F F F
111582024-02-01T17:32:32.686Z 16 320-339 101 101 101 150 150 150 F F F
111592024-02-01T17:32:32.686Z 17 340-359 96 96 96 142 142 142 F F F
111602024-02-01T17:32:32.686Z 18 360-379 103 103 103 154 154 154 F F F
111612024-02-01T17:32:32.686Z 19 380-399 98 98 98 147 147 147 F F F
111622024-02-01T17:32:32.686Z 20 400-419 102 102 102 151 151 151 F F F
111632024-02-01T17:32:32.686Z 21 420-439 103 103 103 154 154 154 F F F
111642024-02-01T17:32:32.686Z 22 440-459 95 95 95 140 140 140 F F F
111652024-02-01T17:32:32.686Z 23 460-479 102 102 102 153 153 153 F F F
111662024-02-01T17:32:32.686Z 24 480-499 102 102 102 151 151 151 F F F
111672024-02-01T17:32:32.686Z 25 500-519 103 103 103 154 154 154 F F F
111682024-02-01T17:32:32.687Z 26 520-539 102 102 102 151 151 151 F F F
111692024-02-01T17:32:32.687Z 27 540-559 102 102 102 153 153 153 F F F
111702024-02-01T17:32:32.687Z 28 560-579 90 90 90 134 134 134 F F F
111712024-02-01T17:32:32.687Z 29 580-599 86 86 86 128 128 128 F F F
111722024-02-01T17:32:32.687ZMax gen: 103, Max flush: 154
111732024-02-01T17:32:32.687Z
111742024-02-01T17:32:32.687ZBegin loop 52
111752024-02-01T17:32:32.687ZDownstairs to restart: 0
111762024-02-01T17:32:32.687Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.159548887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
111772024-02-01T17:32:32.687Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160015276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
111782024-02-01T17:32:32.688Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160089058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
111792024-02-01T17:32:32.688Z{"msg":"Crucible cbdda771-9676-4430-90c7-9c24de5f71b9 has session id: 57d76b0a-e3cc-4894-a4f6-229c5aef7e4d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160129373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid"Crucible runtime is spawned
111802024-02-01T17:32:32.688Z:1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111812024-02-01T17:32:32.688Z{"msg":"Upstairs opts: Upstairs UUID: cbdda771-9676-4430-90c7-9c24de5f71b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160311963Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111822024-02-01T17:32:32.688Z{"msg":"Crucible stats registered with UUID: cbdda771-9676-4430-90c7-9c24de5f71b9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16038848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111832024-02-01T17:32:32.689Z{"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160441912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
111842024-02-01T17:32:32.689Z{"msg":"cbdda771-9676-4430-90c7-9c24de5f71b9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160612488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111852024-02-01T17:32:32.689Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160655417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111862024-02-01T17:32:32.689Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160689637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111872024-02-01T17:32:32.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160763172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111882024-02-01T17:32:32.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160808266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111892024-02-01T17:32:32.690Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160887122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111902024-02-01T17:32:32.690Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.160932865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111912024-02-01T17:32:32.690Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16097304Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111922024-02-01T17:32:32.690Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.161011072Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111932024-02-01T17:32:32.690Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.161118511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111942024-02-01T17:32:32.690Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.161164675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111952024-02-01T17:32:32.690Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.161212376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111962024-02-01T17:32:32.690Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.161289275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"io task","client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111972024-02-01T17:32:32.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.162616031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111982024-02-01T17:32:32.691Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.162684957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
111992024-02-01T17:32:32.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.162711291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112002024-02-01T17:32:32.691Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.162733639Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112012024-02-01T17:32:32.691Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.162757314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112022024-02-01T17:32:32.691Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16278842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112032024-02-01T17:32:32.692Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.163665081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112042024-02-01T17:32:32.692Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.163949035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112052024-02-01T17:32:32.692Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.164058456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112062024-02-01T17:32:32.692Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.164467226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112072024-02-01T17:32:32.692Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16453071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112082024-02-01T17:32:32.692Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.164557803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112092024-02-01T17:32:32.692Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165048918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112102024-02-01T17:32:32.692Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165143328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112112024-02-01T17:32:32.693Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165222241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112122024-02-01T17:32:32.693Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165290565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112132024-02-01T17:32:32.693Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165360372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112142024-02-01T17:32:32.693Z{"msg":"[0]R flush_numbers[0..12]: [143, 142, 151, 154, 135, 148, 154, 149, 147, 154, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165431594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112152024-02-01T17:32:32.693Z{"msg":"[0]R generation[0..12]: [96, 96, 102, 103, 92, 100, 103, 100, 98, 103, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165501455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112162024-02-01T17:32:32.693Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165573997Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112172024-02-01T17:32:32.693ZWait for a query_work_queue command to finish before sending IO
112182024-02-01T17:32:32.693Z{"msg":"[1]R flush_numbers[0..12]: [143, 142, 151, 154, 135, 148, 154, 149, 147, 154, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165628134Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112192024-02-01T17:32:32.693Z{"msg":"[1]R generation[0..12]: [96, 96, 102, 103, 92, 100, 103, 100, 98, 103, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16570581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112202024-02-01T17:32:32.693Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165754462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112212024-02-01T17:32:32.693Z{"msg":"[2]R flush_numbers[0..12]: [143, 142, 151, 154, 135, 148, 154, 149, 147, 154, 147, 150]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165825887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112222024-02-01T17:32:32.694Z{"msg":"[2]R generation[0..12]: [96, 96, 102, 103, 92, 100, 103, 100, 98, 103, 98, 101]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165870537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112232024-02-01T17:32:32.694Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.165939387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112242024-02-01T17:32:32.694Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166013425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112252024-02-01T17:32:32.694Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166083315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112262024-02-01T17:32:32.694Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166128153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112272024-02-01T17:32:32.694Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166199584Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"":"downstairs","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112282024-02-01T17:32:32.694Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166278135Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112292024-02-01T17:32:32.695Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16631918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"0","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112302024-02-01T17:32:32.695Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16636047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"1","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112312024-02-01T17:32:32.695Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166429582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"client":"2","":"downstairs","session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112322024-02-01T17:32:32.695Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166478039Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112332024-02-01T17:32:32.696Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
112342024-02-01T17:32:32.696Z,"time":"2024-02-01T17:32:31.166545708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112352024-02-01T17:32:32.696Z{"msg":"cbdda771-9676-4430-90c7-9c24de5f71b9 is now active with session: 57d76b0a-e3cc-4894-a4f6-229c5aef7e4d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166594785Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112362024-02-01T17:32:32.696Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.16665921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545,"session_id":"57d76b0a-e3cc-4894-a4f6-229c5aef7e4d"}
112372024-02-01T17:32:32.696Z{"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:31.166689349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
112382024-02-01T17:32:32.696ZLoading write count information from file "/tmp/test_repair_verify.data"
112392024-02-01T17:32:32.696ZRun Repair workload
112402024-02-01T17:32:32.697Z01/30 Write block 339 len 12288 data: 7 5 5
112412024-02-01T17:32:32.953Z02/30 Flush
112422024-02-01T17:32:32.953Z03/30 Write block 327 len 28672 data: 7 6 6 7 9 9 10
112432024-02-01T17:32:32.953Z04/30 Read block 456 len 28672
112442024-02-01T17:32:32.953Z05/30 Read block 482 len 40960
112452024-02-01T17:32:32.953Z06/30 Read block 290 len 16384
112462024-02-01T17:32:32.953Z07/30 Write block 267 len 32768 data: 9 9 8 9 8 7 7 6
112472024-02-01T17:32:32.954Z08/30 Write block 354 len 32768 data: 13 11 9 10 11 11 11 12
112482024-02-01T17:32:32.954Z09/30 Write block 474 len 20480 data: 5 7 7 6 6
112492024-02-01T17:32:32.954Z10/30 Write block 331 len 8192 data: 10 10
112502024-02-01T17:32:32.954Z11/30 Read block 533 len 40960
112512024-02-01T17:32:32.954Z12/30 Read block 497 len 28672
112522024-02-01T17:32:32.954Z13/30 Read block 15 len 40960
112532024-02-01T17:32:32.954Z14/30 Write block 229 len 28672 data: 7 6 6 5 5 6 5
112542024-02-01T17:32:32.954Z15/30 Read block 77 len 36864
112552024-02-01T17:32:32.955Z16/30 Write block 200 len 20480 data: 8 9 11 13 13
112562024-02-01T17:32:32.955Z17/30 Flush
112572024-02-01T17:32:32.955Z18/30 Write block 504 len 32768 data: 3 3 4 4 7 8 8 9
112582024-02-01T17:32:32.955Z19/30 Write block 402 len 36864 data: 6 8 7 6 7 8 10 7 4
112592024-02-01T17:32:32.956Z20/30 Read block 313 len 20480
112602024-02-01T17:32:32.956Z21/30 Read block 454 len 4096
112612024-02-01T17:32:32.956Z22/30 Write block 82 len 28672 data: 4 6 5 4 5 4 5
112622024-02-01T17:32:32.956Z23/30 Write block 186 len 40960 data: 11 11 8 9 10 9 6 7 8 8
112632024-02-01T17:32:32.956Z24/30 Read block 294 len 32768
112642024-02-01T17:32:32.956Z25/30 Read block 260 len 32768
112652024-02-01T17:32:32.956Z26/30 Write block 403 len 8192 data: 9 8
112662024-02-01T17:32:32.957Z27/30 Write block 397 len 16384 data: 6 5 3 4
112672024-02-01T17:32:32.957Z28/30 Write block 559 len 4096 data: 11
112682024-02-01T17:32:32.957Z29/30 Read block 218 len 32768
112692024-02-01T17:32:32.957Z30/30 Read block 406 len 36864
112702024-02-01T17:32:32.957Z----------------------------------------------------------------
112712024-02-01T17:32:32.957Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:28
112722024-02-01T17:32:32.957ZStates: Active Active Active
112732024-02-01T17:32:32.957ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
112742024-02-01T17:32:32.957Z 3 Acked 1002 Write 7 Done Done Done false
112752024-02-01T17:32:32.958Z 4 Acked 1003 Read 7 Done Done Done false
112762024-02-01T17:32:32.958Z 5 Acked 1004 Read 10 Done Done Done false
112772024-02-01T17:32:32.958Z 6 Acked 1005 Read 4 Sent Done Done false
112782024-02-01T17:32:32.958Z 7 Acked 1006 Write 8 Sent Done Done false
112792024-02-01T17:32:32.958Z 8 Acked 1007 Write 8 Sent Done Done false
112802024-02-01T17:32:32.958Z 9 Acked 1008 Write 5 Sent Done Done false
112812024-02-01T17:32:32.958Z 10 Acked 1009 Write 2 Sent Done Done false
112822024-02-01T17:32:32.958Z 11 Acked 1010 Read 10 Sent Done Done false
112832024-02-01T17:32:32.959Z 12 Acked 1011 Read 7 Sent Done Done false
112842024-02-01T17:32:32.959Z 13 Acked 1012 Read 10 Sent Done Done false
112852024-02-01T17:32:32.959Z 14 Acked 1013 Write 7 Sent Done Done false
112862024-02-01T17:32:32.959Z 15 Acked 1014 Read 9 Sent Done Done false
112872024-02-01T17:32:32.959Z 16 Acked 1015 Write 5 Sent Done Done false
112882024-02-01T17:32:32.959Z 17 Acked 1016 Flush 0 Sent Done Done false
112892024-02-01T17:32:32.959Z 18 Acked 1017 Write 8 Sent Done Done false
112902024-02-01T17:32:32.959Z 19 Acked 1018 Write 9 Sent Done Done false
112912024-02-01T17:32:32.960Z 20 Acked 1019 Read 5 Sent Done Done false
112922024-02-01T17:32:32.960Z 21 Acked 1020 Read 1 Sent Done Done false
112932024-02-01T17:32:32.960Z 22 Acked 1021 Write 7 Sent Done Done false
112942024-02-01T17:32:32.960Z 23 Acked 1022 Write 10 Sent Done Done false
112952024-02-01T17:32:32.960Z 24 Acked 1023 Read 8 Sent Done Done false
112962024-02-01T17:32:32.960Z 25 Acked 1024 Read 8 Sent Done Done false
112972024-02-01T17:32:32.960Z 26 Acked 1025 Write 2 Sent Done Done false
112982024-02-01T17:32:32.960Z 27 Acked 1026 Write 4 Sent Done Done false
112992024-02-01T17:32:32.961Z 28 Acked 1027 Write 1 Sent Done Done false
113002024-02-01T17:32:32.961Z 29 Acked 1028 Read 8 Sent Done Done false
113012024-02-01T17:32:32.961Z 30 Acked 1029 Read 9 Sent Sent Done false
113022024-02-01T17:32:32.961Z STATES DS:0 DS:1 DS:2 TOTAL
113032024-02-01T17:32:32.961Z New 0 0 0 0
113042024-02-01T17:32:32.961Z Sent 25 1 0 26
113052024-02-01T17:32:32.961Z Done 3 27 28 58
113062024-02-01T17:32:32.961Z Skipped 0 0 0 0
113072024-02-01T17:32:32.961Z Error 0 0 0 0
113082024-02-01T17:32:32.962ZLast Flush: 1001 1016 1016
113092024-02-01T17:32:32.962ZDownstairs last five completed: 1001 1000
113102024-02-01T17:32:32.962ZUpstairs last five completed: 30 29 28 27 26
113112024-02-01T17:32:32.962Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:31.216652435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1545}
113122024-02-01T17:32:32.962ZWrote out file "/tmp/test_repair_verify.data"
113132024-02-01T17:32:32.962Z
113142024-02-01T17:32:32.962ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
113152024-02-01T17:32:32.962ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113162024-02-01T17:32:32.962Z 0 000-019 96 96 96 143 143 143 F F F
113172024-02-01T17:32:32.963Z 1 020-039 96 96 96 142 142 142 F F F
113182024-02-01T17:32:32.963Z 2 040-059 102 102 102 151 151 151 F F F
113192024-02-01T17:32:32.963Z 3 060-079 103 103 103 154 154 154 F F F
113202024-02-01T17:32:32.963Z 4 080-099 92 92 92 135 135 135 F T T <---
113212024-02-01T17:32:32.963Z 5 100-119 100 100 100 148 148 148 F F F
113222024-02-01T17:32:32.963Z 6 120-139 103 103 103 154 154 154 F F F
113232024-02-01T17:32:32.963Z 7 140-159 100 100 100 149 149 149 F F F
113242024-02-01T17:32:32.963Z 8 160-179 98 98 98 147 147 147 F F F
113252024-02-01T17:32:32.964Z 9 180-199 103 103 103 154 154 154 F T T <---
113262024-02-01T17:32:32.964Z 10 200-219 98 104 104 147 156 156 F F F <---
113272024-02-01T17:32:32.964Z 11 220-239 101 104 104 150 156 156 F F F <---
113282024-02-01T17:32:32.964Z 12 240-259 101 101 101 150 150 150 F F F
113292024-02-01T17:32:32.964Z 13 260-279 101 104 104 150 156 156 F F F <---
113302024-02-01T17:32:32.964Z 14 280-299 103 103 103 154 154 154 F F F
113312024-02-01T17:32:32.964Z 15 300-319 101 101 101 150 150 150 F F F
113322024-02-01T17:32:32.964Z 16 320-339 104 104 104 155 156 156 T F F <---
113332024-02-01T17:32:32.964Z 17 340-359 104 104 104 155 156 156 F F F <---
113342024-02-01T17:32:32.965Z 18 360-379 103 104 104 154 156 156 F F F <---
113352024-02-01T17:32:32.965Z 19 380-399 98 98 98 147 147 147 F T T <---
113362024-02-01T17:32:32.965Z 20 400-419 102 102 102 151 151 151 F T T <---
113372024-02-01T17:32:32.965Z 21 420-439 103 103 103 154 154 154 F F F
113382024-02-01T17:32:32.965Z 22 440-459 95 95 95 140 140 140 F F F
113392024-02-01T17:32:32.965Z 23 460-479 102 104 104 153 156 156 F F F <---
113402024-02-01T17:32:32.965Z 24 480-499 102 102 102 151 151 151 F F F
113412024-02-01T17:32:32.974Z 25 500-519 103 103 103 154 154 154 F T T <---
113422024-02-01T17:32:32.974Z 26 520-539 102 102 102 151 151 151 F F F
113432024-02-01T17:32:32.974Z 27 540-559 102 102 102 153 153 153 F T T <---
113442024-02-01T17:32:32.974Z 28 560-579 90 90 90 134 134 134 F F F
113452024-02-01T17:32:32.974Z 29 580-599 86 86 86 128 128 128 F F F
113462024-02-01T17:32:32.974ZMax gen: 104, Max flush: 156
113472024-02-01T17:32:32.975ZError: Difference in extent metadata found!
113482024-02-01T17:32:32.975ZA Difference in extent metadata is expected here
113492024-02-01T17:32:32.975ZOn loop 52
113502024-02-01T17:32:32.975Z
113512024-02-01T17:32:32.975ZVerifying data now
113522024-02-01T17:32:37.120ZNo change after verify
113532024-02-01T17:32:37.120ZLoop: 52 Downstairs dump after verify (and repair):
113542024-02-01T17:32:37.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
113552024-02-01T17:32:37.143Z 0 000-019 96 96 96 143 143 143 F F F
113562024-02-01T17:32:37.151Z 1 020-039 96 96 96 142 142 142 F F F
113572024-02-01T17:32:37.151Z 2 040-059 102 102 102 151 151 151 F F F
113582024-02-01T17:32:37.151Z 3 060-079 103 103 103 154 154 154 F F F
113592024-02-01T17:32:37.151Z 4 080-099 105 105 105 157 157 157 F F F
113602024-02-01T17:32:37.151Z 5 100-119 100 100 100 148 148 148 F F F
113612024-02-01T17:32:37.151Z 6 120-139 103 103 103 154 154 154 F F F
113622024-02-01T17:32:37.151Z 7 140-159 100 100 100 149 149 149 F F F
113632024-02-01T17:32:37.151Z 8 160-179 98 98 98 147 147 147 F F F
113642024-02-01T17:32:37.152Z 9 180-199 105 105 105 157 157 157 F F F
113652024-02-01T17:32:37.152Z 10 200-219 104 104 104 156 156 156 F F F
113662024-02-01T17:32:37.152Z 11 220-239 104 104 104 156 156 156 F F F
113672024-02-01T17:32:37.152Z 12 240-259 101 101 101 150 150 150 F F F
113682024-02-01T17:32:37.152Z 13 260-279 104 104 104 156 156 156 F F F
113692024-02-01T17:32:37.153Z 14 280-299 103 103 103 154 154 154 F F F
113702024-02-01T17:32:37.153Z 15 300-319 101 101 101 150 150 150 F F F
113712024-02-01T17:32:37.153Z 16 320-339 104 104 104 156 156 156 F F F
113722024-02-01T17:32:37.153Z 17 340-359 104 104 104 156 156 156 F F F
113732024-02-01T17:32:37.153Z 18 360-379 104 104 104 156 156 156 F F F
113742024-02-01T17:32:37.153Z 19 380-399 105 105 105 157 157 157 F F F
113752024-02-01T17:32:37.153Z 20 400-419 105 105 105 157 157 157 F F F
113762024-02-01T17:32:37.153Z 21 420-439 103 103 103 154 154 154 F F F
113772024-02-01T17:32:37.154Z 22 440-459 95 95 95 140 140 140 F F F
113782024-02-01T17:32:37.154Z 23 460-479 104 104 104 156 156 156 F F F
113792024-02-01T17:32:37.154Z 24 480-499 102 102 102 151 151 151 F F F
113802024-02-01T17:32:37.154Z 25 500-519 105 105 105 157 157 157 F F F
113812024-02-01T17:32:37.154Z 26 520-539 102 102 102 151 151 151 F F F
113822024-02-01T17:32:37.154Z 27 540-559 105 105 105 157 157 157 F F F
113832024-02-01T17:32:37.154Z 28 560-579 90 90 90 134 134 134 F F F
113842024-02-01T17:32:37.154Z 29 580-599 86 86 86 128 128 128 F F F
113852024-02-01T17:32:37.155ZMax gen: 105, Max flush: 157
113862024-02-01T17:32:37.161Z
113872024-02-01T17:32:37.161ZBegin loop 53
113882024-02-01T17:32:37.161ZDownstairs to restart: 0
113892024-02-01T17:32:37.187Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.452646453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
113902024-02-01T17:32:37.220Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453156659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
113912024-02-01T17:32:37.220Z{"msg":"Crucible runtime is spawned
113922024-02-01T17:32:37.220ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453285014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
113932024-02-01T17:32:37.220Z{"msg":"Crucible 452087ca-cb1c-479b-b3ae-9d7f80e4b079 has session id: f2ad9100-dee3-44a2-9019-0dc03d9098a8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453381318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
113942024-02-01T17:32:37.221Z{"msg":"Upstairs opts: Upstairs UUID: 452087ca-cb1c-479b-b3ae-9d7f80e4b079, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
113952024-02-01T17:32:37.221Z{"msg":"Crucible stats registered with UUID: 452087ca-cb1c-479b-b3ae-9d7f80e4b079","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453622587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
113962024-02-01T17:32:37.221Z{"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.45369633Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
113972024-02-01T17:32:37.221Z{"msg":"452087ca-cb1c-479b-b3ae-9d7f80e4b079 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453815245Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
113982024-02-01T17:32:37.229Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.453894651Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
113992024-02-01T17:32:37.229Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454054265Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114002024-02-01T17:32:37.229Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454152101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114012024-02-01T17:32:37.230Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454302043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114022024-02-01T17:32:37.230Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454346837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114032024-02-01T17:32:37.230Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454409672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114042024-02-01T17:32:37.230Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454484399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114052024-02-01T17:32:37.231Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454528495Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114062024-02-01T17:32:37.231Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454616532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114072024-02-01T17:32:37.231Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454659131Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114082024-02-01T17:32:37.231Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454737725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114092024-02-01T17:32:37.231Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.454833316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"io task","client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114102024-02-01T17:32:37.231Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.455352122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114112024-02-01T17:32:37.232Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.455415124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114122024-02-01T17:32:37.232Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.455515281Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114132024-02-01T17:32:37.232Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.455576398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114142024-02-01T17:32:37.232Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.456164129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114152024-02-01T17:32:37.232Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.456234821Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114162024-02-01T17:32:37.233Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.456283755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114172024-02-01T17:32:37.233Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.456327779Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114182024-02-01T17:32:37.239Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457118546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114192024-02-01T17:32:37.239Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457203748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114202024-02-01T17:32:37.239Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457234481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114212024-02-01T17:32:37.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457329598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114222024-02-01T17:32:37.240Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457381837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114232024-02-01T17:32:37.240Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457407453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114242024-02-01T17:32:37.240Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.457429615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114252024-02-01T17:32:37.240Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458182837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114262024-02-01T17:32:37.241Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458265199Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114272024-02-01T17:32:37.241Z{"msg":"[0]R flush_numbers[0..12]: [143, 142, 151, 154, 157, 148, 154, 149, 147, 157, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458297714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114282024-02-01T17:32:37.241Z{"msg":"[0]R generation[0..12]: [96, 96, 102, 103, 105, 100, 103, 100, 98, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.45834154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114292024-02-01T17:32:37.242Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458368576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114302024-02-01T17:32:37.242Z{"msg":"[1]R flush_numbers[0..12]: [143, 142, 151, 154, 157, 148, 154, 149, 147, 157, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458410784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114312024-02-01T17:32:37.242Z{"msg":"[1]R generation[0..12]: [96, 96, 102, 103, 105, 100, 103, 100, 98, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458435586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114322024-02-01T17:32:37.243Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458497408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
114332024-02-01T17:32:37.243Zdownstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114342024-02-01T17:32:37.243Z{"msg":"[2]R flush_numbers[0..12]: [143, 142, 151, 154, 157, 148, 154, 149, 147, 157, 156, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458543759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114352024-02-01T17:32:37.243Z{"msg":"[2]R generation[0..12]: [96, 96, 102, 103, 105, 100, 103, 100, 98, 105, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458606774Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114362024-02-01T17:32:37.243Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458711066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114372024-02-01T17:32:37.246Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458755604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114382024-02-01T17:32:37.246Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458819394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114392024-02-01T17:32:37.246Z{"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.45886101Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114402024-02-01T17:32:37.247Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458926019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"":"downstairs","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114412024-02-01T17:32:37.247Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.458967436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114422024-02-01T17:32:37.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459025406Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"0","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114432024-02-01T17:32:37.247Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.45906852Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"1","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114442024-02-01T17:32:37.248Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459124628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"client":"2","":"downstairs","session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114452024-02-01T17:32:37.248Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459167092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114462024-02-01T17:32:37.248Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
114472024-02-01T17:32:37.255Z,"time":"2024-02-01T17:32:36.459229441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114482024-02-01T17:32:37.255Z{"msg":"452087ca-cb1c-479b-b3ae-9d7f80e4b079 is now active with session: f2ad9100-dee3-44a2-9019-0dc03d9098a8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459271194Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114492024-02-01T17:32:37.255Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459323214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553,"session_id":"f2ad9100-dee3-44a2-9019-0dc03d9098a8"}
114502024-02-01T17:32:37.255Z{"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:36.459363131Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
114512024-02-01T17:32:37.255ZLoading write count information from file "/tmp/test_repair_verify.data"
114522024-02-01T17:32:37.256ZRun Repair workload
114532024-02-01T17:32:37.256Z01/30 Write block 208 len 32768 data: 14 13 9 10 7 8 8 8
114542024-02-01T17:32:37.256Z02/30 Write block 188 len 32768 data: 9 10 11 10 7 8 9 9
114552024-02-01T17:32:37.256Z03/30 Read block 466 len 16384
114562024-02-01T17:32:37.256Z04/30 Write block 559 len 8192 data: 12 10
114572024-02-01T17:32:37.257Z05/30 Read block 276 len 24576
114582024-02-01T17:32:37.257Z06/30 Read block 240 len 16384
114592024-02-01T17:32:37.257Z07/30 Write block 354 len 36864 data: 14 12 10 11 12 12 12 13 14
114602024-02-01T17:32:37.257Z08/30 Read block 43 len 4096
114612024-02-01T17:32:37.257Z09/30 Write block 573 len 12288 data: 4 4 6
114622024-02-01T17:32:37.257Z10/30 Read block 476 len 40960
114632024-02-01T17:32:37.270Z11/30 Write block 9 len 12288 data: 6 7 4
114642024-02-01T17:32:37.271Z12/30 Read block 541 len 4096
114652024-02-01T17:32:37.271Z13/30 Read block 78 len 4096
114662024-02-01T17:32:37.271Z14/30 Flush
114672024-02-01T17:32:37.271Z15/30 Write block 334 len 16384 data: 12 12 12 9
114682024-02-01T17:32:37.271Z16/30 Write block 71 len 24576 data: 13 13 12 12 11 6
114692024-02-01T17:32:37.271Z17/30 Write block 339 len 20480 data: 8 6 6 6 5
114702024-02-01T17:32:37.272Z18/30 Flush
114712024-02-01T17:32:37.272Z19/30 Write block 534 len 8192 data: 14 13
114722024-02-01T17:32:37.272Z20/30 Flush
114732024-02-01T17:32:37.272Z21/30 Write block 8 len 32768 data: 9 7 8 5 4 4 4 5
114742024-02-01T17:32:37.272Z22/30 Read block 102 len 32768
114752024-02-01T17:32:37.273Z23/30 Read block 396 len 16384
114762024-02-01T17:32:37.273Z24/30 Write block 155 len 4096 data: 10
114772024-02-01T17:32:37.273Z25/30 Read block 313 len 8192
114782024-02-01T17:32:37.273Z26/30 Write block 96 len 28672 data: 7 7 7 7 7 7 8
114792024-02-01T17:32:37.273Z27/30 Read block 89 len 16384
114802024-02-01T17:32:37.273Z28/30 Write block 514 len 32768 data: 7 7 7 8 10 9 9 9
114812024-02-01T17:32:37.273Z29/30 Read block 545 len 4096
114822024-02-01T17:32:37.273Z30/30 Read block 365 len 40960
114832024-02-01T17:32:37.273Z----------------------------------------------------------------
114842024-02-01T17:32:37.273Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
114852024-02-01T17:32:37.274ZStates: Active Active Active
114862024-02-01T17:32:37.274ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114872024-02-01T17:32:37.275Z 1 Acked 1000 Write 8 Done Done Done false
114882024-02-01T17:32:37.275Z 2 Acked 1001 Write 8 Done Done Done false
114892024-02-01T17:32:37.275Z 3 Acked 1002 Read 4 Done Done Done false
114902024-02-01T17:32:37.275Z 4 Acked 1003 Write 2 Done Done Done false
114912024-02-01T17:32:37.275Z 5 Acked 1004 Read 6 Done Done Done false
114922024-02-01T17:32:37.275Z 6 Acked 1005 Read 4 Done Done Done false
114932024-02-01T17:32:37.275Z 7 Acked 1006 Write 9 Done Done Done false
114942024-02-01T17:32:37.275Z 8 Acked 1007 Read 1 Done Done Done false
114952024-02-01T17:32:37.275Z 9 Acked 1008 Write 3 Sent Done Done false
114962024-02-01T17:32:37.275Z 10 Acked 1009 Read 10 Sent Done Done false
114972024-02-01T17:32:37.275Z 11 Acked 1010 Write 3 Sent Done Done false
114982024-02-01T17:32:37.276Z 12 Acked 1011 Read 1 Sent Done Done false
114992024-02-01T17:32:37.276Z 13 Acked 1012 Read 1 Sent Done Done false
115002024-02-01T17:32:37.276Z 14 Acked 1013 Flush 0 Sent Done Done false
115012024-02-01T17:32:37.276Z 15 Acked 1014 Write 4 Sent Done Done false
115022024-02-01T17:32:37.276Z 16 Acked 1015 Write 6 Sent Done Done false
115032024-02-01T17:32:37.276Z 17 Acked 1016 Write 5 Sent Done Done false
115042024-02-01T17:32:37.276Z 18 Acked 1017 Flush 0 Sent Done Done false
115052024-02-01T17:32:37.276Z 19 Acked 1018 Write 2 Sent Done Done false
115062024-02-01T17:32:37.276Z 20 Acked 1019 Flush 0 Sent Done Done false
115072024-02-01T17:32:37.276Z 21 Acked 1020 Write 8 Sent Done Done false
115082024-02-01T17:32:37.277Z 22 Acked 1021 Read 8 Sent Done Done false
115092024-02-01T17:32:37.277Z 23 Acked 1022 Read 4 Sent Done Done false
115102024-02-01T17:32:37.278Z 24 Acked 1023 Write 1 Sent Done Done false
115112024-02-01T17:32:37.278Z 25 Acked 1024 Read 2 Sent Done Done false
115122024-02-01T17:32:37.278Z 26 Acked 1025 Write 7 Sent Done Done false
115132024-02-01T17:32:37.278Z 27 Acked 1026 Read 4 Sent Done Done false
115142024-02-01T17:32:37.278Z 28 Acked 1027 Write 8 Sent Done Done false
115152024-02-01T17:32:37.278Z 29 Acked 1028 Read 1 Sent Done Done false
115162024-02-01T17:32:37.278Z 30 Acked 1029 Read 10 Sent Sent Done false
115172024-02-01T17:32:37.278Z STATES DS:0 DS:1 DS:2 TOTAL
115182024-02-01T17:32:37.278Z New 0 0 0 0
115192024-02-01T17:32:37.278Z Sent 22 1 0 23
115202024-02-01T17:32:37.278Z Done 8 29 30 67
115212024-02-01T17:32:37.279Z Skipped 0 0 0 0
115222024-02-01T17:32:37.279Z Error 0 0 0 0
115232024-02-01T17:32:37.279ZLast Flush: 0 1019 1019
115242024-02-01T17:32:37.279ZDownstairs last five completed:
115252024-02-01T17:32:37.279ZUpstairs last five completed: 30 29 28 27 26
115262024-02-01T17:32:37.279Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:36.506213465Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1553}
115272024-02-01T17:32:37.279ZWrote out file "/tmp/test_repair_verify.data"
115282024-02-01T17:32:37.279Z
115292024-02-01T17:32:37.279ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
115302024-02-01T17:32:37.295ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115312024-02-01T17:32:37.295Z 0 000-019 96 106 106 143 158 158 F T T <---
115322024-02-01T17:32:37.295Z 1 020-039 96 96 96 142 142 142 F F F
115332024-02-01T17:32:37.295Z 2 040-059 102 102 102 151 151 151 F F F
115342024-02-01T17:32:37.295Z 3 060-079 103 106 106 154 159 159 F F F <---
115352024-02-01T17:32:37.295Z 4 080-099 105 105 105 157 157 157 F T T <---
115362024-02-01T17:32:37.295Z 5 100-119 100 100 100 148 148 148 F T T <---
115372024-02-01T17:32:37.295Z 6 120-139 103 103 103 154 154 154 F F F
115382024-02-01T17:32:37.296Z 7 140-159 100 100 100 149 149 149 F T T <---
115392024-02-01T17:32:37.296Z 8 160-179 98 98 98 147 147 147 F F F
115402024-02-01T17:32:37.296Z 9 180-199 105 106 106 157 158 158 T F F <---
115412024-02-01T17:32:37.296Z 10 200-219 104 106 106 156 158 158 T F F <---
115422024-02-01T17:32:37.296Z 11 220-239 104 104 104 156 156 156 F F F
115432024-02-01T17:32:37.297Z 12 240-259 101 101 101 150 150 150 F F F
115442024-02-01T17:32:37.297Z 13 260-279 104 104 104 156 156 156 F F F
115452024-02-01T17:32:37.297Z 14 280-299 103 103 103 154 154 154 F F F
115462024-02-01T17:32:37.297Z 15 300-319 101 101 101 150 150 150 F F F
115472024-02-01T17:32:37.297Z 16 320-339 104 106 106 156 159 159 F F F <---
115482024-02-01T17:32:37.297Z 17 340-359 104 106 106 156 159 159 T F F <---
115492024-02-01T17:32:37.297Z 18 360-379 104 106 106 156 158 158 T F F <---
115502024-02-01T17:32:37.298Z 19 380-399 105 105 105 157 157 157 F F F
115512024-02-01T17:32:37.298Z 20 400-419 105 105 105 157 157 157 F F F
115522024-02-01T17:32:37.298Z 21 420-439 103 103 103 154 154 154 F F F
115532024-02-01T17:32:37.298Z 22 440-459 95 95 95 140 140 140 F F F
115542024-02-01T17:32:37.298Z 23 460-479 104 104 104 156 156 156 F F F
115552024-02-01T17:32:37.298Z 24 480-499 102 102 102 151 151 151 F F F
115562024-02-01T17:32:37.299Z 25 500-519 105 105 105 157 157 157 F T T <---
115572024-02-01T17:32:37.299Z 26 520-539 102 106 106 151 160 160 F T T <---
115582024-02-01T17:32:37.299Z 27 540-559 105 106 106 157 158 158 T F F <---
115592024-02-01T17:32:37.299Z 28 560-579 90 106 106 134 158 158 T F F <---
115602024-02-01T17:32:37.299Z 29 580-599 86 86 86 128 128 128 F F F
115612024-02-01T17:32:37.299ZMax gen: 106, Max flush: 160
115622024-02-01T17:32:37.299ZError: Difference in extent metadata found!
115632024-02-01T17:32:37.299ZA Difference in extent metadata is expected here
115642024-02-01T17:32:37.300ZOn loop 53
115652024-02-01T17:32:37.300Z
115662024-02-01T17:32:37.300ZVerifying data now
115672024-02-01T17:32:43.468ZNo change after verify
115682024-02-01T17:32:43.468ZLoop: 53 Downstairs dump after verify (and repair):
115692024-02-01T17:32:43.489ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115702024-02-01T17:32:43.489Z 0 000-019 107 107 107 161 161 161 F F F
115712024-02-01T17:32:43.493Z 1 020-039 96 96 96 142 142 142 F F F
115722024-02-01T17:32:43.493Z 2 040-059 102 102 102 151 151 151 F F F
115732024-02-01T17:32:43.494Z 3 060-079 106 106 106 159 159 159 F F F
115742024-02-01T17:32:43.494Z 4 080-099 107 107 107 161 161 161 F F F
115752024-02-01T17:32:43.494Z 5 100-119 107 107 107 161 161 161 F F F
115762024-02-01T17:32:43.494Z 6 120-139 103 103 103 154 154 154 F F F
115772024-02-01T17:32:43.494Z 7 140-159 107 107 107 161 161 161 F F F
115782024-02-01T17:32:43.494Z 8 160-179 98 98 98 147 147 147 F F F
115792024-02-01T17:32:43.494Z 9 180-199 106 106 106 158 158 158 F F F
115802024-02-01T17:32:43.495Z 10 200-219 106 106 106 158 158 158 F F F
115812024-02-01T17:32:43.495Z 11 220-239 104 104 104 156 156 156 F F F
115822024-02-01T17:32:43.497Z 12 240-259 101 101 101 150 150 150 F F F
115832024-02-01T17:32:43.497Z 13 260-279 104 104 104 156 156 156 F F F
115842024-02-01T17:32:43.497Z 14 280-299 103 103 103 154 154 154 F F F
115852024-02-01T17:32:43.497Z 15 300-319 101 101 101 150 150 150 F F F
115862024-02-01T17:32:43.497Z 16 320-339 106 106 106 159 159 159 F F F
115872024-02-01T17:32:43.497Z 17 340-359 106 106 106 159 159 159 F F F
115882024-02-01T17:32:43.497Z 18 360-379 106 106 106 158 158 158 F F F
115892024-02-01T17:32:43.498Z 19 380-399 105 105 105 157 157 157 F F F
115902024-02-01T17:32:43.498Z 20 400-419 105 105 105 157 157 157 F F F
115912024-02-01T17:32:43.498Z 21 420-439 103 103 103 154 154 154 F F F
115922024-02-01T17:32:43.498Z 22 440-459 95 95 95 140 140 140 F F F
115932024-02-01T17:32:43.498Z 23 460-479 104 104 104 156 156 156 F F F
115942024-02-01T17:32:43.498Z 24 480-499 102 102 102 151 151 151 F F F
115952024-02-01T17:32:43.498Z 25 500-519 107 107 107 161 161 161 F F F
115962024-02-01T17:32:43.498Z 26 520-539 107 107 107 161 161 161 F F F
115972024-02-01T17:32:43.498Z 27 540-559 106 106 106 158 158 158 F F F
115982024-02-01T17:32:43.498Z 28 560-579 106 106 106 158 158 158 F F F
115992024-02-01T17:32:43.498Z 29 580-599 86 86 86 128 128 128 F F F
116002024-02-01T17:32:43.498ZMax gen: 107, Max flush: 161
116012024-02-01T17:32:43.498Z
116022024-02-01T17:32:43.498ZBegin loop 54
116032024-02-01T17:32:43.498ZDownstairs to restart: 1
116042024-02-01T17:32:43.524Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.798280888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
116052024-02-01T17:32:43.535Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.798784373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
116062024-02-01T17:32:43.535Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
116072024-02-01T17:32:43.536Z,"time":"2024-02-01T17:32:42.799002667Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
116082024-02-01T17:32:43.536Z{"msg":"Crucible 8578a43f-4a20-483f-97e9-bb4e0020e561 has session id: d8a68deb-db3b-448d-b525-3e19f086a1a7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.7991797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116092024-02-01T17:32:43.536Z{"msg":"Upstairs opts: Upstairs UUID: 8578a43f-4a20-483f-97e9-bb4e0020e561, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799270157Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116102024-02-01T17:32:43.536Z{"msg":"Crucible stats registered with UUID: 8578a43f-4a20-483f-97e9-bb4e0020e561","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799299573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116112024-02-01T17:32:43.537Z{"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799321472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
116122024-02-01T17:32:43.537Z{"msg":"8578a43f-4a20-483f-97e9-bb4e0020e561 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799361141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116132024-02-01T17:32:43.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799383985Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116142024-02-01T17:32:43.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799415824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116152024-02-01T17:32:43.537Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799459411Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116162024-02-01T17:32:43.542Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799484744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116172024-02-01T17:32:43.542Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799507058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116182024-02-01T17:32:43.542Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799530549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116192024-02-01T17:32:43.542Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.79959438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116202024-02-01T17:32:43.542Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799628301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116212024-02-01T17:32:43.542Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.799651706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116222024-02-01T17:32:43.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.800085748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116232024-02-01T17:32:43.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.800192212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116242024-02-01T17:32:43.543Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.800235545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"io task","client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116252024-02-01T17:32:43.543Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801383856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116262024-02-01T17:32:43.545Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801491139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116272024-02-01T17:32:43.545Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801627449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116282024-02-01T17:32:43.545Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801731418Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116292024-02-01T17:32:43.546Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801779323Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116302024-02-01T17:32:43.546Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.801835376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116312024-02-01T17:32:43.546Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.802472887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116322024-02-01T17:32:43.546Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.802557483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116332024-02-01T17:32:43.546Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.80271612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116342024-02-01T17:32:43.547Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803153423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116352024-02-01T17:32:43.547Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803225707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116362024-02-01T17:32:43.549Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.80327067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116372024-02-01T17:32:43.549Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803511512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116382024-02-01T17:32:43.550Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803570235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116392024-02-01T17:32:43.550Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803596413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116402024-02-01T17:32:43.550Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.80361711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116412024-02-01T17:32:43.550Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803658316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116422024-02-01T17:32:43.550Z{"msg":"[0]R flush_numbers[0..12]: [161, 142, 151, 159, 161, 161, 154, 161, 147, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803690692Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116432024-02-01T17:32:43.550Z{"msg":"[0]R generation[0..12]: [107, 96, 102, 106, 107, 107, 103, 107, 98, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803714983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116442024-02-01T17:32:43.551Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803758261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116452024-02-01T17:32:43.551Z{"msg":"[1]R flush_numbers[0..12]: [161, 142, 151, 159, 161, 161, 154, 161, 147, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.8038013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116462024-02-01T17:32:43.551Z{"msg":"[1]R generation[0..12]: [107, 96, 102, 106, 107, 107, 103, 107, 98, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803827078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116472024-02-01T17:32:43.551Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803851235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116482024-02-01T17:32:43.551Z{"msg":"[2]R flush_numbers[0..12]: [161, 142, 151, 159, 161, 161, 154, 161, 147, 158, 158, 156]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803892683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116492024-02-01T17:32:43.551Z{"msg":"[2]R generation[0..12]: [107, 96, 102, 106, 107, 107, 103, 107, 98, 106, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803917619Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116502024-02-01T17:32:43.551Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803941049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116512024-02-01T17:32:43.551Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.803968714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116522024-02-01T17:32:43.552Z{"msg":Wait for a query_work_queue command to finish before sending IO
116532024-02-01T17:32:43.552Z"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804042995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116542024-02-01T17:32:43.552Z{"msg":"Next flush: 162","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804088754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116552024-02-01T17:32:43.552Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804152833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"":"downstairs","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116562024-02-01T17:32:43.552Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804179802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116572024-02-01T17:32:43.552Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.80420044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"0","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116582024-02-01T17:32:43.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804249213Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"1","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116592024-02-01T17:32:43.553Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804291446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"client":"2","":"downstairs","session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116602024-02-01T17:32:43.553Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804368669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116612024-02-01T17:32:43.553Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804412087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116622024-02-01T17:32:43.553Z{"msg":"8578a43f-4a20-483f-97e9-bb4e0020e561 is now active with session: d8a68deb-db3b-448d-b525-3e19f086a1a7","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804463731Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116632024-02-01T17:32:43.569Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804494181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561,"session_id":"d8a68deb-db3b-448d-b525-3e19f086a1a7"}
116642024-02-01T17:32:43.569Z{"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:42.804566342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
116652024-02-01T17:32:43.569ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116662024-02-01T17:32:43.570ZLoading write count information from file "/tmp/test_repair_verify.data"
116672024-02-01T17:32:43.570ZRun Repair workload
116682024-02-01T17:32:43.570Z01/30 Write block 583 len 12288 data: 6 7 8
116692024-02-01T17:32:43.571Z02/30 Write block 329 len 12288 data: 7 8 11
116702024-02-01T17:32:43.571Z03/30 Read block 199 len 12288
116712024-02-01T17:32:43.571Z04/30 Read block 47 len 8192
116722024-02-01T17:32:43.571Z05/30 Write block 138 len 40960 data: 14 11 11 11 9 8 7 7 7 6
116732024-02-01T17:32:43.571Z06/30 Read block 154 len 28672
116742024-02-01T17:32:43.571Z07/30 Flush
116752024-02-01T17:32:43.571Z08/30 Write block 570 len 40960 data: 6 4 3 5 5 7 7 7 7 7
116762024-02-01T17:32:43.571Z09/30 Write block 492 len 12288 data: 13 10 8
116772024-02-01T17:32:43.572Z10/30 Read block 375 len 4096
116782024-02-01T17:32:43.572Z11/30 Read block 74 len 28672
116792024-02-01T17:32:43.572Z12/30 Read block 562 len 28672
116802024-02-01T17:32:43.572Z13/30 Write block 219 len 24576 data: 12 12 10 9 8 7
116812024-02-01T17:32:43.572Z14/30 Write block 289 len 40960 data: 10 8 11 9 10 10 10 11 10 13
116822024-02-01T17:32:43.572Z15/30 Write block 17 len 20480 data: 9 11 9 10 9
116832024-02-01T17:32:43.572Z16/30 Read block 227 len 4096
116842024-02-01T17:32:43.572Z17/30 Flush
116852024-02-01T17:32:43.572Z18/30 Write block 360 len 20480 data: 13 14 15 12 9
116862024-02-01T17:32:43.572Z19/30 Read block 400 len 12288
116872024-02-01T17:32:43.572Z20/30 Flush
116882024-02-01T17:32:43.578Z21/30 Write block 196 len 40960 data: 10 9 6 7 9 10 12 14 14 12
116892024-02-01T17:32:43.578Z22/30 Write block 65 len 24576 data: 8 10 10 11 14 11
116902024-02-01T17:32:43.578Z23/30 Write block 39 len 28672 data: 10 6 9 8 10 11 11
116912024-02-01T17:32:43.578Z24/30 Read block 113 len 36864
116922024-02-01T17:32:43.578Z25/30 Read block 477 len 24576
116932024-02-01T17:32:43.579Z26/30 Flush
116942024-02-01T17:32:43.579Z27/30 Write block 217 len 8192 data: 5 10
116952024-02-01T17:32:43.579Z28/30 Write block 169 len 20480 data: 12 13 14 13 12
116962024-02-01T17:32:43.579Z29/30 Read block 454 len 4096
116972024-02-01T17:32:43.579Z30/30 Write block 356 len 12288 data: 11 12 13
116982024-02-01T17:32:43.579Z----------------------------------------------------------------
116992024-02-01T17:32:43.579Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
117002024-02-01T17:32:43.579ZStates: Active Active Active
117012024-02-01T17:32:43.579ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
117022024-02-01T17:32:43.579Z 1 Acked 1000 Write 3 Done Sent Done false
117032024-02-01T17:32:43.579Z 2 Acked 1001 Write 3 Done Sent Done false
117042024-02-01T17:32:43.579Z 3 Acked 1002 Read 3 Done Sent Done false
117052024-02-01T17:32:43.579Z 4 Acked 1003 Read 2 Done Sent Done false
117062024-02-01T17:32:43.579Z 5 Acked 1004 Write 10 Done Sent Done false
117072024-02-01T17:32:43.580Z 6 Acked 1005 Read 7 Done Sent Done false
117082024-02-01T17:32:43.580Z 7 Acked 1006 Flush 0 Done Sent Done false
117092024-02-01T17:32:43.580Z 8 Acked 1007 Write 10 Done Sent Done false
117102024-02-01T17:32:43.580Z 9 Acked 1008 Write 3 Done Sent Done false
117112024-02-01T17:32:43.580Z 10 Acked 1009 Read 1 Done Sent Done false
117122024-02-01T17:32:43.580Z 11 Acked 1010 Read 7 Done Sent Done false
117132024-02-01T17:32:43.580Z 12 Acked 1011 Read 7 Done Sent Done false
117142024-02-01T17:32:43.580Z 13 Acked 1012 Write 6 Done Sent Done false
117152024-02-01T17:32:43.580Z 14 Acked 1013 Write 10 Done Sent Done false
117162024-02-01T17:32:43.580Z 15 Acked 1014 Write 5 Done Sent Done false
117172024-02-01T17:32:43.580Z 16 Acked 1015 Read 1 Done Sent Done false
117182024-02-01T17:32:43.580Z 17 Acked 1016 Flush 0 Done Sent Done false
117192024-02-01T17:32:43.580Z 18 Acked 1017 Write 5 Done Sent Done false
117202024-02-01T17:32:43.580Z 19 Acked 1018 Read 3 Done Sent Done false
117212024-02-01T17:32:43.580Z 20 Acked 1019 Flush 0 Done Sent Done false
117222024-02-01T17:32:43.580Z 21 Acked 1020 Write 10 Done Sent Done false
117232024-02-01T17:32:43.580Z 22 Acked 1021 Write 6 Done Sent Done false
117242024-02-01T17:32:43.580Z 23 Acked 1022 Write 7 Done Sent Done false
117252024-02-01T17:32:43.580Z 24 Acked 1023 Read 9 Done Sent Done false
117262024-02-01T17:32:43.580Z 25 Acked 1024 Read 6 Done Sent Done false
117272024-02-01T17:32:43.581Z 26 Acked 1025 Flush 0 Done Sent Done false
117282024-02-01T17:32:43.581Z 27 Acked 1026 Write 2 Done Sent Done false
117292024-02-01T17:32:43.581Z 28 Acked 1027 Write 5 Done Sent Done false
117302024-02-01T17:32:43.581Z 29 Acked 1028 Read 1 Done Sent Done false
117312024-02-01T17:32:43.581Z 30 Acked 1029 Write 3 Sent Sent Sent false
117322024-02-01T17:32:43.581Z STATES DS:0 DS:1 DS:2 TOTAL
117332024-02-01T17:32:43.581Z New 0 0 0 0
117342024-02-01T17:32:43.581Z Sent 1 30 1 32
117352024-02-01T17:32:43.582Z Done 29 0 29 58
117362024-02-01T17:32:43.582Z Skipped 0 0 0 0
117372024-02-01T17:32:43.582Z Error 0 0 0 0
117382024-02-01T17:32:43.582ZLast Flush: 1025 0 1025
117392024-02-01T17:32:43.582ZDownstairs last five completed:
117402024-02-01T17:32:43.582ZUpstairs last five completed: 30 29 28 27 26
117412024-02-01T17:32:43.583Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:42.852819737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1561}
117422024-02-01T17:32:43.586ZWrote out file "/tmp/test_repair_verify.data"
117432024-02-01T17:32:43.586Z
117442024-02-01T17:32:43.591ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117452024-02-01T17:32:43.615ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117462024-02-01T17:32:43.615Z 0 000-019 108 107 108 163 161 163 F F F <---
117472024-02-01T17:32:43.615Z 1 020-039 108 96 108 165 142 165 F F F <---
117482024-02-01T17:32:43.615Z 2 040-059 108 102 108 165 151 165 F F F <---
117492024-02-01T17:32:43.615Z 3 060-079 108 106 108 165 159 165 F F F <---
117502024-02-01T17:32:43.618Z 4 080-099 107 107 107 161 161 161 F F F
117512024-02-01T17:32:43.618Z 5 100-119 107 107 107 161 161 161 F F F
117522024-02-01T17:32:43.618Z 6 120-139 108 103 108 162 154 162 F F F <---
117532024-02-01T17:32:43.618Z 7 140-159 108 107 108 162 161 162 F F F <---
117542024-02-01T17:32:43.618Z 8 160-179 98 98 98 147 147 147 T F T <---
117552024-02-01T17:32:43.618Z 9 180-199 108 106 108 165 158 165 F F F <---
117562024-02-01T17:32:43.618Z 10 200-219 108 106 108 165 158 165 T F T <---
117572024-02-01T17:32:43.618Z 11 220-239 108 104 108 163 156 163 F F F <---
117582024-02-01T17:32:43.618Z 12 240-259 101 101 101 150 150 150 F F F
117592024-02-01T17:32:43.619Z 13 260-279 104 104 104 156 156 156 F F F
117602024-02-01T17:32:43.619Z 14 280-299 108 103 108 163 154 163 F F F <---
117612024-02-01T17:32:43.619Z 15 300-319 101 101 101 150 150 150 F F F
117622024-02-01T17:32:43.619Z 16 320-339 108 106 108 162 159 162 F F F <---
117632024-02-01T17:32:43.620Z 17 340-359 106 106 106 159 159 159 T F T <---
117642024-02-01T17:32:43.620Z 18 360-379 108 106 108 164 158 164 F F F <---
117652024-02-01T17:32:43.620Z 19 380-399 105 105 105 157 157 157 F F F
117662024-02-01T17:32:43.620Z 20 400-419 105 105 105 157 157 157 F F F
117672024-02-01T17:32:43.620Z 21 420-439 103 103 103 154 154 154 F F F
117682024-02-01T17:32:43.620Z 22 440-459 95 95 95 140 140 140 F F F
117692024-02-01T17:32:43.620Z 23 460-479 104 104 104 156 156 156 F F F
117702024-02-01T17:32:43.620Z 24 480-499 108 102 108 163 151 163 F F F <---
117712024-02-01T17:32:43.620Z 25 500-519 107 107 107 161 161 161 F F F
117722024-02-01T17:32:43.620Z 26 520-539 107 107 107 161 161 161 F F F
117732024-02-01T17:32:43.620Z 27 540-559 106 106 106 158 158 158 F F F
117742024-02-01T17:32:43.620Z 28 560-579 108 106 108 163 158 163 F F F <---
117752024-02-01T17:32:43.620Z 29 580-599 108 86 108 162 128 162 F F F <---
117762024-02-01T17:32:43.620ZMax gen: 108, Max flush: 165
117772024-02-01T17:32:43.620ZError: Difference in extent metadata found!
117782024-02-01T17:32:43.623ZA Difference in extent metadata is expected here
117792024-02-01T17:32:43.623ZOn loop 54
117802024-02-01T17:32:43.624Z
117812024-02-01T17:32:43.624ZVerifying data now
117822024-02-01T17:32:49.413ZNo change after verify
117832024-02-01T17:32:49.413ZLoop: 54 Downstairs dump after verify (and repair):
117842024-02-01T17:32:49.450ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117852024-02-01T17:32:49.450Z 0 000-019 108 108 108 163 163 163 F F F
117862024-02-01T17:32:49.450Z 1 020-039 108 108 108 165 165 165 F F F
117872024-02-01T17:32:49.450Z 2 040-059 108 108 108 165 165 165 F F F
117882024-02-01T17:32:49.450Z 3 060-079 108 108 108 165 165 165 F F F
117892024-02-01T17:32:49.450Z 4 080-099 107 107 107 161 161 161 F F F
117902024-02-01T17:32:49.451Z 5 100-119 107 107 107 161 161 161 F F F
117912024-02-01T17:32:49.451Z 6 120-139 108 108 108 162 162 162 F F F
117922024-02-01T17:32:49.451Z 7 140-159 108 108 108 162 162 162 F F F
117932024-02-01T17:32:49.451Z 8 160-179 109 109 109 166 166 166 F F F
117942024-02-01T17:32:49.451Z 9 180-199 108 108 108 165 165 165 F F F
117952024-02-01T17:32:49.451Z 10 200-219 109 109 109 166 166 166 F F F
117962024-02-01T17:32:49.452Z 11 220-239 108 108 108 163 163 163 F F F
117972024-02-01T17:32:49.452Z 12 240-259 101 101 101 150 150 150 F F F
117982024-02-01T17:32:49.452Z 13 260-279 104 104 104 156 156 156 F F F
117992024-02-01T17:32:49.452Z 14 280-299 108 108 108 163 163 163 F F F
118002024-02-01T17:32:49.452Z 15 300-319 101 101 101 150 150 150 F F F
118012024-02-01T17:32:49.452Z 16 320-339 108 108 108 162 162 162 F F F
118022024-02-01T17:32:49.454Z 17 340-359 109 109 109 166 166 166 F F F
118032024-02-01T17:32:49.454Z 18 360-379 108 108 108 164 164 164 F F F
118042024-02-01T17:32:49.457Z 19 380-399 105 105 105 157 157 157 F F F
118052024-02-01T17:32:49.458Z 20 400-419 105 105 105 157 157 157 F F F
118062024-02-01T17:32:49.459Z 21 420-439 103 103 103 154 154 154 F F F
118072024-02-01T17:32:49.460Z 22 440-459 95 95 95 140 140 140 F F F
118082024-02-01T17:32:49.461Z 23 460-479 104 104 104 156 156 156 F F F
118092024-02-01T17:32:49.461Z 24 480-499 108 108 108 163 163 163 F F F
118102024-02-01T17:32:49.462Z 25 500-519 107 107 107 161 161 161 F F F
118112024-02-01T17:32:49.462Z 26 520-539 107 107 107 161 161 161 F F F
118122024-02-01T17:32:49.462Z 27 540-559 106 106 106 158 158 158 F F F
118132024-02-01T17:32:49.462Z 28 560-579 108 108 108 163 163 163 F F F
118142024-02-01T17:32:49.462Z 29 580-599 108 108 108 162 162 162 F F F
118152024-02-01T17:32:49.463ZMax gen: 109, Max flush: 166
118162024-02-01T17:32:49.463Z
118172024-02-01T17:32:49.463ZBegin loop 55
118182024-02-01T17:32:49.463ZDownstairs to restart: 1
118192024-02-01T17:32:49.464Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.324357477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
118202024-02-01T17:32:49.464Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.324842737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
118212024-02-01T17:32:49.464Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.324913943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
118222024-02-01T17:32:49.464Z{"msg":"Crucible cc456096-940d-4bc2-9ba4-693d9abda7ed has session id: bcc30509-104b-471c-acd5-6201d380e491","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.324955166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118232024-02-01T17:32:49.465Z{"msg":"Upstairs opts: Upstairs UUID: cc456096-940d-4bc2-9ba4-693d9abda7ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325004409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118242024-02-01T17:32:49.465Z{"msg":"Crucible stats registered with UUID: cc456096-940d-4bc2-9ba4-693d9abda7ed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325068208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118252024-02-01T17:32:49.465ZCrucible runtime is spawned
118262024-02-01T17:32:49.465Z{"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325233826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
118272024-02-01T17:32:49.465Z{"msg":"cc456096-940d-4bc2-9ba4-693d9abda7ed active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325504823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118282024-02-01T17:32:49.465Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325559385Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118292024-02-01T17:32:49.465Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325640144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118302024-02-01T17:32:49.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.3257547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118312024-02-01T17:32:49.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325845176Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118322024-02-01T17:32:49.466Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.325919116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118332024-02-01T17:32:49.466Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326124214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118342024-02-01T17:32:49.466Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326211922Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118352024-02-01T17:32:49.466Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326259939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118362024-02-01T17:32:49.467Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326327909Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118372024-02-01T17:32:49.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326473073Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118382024-02-01T17:32:49.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326577476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118392024-02-01T17:32:49.467Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.326627147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"io task","client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118402024-02-01T17:32:49.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327456682Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118412024-02-01T17:32:49.467Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327505843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118422024-02-01T17:32:49.467Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327530621Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118432024-02-01T17:32:49.468Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327589447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118442024-02-01T17:32:49.468Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327614824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118452024-02-01T17:32:49.468Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.327637285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118462024-02-01T17:32:49.468Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.328562036Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118472024-02-01T17:32:49.468Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.328823402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118482024-02-01T17:32:49.468Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.328963345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118492024-02-01T17:32:49.468Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329688897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118502024-02-01T17:32:49.469Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329749467Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118512024-02-01T17:32:49.469Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329779136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118522024-02-01T17:32:49.469Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329814967Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118532024-02-01T17:32:49.469Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329845843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118542024-02-01T17:32:49.469Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.329977692Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118552024-02-01T17:32:49.469Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330048278Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118562024-02-01T17:32:49.470Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330095828Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118572024-02-01T17:32:49.470Z{"msg":"[0]R flush_numbers[0..12]: [163, 165, 165, 165, 161, 161, 162, 162, 166, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330152757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118582024-02-01T17:32:49.470Z{"msg":"[0]R generation[0..12]: [108, 108, 108, 108, 107, 107, 108, 108, 109, 108, 109, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330198738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118592024-02-01T17:32:49.470Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
118602024-02-01T17:32:49.470Z,"time":"2024-02-01T17:32:48.33026122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118612024-02-01T17:32:49.471Z{"msg":"[1]R flush_numbers[0..12]: [163, 165, 165, 165, 161, 161, 162, 162, 166, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330319464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118622024-02-01T17:32:49.471Z{"msg":"[1]R generation[0..12]: [108, 108, 108, 108, 107, 107, 108, 108, 109, 108, 109, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330385702Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118632024-02-01T17:32:49.471Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330432501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118642024-02-01T17:32:49.471Z{"msg":"[2]R flush_numbers[0..12]: [163, 165, 165, 165, 161, 161, 162, 162, 166, 165, 166, 163]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330488305Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118652024-02-01T17:32:49.471Z{"msg":"[2]R generation[0..12]: [108, 108, 108, 108, 107, 107, 108, 108, 109, 108, 109, 108]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330532336Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118662024-02-01T17:32:49.471Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330577963Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118672024-02-01T17:32:49.472Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330647937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118682024-02-01T17:32:49.472Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330740583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118692024-02-01T17:32:49.472Z{"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330783437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118702024-02-01T17:32:49.472Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330850645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"":"downstairs","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118712024-02-01T17:32:49.476Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330922487Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118722024-02-01T17:32:49.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.330992602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"0","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118732024-02-01T17:32:49.476Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.33106371Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"1","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118742024-02-01T17:32:49.476Z{"msg":"Transition from WaitQuorum to Active","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118752024-02-01T17:32:49.477Z0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331139328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"client":"2","":"downstairs","session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118762024-02-01T17:32:49.477Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331202488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118772024-02-01T17:32:49.477Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331227872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118782024-02-01T17:32:49.477Z{"msg":"cc456096-940d-4bc2-9ba4-693d9abda7ed is now active with session: bcc30509-104b-471c-acd5-6201d380e491","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331268632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118792024-02-01T17:32:49.477Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331290689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569,"session_id":"bcc30509-104b-471c-acd5-6201d380e491"}
118802024-02-01T17:32:49.477Z{"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:48.331329933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
118812024-02-01T17:32:49.478ZLoading write count information from file "/tmp/test_repair_verify.data"
118822024-02-01T17:32:49.478ZRun Repair workload
118832024-02-01T17:32:49.478Z01/30 Write block 308 len 40960 data: 7 6 5 5 8 10 8 10 10 11
118842024-02-01T17:32:49.894Z02/30 Write block 313 len 36864 data: 11 9 11 11 12 10 10 11 9
118852024-02-01T17:32:49.894Z03/30 Write block 85 len 24576 data: 5 6 5 6 5 5
118862024-02-01T17:32:49.894Z04/30 Read block 275 len 40960
118872024-02-01T17:32:49.894Z05/30 Read block 339 len 28672
118882024-02-01T17:32:49.895Z06/30 Read block 378 len 12288
118892024-02-01T17:32:49.895Z07/30 Write block 146 len 16384 data: 8 7 10 10
118902024-02-01T17:32:49.895Z08/30 Read block 25 len 4096
118912024-02-01T17:32:49.895Z09/30 Read block 502 len 24576
118922024-02-01T17:32:49.895Z10/30 Write block 435 len 40960 data: 12 11 12 10 10 10 10 9 8 9
118932024-02-01T17:32:49.895Z11/30 Write block 38 len 28672 data: 11 11 7 10 9 11 12
118942024-02-01T17:32:49.896Z12/30 Read block 346 len 40960
118952024-02-01T17:32:49.896Z13/30 Write block 341 len 36864 data: 7 7 6 6 6 7 8 9 7
118962024-02-01T17:32:49.896Z14/30 Write block 220 len 40960 data: 13 11 10 9 8 6 5 5 6 8
118972024-02-01T17:32:49.896Z15/30 Read block 538 len 28672
118982024-02-01T17:32:49.896Z16/30 Write block 247 len 28672 data: 7 7 6 7 7 6 5
118992024-02-01T17:32:49.896Z17/30 Write block 231 len 12288 data: 7 6 6
119002024-02-01T17:32:49.896Z18/30 Read block 211 len 8192
119012024-02-01T17:32:49.896Z19/30 Write block 136 len 4096 data: 13
119022024-02-01T17:32:49.896Z20/30 Write block 153 len 20480 data: 9 8 11 8 7
119032024-02-01T17:32:49.896Z21/30 Write block 222 len 24576 data: 11 10 9 7 6 6
119042024-02-01T17:32:49.896Z22/30 Read block 18 len 24576
119052024-02-01T17:32:49.896Z23/30 Write block 96 len 40960 data: 8 8 8 8 8 8 9 7 6 9
119062024-02-01T17:32:49.897Z24/30 Read block 7 len 28672
119072024-02-01T17:32:49.897Z25/30 Read block 59 len 32768
119082024-02-01T17:32:49.897Z26/30 Read block 447 len 16384
119092024-02-01T17:32:49.897Z27/30 Read block 163 len 8192
119102024-02-01T17:32:49.897Z28/30 Write block 479 len 24576 data: 7 8 9 9 9 10
119112024-02-01T17:32:49.897Z29/30 Write block 82 len 40960 data: 5 7 6 6 7 6 7 6 6 4
119122024-02-01T17:32:49.897Z30/30 Write block 500 len 24576 data: 7 7 8 6 4 4
119132024-02-01T17:32:49.897Z----------------------------------------------------------------
119142024-02-01T17:32:49.897Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
119152024-02-01T17:32:49.897ZStates: Active Active Active
119162024-02-01T17:32:49.897ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119172024-02-01T17:32:49.898Z 1 Acked 1000 Write 10 Done Done Done false
119182024-02-01T17:32:49.898Z 2 Acked 1001 Write 9 Done Sent Done false
119192024-02-01T17:32:49.898Z 3 Acked 1002 Write 6 Done Sent Done false
119202024-02-01T17:32:49.898Z 4 Acked 1003 Read 10 Done Sent Done false
119212024-02-01T17:32:49.898Z 5 Acked 1004 Read 7 Done Sent Done false
119222024-02-01T17:32:49.898Z 6 Acked 1005 Read 3 Done Sent Done false
119232024-02-01T17:32:49.898Z 7 Acked 1006 Write 4 Done Sent Done false
119242024-02-01T17:32:49.898Z 8 Acked 1007 Read 1 Done Sent Done false
119252024-02-01T17:32:49.898Z 9 Acked 1008 Read 6 Done Sent Done false
119262024-02-01T17:32:49.898Z 10 Acked 1009 Write 10 Done Sent Done false
119272024-02-01T17:32:49.899Z 11 Acked 1010 Write 7 Done Sent Done false
119282024-02-01T17:32:49.899Z 12 Acked 1011 Read 10 Done Sent Done false
119292024-02-01T17:32:49.899Z 13 Acked 1012 Write 9 Done Sent Done false
119302024-02-01T17:32:49.899Z 14 Acked 1013 Write 10 Done Sent Done false
119312024-02-01T17:32:49.899Z 15 Acked 1014 Read 7 Done Sent Done false
119322024-02-01T17:32:49.899Z 16 Acked 1015 Write 7 Done Sent Done false
119332024-02-01T17:32:49.899Z 17 Acked 1016 Write 3 Done Sent Done false
119342024-02-01T17:32:49.899Z 18 Acked 1017 Read 2 Done Sent Done false
119352024-02-01T17:32:49.899Z 19 Acked 1018 Write 1 Done Sent Done false
119362024-02-01T17:32:49.899Z 20 Acked 1019 Write 5 Done Sent Done false
119372024-02-01T17:32:49.900Z 21 Acked 1020 Write 6 Done Sent Done false
119382024-02-01T17:32:49.900Z 22 Acked 1021 Read 6 Done Sent Done false
119392024-02-01T17:32:49.900Z 23 Acked 1022 Write 10 Done Sent Done false
119402024-02-01T17:32:49.904Z 24 Acked 1023 Read 7 Done Sent Done false
119412024-02-01T17:32:49.904Z 25 Acked 1024 Read 8 Done Sent Done false
119422024-02-01T17:32:49.904Z 26 Acked 1025 Read 4 Done Sent Done false
119432024-02-01T17:32:49.905Z 27 Acked 1026 Read 2 Done Sent Done false
119442024-02-01T17:32:49.905Z 28 Acked 1027 Write 6 Done Sent Done false
119452024-02-01T17:32:49.905Z 29 Acked 1028 Write 10 Done Sent Done false
119462024-02-01T17:32:49.905Z 30 Acked 1029 Write 6 Sent Sent Sent false
119472024-02-01T17:32:49.905Z STATES DS:0 DS:1 DS:2 TOTAL
119482024-02-01T17:32:49.905Z New 0 0 0 0
119492024-02-01T17:32:49.905Z Sent 1 29 1 31
119502024-02-01T17:32:49.905Z Done 29 1 29 59
119512024-02-01T17:32:49.905Z Skipped 0 0 0 0
119522024-02-01T17:32:49.905Z Error 0 0 0 0
119532024-02-01T17:32:49.905ZLast Flush: 0 0 0
119542024-02-01T17:32:49.905ZDownstairs last five completed:
119552024-02-01T17:32:49.906ZUpstairs last five completed: 30 29 28 27 26
119562024-02-01T17:32:49.906Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:48.377184629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1569}
119572024-02-01T17:32:49.906ZWrote out file "/tmp/test_repair_verify.data"
119582024-02-01T17:32:49.906Z
119592024-02-01T17:32:49.906ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119602024-02-01T17:32:49.906ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119612024-02-01T17:32:49.906Z 0 000-019 108 108 108 163 163 163 F F F
119622024-02-01T17:32:49.906Z 1 020-039 108 108 108 165 165 165 T F T <---
119632024-02-01T17:32:49.906Z 2 040-059 108 108 108 165 165 165 T F T <---
119642024-02-01T17:32:49.909Z 3 060-079 108 108 108 165 165 165 F F F
119652024-02-01T17:32:49.909Z 4 080-099 107 107 107 161 161 161 T F T <---
119662024-02-01T17:32:49.909Z 5 100-119 107 107 107 161 161 161 T F T <---
119672024-02-01T17:32:49.910Z 6 120-139 108 108 108 162 162 162 T F T <---
119682024-02-01T17:32:49.910Z 7 140-159 108 108 108 162 162 162 T F T <---
119692024-02-01T17:32:49.910Z 8 160-179 109 109 109 166 166 166 F F F
119702024-02-01T17:32:49.910Z 9 180-199 108 108 108 165 165 165 F F F
119712024-02-01T17:32:49.910Z 10 200-219 109 109 109 166 166 166 F F F
119722024-02-01T17:32:49.910Z 11 220-239 108 108 108 163 163 163 T F T <---
119732024-02-01T17:32:49.910Z 12 240-259 101 101 101 150 150 150 T F T <---
119742024-02-01T17:32:49.910Z 13 260-279 104 104 104 156 156 156 F F F
119752024-02-01T17:32:49.910Z 14 280-299 108 108 108 163 163 163 F F F
119762024-02-01T17:32:49.910Z 15 300-319 101 101 101 150 150 150 T T T
119772024-02-01T17:32:49.910Z 16 320-339 108 108 108 162 162 162 T F T <---
119782024-02-01T17:32:49.911Z 17 340-359 109 109 109 166 166 166 T F T <---
119792024-02-01T17:32:49.911Z 18 360-379 108 108 108 164 164 164 F F F
119802024-02-01T17:32:49.911Z 19 380-399 105 105 105 157 157 157 F F F
119812024-02-01T17:32:49.911Z 20 400-419 105 105 105 157 157 157 F F F
119822024-02-01T17:32:49.911Z 21 420-439 103 103 103 154 154 154 T F T <---
119832024-02-01T17:32:49.911Z 22 440-459 95 95 95 140 140 140 T F T <---
119842024-02-01T17:32:49.925Z 23 460-479 104 104 104 156 156 156 T F T <---
119852024-02-01T17:32:49.925Z 24 480-499 108 108 108 163 163 163 T F T <---
119862024-02-01T17:32:49.926Z 25 500-519 107 107 107 161 161 161 T F T <---
119872024-02-01T17:32:49.926Z 26 520-539 107 107 107 161 161 161 F F F
119882024-02-01T17:32:49.926Z 27 540-559 106 106 106 158 158 158 F F F
119892024-02-01T17:32:49.926Z 28 560-579 108 108 108 163 163 163 F F F
119902024-02-01T17:32:49.926Z 29 580-599 108 108 108 162 162 162 F F F
119912024-02-01T17:32:49.927ZMax gen: 109, Max flush: 166
119922024-02-01T17:32:49.927ZError: Difference in extent metadata found!
119932024-02-01T17:32:49.927ZA Difference in extent metadata is expected here
119942024-02-01T17:32:49.927ZOn loop 55
119952024-02-01T17:32:49.927Z
119962024-02-01T17:32:49.927ZVerifying data now
119972024-02-01T17:32:54.525ZNo change after verify
119982024-02-01T17:32:54.525ZLoop: 55 Downstairs dump after verify (and repair):
119992024-02-01T17:32:54.554ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120002024-02-01T17:32:54.554Z 0 000-019 108 108 108 163 163 163 F F F
120012024-02-01T17:32:54.565Z 1 020-039 110 110 110 167 167 167 F F F
120022024-02-01T17:32:54.566Z 2 040-059 110 110 110 167 167 167 F F F
120032024-02-01T17:32:54.566Z 3 060-079 108 108 108 165 165 165 F F F
120042024-02-01T17:32:54.566Z 4 080-099 110 110 110 167 167 167 F F F
120052024-02-01T17:32:54.566Z 5 100-119 110 110 110 167 167 167 F F F
120062024-02-01T17:32:54.566Z 6 120-139 110 110 110 167 167 167 F F F
120072024-02-01T17:32:54.566Z 7 140-159 110 110 110 167 167 167 F F F
120082024-02-01T17:32:54.566Z 8 160-179 109 109 109 166 166 166 F F F
120092024-02-01T17:32:54.566Z 9 180-199 108 108 108 165 165 165 F F F
120102024-02-01T17:32:54.566Z 10 200-219 109 109 109 166 166 166 F F F
120112024-02-01T17:32:54.567Z 11 220-239 110 110 110 167 167 167 F F F
120122024-02-01T17:32:54.567Z 12 240-259 110 110 110 167 167 167 F F F
120132024-02-01T17:32:54.567Z 13 260-279 104 104 104 156 156 156 F F F
120142024-02-01T17:32:54.567Z 14 280-299 108 108 108 163 163 163 F F F
120152024-02-01T17:32:54.567Z 15 300-319 110 110 110 167 167 167 F F F
120162024-02-01T17:32:54.567Z 16 320-339 110 110 110 167 167 167 F F F
120172024-02-01T17:32:54.567Z 17 340-359 110 110 110 167 167 167 F F F
120182024-02-01T17:32:54.567Z 18 360-379 108 108 108 164 164 164 F F F
120192024-02-01T17:32:54.567Z 19 380-399 105 105 105 157 157 157 F F F
120202024-02-01T17:32:54.567Z 20 400-419 105 105 105 157 157 157 F F F
120212024-02-01T17:32:54.568Z 21 420-439 110 110 110 167 167 167 F F F
120222024-02-01T17:32:54.568Z 22 440-459 110 110 110 167 167 167 F F F
120232024-02-01T17:32:54.568Z 23 460-479 110 110 110 167 167 167 F F F
120242024-02-01T17:32:54.568Z 24 480-499 110 110 110 167 167 167 F F F
120252024-02-01T17:32:54.568Z 25 500-519 110 110 110 167 167 167 F F F
120262024-02-01T17:32:54.568Z 26 520-539 107 107 107 161 161 161 F F F
120272024-02-01T17:32:54.568Z 27 540-559 106 106 106 158 158 158 F F F
120282024-02-01T17:32:54.569Z 28 560-579 108 108 108 163 163 163 F F F
120292024-02-01T17:32:54.569Z 29 580-599 108 108 108 162 162 162 F F F
120302024-02-01T17:32:54.569ZMax gen: 110, Max flush: 167
120312024-02-01T17:32:54.569Z
120322024-02-01T17:32:54.569ZBegin loop 56
120332024-02-01T17:32:54.569ZDownstairs to restart: 2
120342024-02-01T17:32:54.585Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.857413283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
120352024-02-01T17:32:54.601Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.857945578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
120362024-02-01T17:32:54.601Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858010996Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
120372024-02-01T17:32:54.601Z{"msg":"Crucible 54ff5c11-9d89-40ac-97a5-287bc10fb720 has session id: 89da39e0-28b8-48ca-a7a2-6c940406c57b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.85808129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120382024-02-01T17:32:54.602Z{"msg":"Upstairs opts: Upstairs UUID: 54ff5c11-9d89-40ac-97a5-287bc10fb720, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858128652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120392024-02-01T17:32:54.602Z{"msg":"Crucible stats registered with UUID: 54ff5c11-9d89-40ac-97a5-287bc10fb720","v":0,"name":"crucible","level":30Crucible runtime is spawned
120402024-02-01T17:32:54.602Z,"time":"2024-02-01T17:32:53.858199227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120412024-02-01T17:32:54.602Z{"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858321657Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
120422024-02-01T17:32:54.602Z{"msg":"54ff5c11-9d89-40ac-97a5-287bc10fb720 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858488994Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120432024-02-01T17:32:54.602Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858545431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120442024-02-01T17:32:54.602Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858579487Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120452024-02-01T17:32:54.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858704388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120462024-02-01T17:32:54.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858791499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120472024-02-01T17:32:54.603Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858889298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120482024-02-01T17:32:54.603Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858959647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120492024-02-01T17:32:54.603Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.858991462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120502024-02-01T17:32:54.604Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.859116726Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120512024-02-01T17:32:54.604Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.859177185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120522024-02-01T17:32:54.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.859208699Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120532024-02-01T17:32:54.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.859250111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120542024-02-01T17:32:54.605Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.859423607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"io task","client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120552024-02-01T17:32:54.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860335515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120562024-02-01T17:32:54.606Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860393449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120572024-02-01T17:32:54.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860437342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120582024-02-01T17:32:54.606Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860525835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120592024-02-01T17:32:54.606Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860850656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120602024-02-01T17:32:54.611Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.860935209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120612024-02-01T17:32:54.611Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.861398155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120622024-02-01T17:32:54.613Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.861693694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120632024-02-01T17:32:54.614Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862326135Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120642024-02-01T17:32:54.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862401343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120652024-02-01T17:32:54.614Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862446414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120662024-02-01T17:32:54.614Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862564463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120672024-02-01T17:32:54.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862655383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120682024-02-01T17:32:54.614Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862755157Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120692024-02-01T17:32:54.615Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.862800291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120702024-02-01T17:32:54.615Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863304565Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120712024-02-01T17:32:54.615Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863367656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120722024-02-01T17:32:54.616Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 167, 165, 167, 167, 167, 167, 166, 165, 166, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.86344309Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120732024-02-01T17:32:54.616Z{"msg":"[0]R generation[0..12]: [108, 110, 110, 108, 110, 110, 110, 110, 109, 108, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863490049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120742024-02-01T17:32:54.616Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.86352122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120752024-02-01T17:32:54.616Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 167, 165, 167, 167, 167, 167, 166, 165, 166, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.86356778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120762024-02-01T17:32:54.617Z{"msg":"[1]R generation[0..12]: [108, 110, 110, 108, 110, 110, 110, 110, 109, 108, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863593947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120772024-02-01T17:32:54.617Z{"msg":"Wait for a query_work_queue command to finish before sending IO
120782024-02-01T17:32:54.617Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863659198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120792024-02-01T17:32:54.617Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 167, 165, 167, 167, 167, 167, 166, 165, 166, 167]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863729906Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120802024-02-01T17:32:54.618Z{"msg":"[2]R generation[0..12]: [108, 110, 110, 108, 110, 110, 110, 110, 109, 108, 109, 110]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863759731Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120812024-02-01T17:32:54.618Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863804707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120822024-02-01T17:32:54.618Z{"msg":"Max found gen is 111","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.86382914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120832024-02-01T17:32:54.619Z{"msg":"Generation requested: 112 >= found:111","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863893297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120842024-02-01T17:32:54.619Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.863960698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120852024-02-01T17:32:54.619Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864030739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"":"downstairs","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120862024-02-01T17:32:54.619Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864098267Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120872024-02-01T17:32:54.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864123708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"0","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120882024-02-01T17:32:54.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864178557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"1","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120892024-02-01T17:32:54.619Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864246071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"client":"2","":"downstairs","session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120902024-02-01T17:32:54.620Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864315557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120912024-02-01T17:32:54.620Z{"msg":"Set Downstairs and Upstairs active after reconciliation"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120922024-02-01T17:32:54.620Z,"v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864419055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120932024-02-01T17:32:54.621Z{"msg":"54ff5c11-9d89-40ac-97a5-287bc10fb720 is now active with session: 89da39e0-28b8-48ca-a7a2-6c940406c57b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864495799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120942024-02-01T17:32:54.621Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864564138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577,"session_id":"89da39e0-28b8-48ca-a7a2-6c940406c57b"}
120952024-02-01T17:32:54.621Z{"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:53.864632608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
120962024-02-01T17:32:54.621ZLoading write count information from file "/tmp/test_repair_verify.data"
120972024-02-01T17:32:54.621ZRun Repair workload
120982024-02-01T17:32:54.622Z01/30 Write block 414 len 12288 data: 9 10 13
120992024-02-01T17:32:54.669Z02/30 Read block 88 len 24576
121002024-02-01T17:32:54.669Z03/30 Read block 241 len 20480
121012024-02-01T17:32:54.670Z04/30 Read block 148 len 8192
121022024-02-01T17:32:54.670Z05/30 Write block 83 len 12288 data: 8 7 7
121032024-02-01T17:32:54.670Z06/30 Write block 170 len 40960 data: 14 15 14 13 11 10 13 11 9 9
121042024-02-01T17:32:54.670Z07/30 Read block 378 len 28672
121052024-02-01T17:32:54.670Z08/30 Flush
121062024-02-01T17:32:54.670Z09/30 Write block 157 len 32768 data: 8 9 8 9 11 10 10 10
121072024-02-01T17:32:54.671Z10/30 Read block 3 len 36864
121082024-02-01T17:32:54.671Z11/30 Read block 440 len 20480
121092024-02-01T17:32:54.671Z12/30 Write block 248 len 4096 data: 8
121102024-02-01T17:32:54.671Z13/30 Read block 182 len 36864
121112024-02-01T17:32:54.671Z14/30 Read block 356 len 20480
121122024-02-01T17:32:54.671Z15/30 Write block 390 len 4096 data: 8
121132024-02-01T17:32:54.671Z16/30 Read block 254 len 8192
121142024-02-01T17:32:54.672Z17/30 Read block 474 len 32768
121152024-02-01T17:32:54.672Z18/30 Flush
121162024-02-01T17:32:54.672Z19/30 Write block 218 len 28672 data: 11 13 14 12 12 11 10
121172024-02-01T17:32:54.672Z20/30 Read block 15 len 32768
121182024-02-01T17:32:54.672Z21/30 Read block 140 len 40960
121192024-02-01T17:32:54.674Z22/30 Read block 145 len 32768
121202024-02-01T17:32:54.674Z23/30 Read block 565 len 20480
121212024-02-01T17:32:54.674Z24/30 Read block 360 len 36864
121222024-02-01T17:32:54.674Z25/30 Read block 165 len 16384
121232024-02-01T17:32:54.711Z26/30 Read block 493 len 40960
121242024-02-01T17:32:54.712Z27/30 Read block 275 len 20480
121252024-02-01T17:32:54.712Z28/30 Read block 536 len 8192
121262024-02-01T17:32:54.712Z29/30 Write block 194 len 24576 data: 10 10 11 10 7 8
121272024-02-01T17:32:54.712Z30/30 Read block 3 len 4096
121282024-02-01T17:32:54.712Z----------------------------------------------------------------
121292024-02-01T17:32:54.712Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
121302024-02-01T17:32:54.713ZStates: Active Active Active
121312024-02-01T17:32:54.713ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121322024-02-01T17:32:54.713Z 1 Acked 1000 Write 3 Done Done Done false
121332024-02-01T17:32:54.713Z 2 Acked 1001 Read 6 Done Done Done false
121342024-02-01T17:32:54.713Z 3 Acked 1002 Read 5 Done Done Sent false
121352024-02-01T17:32:54.713Z 4 Acked 1003 Read 2 Done Done Sent false
121362024-02-01T17:32:54.713Z 5 Acked 1004 Write 3 Done Done Sent false
121372024-02-01T17:32:54.714Z 6 Acked 1005 Write 10 Done Done Sent false
121382024-02-01T17:32:54.714Z 7 Acked 1006 Read 7 Done Done Sent false
121392024-02-01T17:32:54.714Z 8 Acked 1007 Flush 0 Done Done Sent false
121402024-02-01T17:32:54.715Z 9 Acked 1008 Write 8 Done Done Sent false
121412024-02-01T17:32:54.715Z 10 Acked 1009 Read 9 Done Done Sent false
121422024-02-01T17:32:54.715Z 11 Acked 1010 Read 5 Done Done Sent false
121432024-02-01T17:32:54.715Z 12 Acked 1011 Write 1 Done Done Sent false
121442024-02-01T17:32:54.715Z 13 Acked 1012 Read 9 Done Done Sent false
121452024-02-01T17:32:54.715Z 14 Acked 1013 Read 5 Done Done Sent false
121462024-02-01T17:32:54.715Z 15 Acked 1014 Write 1 Done Done Sent false
121472024-02-01T17:32:54.716Z 16 Acked 1015 Read 2 Done Done Sent false
121482024-02-01T17:32:54.716Z 17 Acked 1016 Read 8 Done Done Sent false
121492024-02-01T17:32:54.716Z 18 Acked 1017 Flush 0 Done Done Sent false
121502024-02-01T17:32:54.716Z 19 Acked 1018 Write 7 Done Done Sent false
121512024-02-01T17:32:54.716Z 20 Acked 1019 Read 8 Done Done Sent false
121522024-02-01T17:32:54.716Z 21 Acked 1020 Read 10 Done Done Sent false
121532024-02-01T17:32:54.717Z 22 Acked 1021 Read 8 Done Done Sent false
121542024-02-01T17:32:54.717Z 23 Acked 1022 Read 5 Done Done Sent false
121552024-02-01T17:32:54.717Z 24 Acked 1023 Read 9 Done Done Sent false
121562024-02-01T17:32:54.717Z 25 Acked 1024 Read 4 Done Done Sent false
121572024-02-01T17:32:54.717Z 26 Acked 1025 Read 10 Done Done Sent false
121582024-02-01T17:32:54.718Z 27 Acked 1026 Read 5 Done Done Sent false
121592024-02-01T17:32:54.718Z 28 Acked 1027 Read 2 Done Done Sent false
121602024-02-01T17:32:54.718Z 29 Acked 1028 Write 6 Done Done Sent false
121612024-02-01T17:32:54.718Z 30 Acked 1029 Read 1 Done Done Sent false
121622024-02-01T17:32:54.718Z STATES DS:0 DS:1 DS:2 TOTAL
121632024-02-01T17:32:54.718Z New 0 0 0 0
121642024-02-01T17:32:54.718Z Sent 0 0 28 28
121652024-02-01T17:32:54.718Z Done 30 30 2 62
121662024-02-01T17:32:54.718Z Skipped 0 0 0 0
121672024-02-01T17:32:54.719Z Error 0 0 0 0
121682024-02-01T17:32:54.719ZLast Flush: 1017 1017 0
121692024-02-01T17:32:54.719ZDownstairs last five completed:
121702024-02-01T17:32:54.719ZUpstairs last five completed: 30 29 28 27 26
121712024-02-01T17:32:54.719Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:53.912343436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1577}
121722024-02-01T17:32:54.720ZWrote out file "/tmp/test_repair_verify.data"
121732024-02-01T17:32:54.720Z
121742024-02-01T17:32:54.720ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121752024-02-01T17:32:54.720ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121762024-02-01T17:32:54.720Z 0 000-019 108 108 108 163 163 163 F F F
121772024-02-01T17:32:54.720Z 1 020-039 110 110 110 167 167 167 F F F
121782024-02-01T17:32:54.721Z 2 040-059 110 110 110 167 167 167 F F F
121792024-02-01T17:32:54.721Z 3 060-079 108 108 108 165 165 165 F F F
121802024-02-01T17:32:54.721Z 4 080-099 112 112 110 168 168 167 F F F <---
121812024-02-01T17:32:54.721Z 5 100-119 110 110 110 167 167 167 F F F
121822024-02-01T17:32:54.721Z 6 120-139 110 110 110 167 167 167 F F F
121832024-02-01T17:32:54.721Z 7 140-159 112 112 110 169 169 167 F F F <---
121842024-02-01T17:32:54.721Z 8 160-179 112 112 109 169 169 166 F F F <---
121852024-02-01T17:32:54.721Z 9 180-199 108 108 108 165 165 165 T T F <---
121862024-02-01T17:32:54.724Z 10 200-219 109 109 109 166 166 166 T T F <---
121872024-02-01T17:32:54.724Z 11 220-239 110 110 110 167 167 167 T T F <---
121882024-02-01T17:32:54.724Z 12 240-259 112 112 110 169 169 167 F F F <---
121892024-02-01T17:32:54.724Z 13 260-279 104 104 104 156 156 156 F F F
121902024-02-01T17:32:54.725Z 14 280-299 108 108 108 163 163 163 F F F
121912024-02-01T17:32:54.725Z 15 300-319 110 110 110 167 167 167 F F F
121922024-02-01T17:32:54.725Z 16 320-339 110 110 110 167 167 167 F F F
121932024-02-01T17:32:54.725Z 17 340-359 110 110 110 167 167 167 F F F
121942024-02-01T17:32:54.725Z 18 360-379 108 108 108 164 164 164 F F F
121952024-02-01T17:32:54.725Z 19 380-399 112 112 105 169 169 157 F F F <---
121962024-02-01T17:32:54.725Z 20 400-419 112 112 105 168 168 157 F F T <---
121972024-02-01T17:32:54.726Z 21 420-439 110 110 110 167 167 167 F F F
121982024-02-01T17:32:54.726Z 22 440-459 110 110 110 167 167 167 F F F
121992024-02-01T17:32:54.726Z 23 460-479 110 110 110 167 167 167 F F F
122002024-02-01T17:32:54.726Z 24 480-499 110 110 110 167 167 167 F F F
122012024-02-01T17:32:54.726Z 25 500-519 110 110 110 167 167 167 F F F
122022024-02-01T17:32:54.727Z 26 520-539 107 107 107 161 161 161 F F F
122032024-02-01T17:32:54.727Z 27 540-559 106 106 106 158 158 158 F F F
122042024-02-01T17:32:54.727Z 28 560-579 108 108 108 163 163 163 F F F
122052024-02-01T17:32:54.727Z 29 580-599 108 108 108 162 162 162 F F F
122062024-02-01T17:32:54.727ZMax gen: 112, Max flush: 169
122072024-02-01T17:32:54.727ZError: Difference in extent metadata found!
122082024-02-01T17:32:54.728ZA Difference in extent metadata is expected here
122092024-02-01T17:32:54.728ZOn loop 56
122102024-02-01T17:32:54.728Z
122112024-02-01T17:32:54.728ZVerifying data now
122122024-02-01T17:32:59.514ZNo change after verify
122132024-02-01T17:32:59.514ZLoop: 56 Downstairs dump after verify (and repair):
122142024-02-01T17:32:59.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122152024-02-01T17:32:59.540Z 0 000-019 108 108 108 163 163 163 F F F
122162024-02-01T17:32:59.555Z 1 020-039 110 110 110 167 167 167 F F F
122172024-02-01T17:32:59.555Z 2 040-059 110 110 110 167 167 167 F F F
122182024-02-01T17:32:59.555Z 3 060-079 108 108 108 165 165 165 F F F
122192024-02-01T17:32:59.555Z 4 080-099 112 112 112 168 168 168 F F F
122202024-02-01T17:32:59.555Z 5 100-119 110 110 110 167 167 167 F F F
122212024-02-01T17:32:59.555Z 6 120-139 110 110 110 167 167 167 F F F
122222024-02-01T17:32:59.555Z 7 140-159 112 112 112 169 169 169 F F F
122232024-02-01T17:32:59.556Z 8 160-179 112 112 112 169 169 169 F F F
122242024-02-01T17:32:59.556Z 9 180-199 113 113 113 170 170 170 F F F
122252024-02-01T17:32:59.556Z 10 200-219 113 113 113 170 170 170 F F F
122262024-02-01T17:32:59.556Z 11 220-239 113 113 113 170 170 170 F F F
122272024-02-01T17:32:59.556Z 12 240-259 112 112 112 169 169 169 F F F
122282024-02-01T17:32:59.556Z 13 260-279 104 104 104 156 156 156 F F F
122292024-02-01T17:32:59.559Z 14 280-299 108 108 108 163 163 163 F F F
122302024-02-01T17:32:59.559Z 15 300-319 110 110 110 167 167 167 F F F
122312024-02-01T17:32:59.559Z 16 320-339 110 110 110 167 167 167 F F F
122322024-02-01T17:32:59.559Z 17 340-359 110 110 110 167 167 167 F F F
122332024-02-01T17:32:59.559Z 18 360-379 108 108 108 164 164 164 F F F
122342024-02-01T17:32:59.560Z 19 380-399 112 112 112 169 169 169 F F F
122352024-02-01T17:32:59.560Z 20 400-419 112 112 112 168 168 168 F F F
122362024-02-01T17:32:59.560Z 21 420-439 110 110 110 167 167 167 F F F
122372024-02-01T17:32:59.560Z 22 440-459 110 110 110 167 167 167 F F F
122382024-02-01T17:32:59.560Z 23 460-479 110 110 110 167 167 167 F F F
122392024-02-01T17:32:59.560Z 24 480-499 110 110 110 167 167 167 F F F
122402024-02-01T17:32:59.561Z 25 500-519 110 110 110 167 167 167 F F F
122412024-02-01T17:32:59.561Z 26 520-539 107 107 107 161 161 161 F F F
122422024-02-01T17:32:59.561Z 27 540-559 106 106 106 158 158 158 F F F
122432024-02-01T17:32:59.561Z 28 560-579 108 108 108 163 163 163 F F F
122442024-02-01T17:32:59.561Z 29 580-599 108 108 108 162 162 162 F F F
122452024-02-01T17:32:59.561ZMax gen: 113, Max flush: 170
122462024-02-01T17:32:59.561Z
122472024-02-01T17:32:59.561ZBegin loop 57
122482024-02-01T17:32:59.562ZDownstairs to restart: 2
122492024-02-01T17:33:00.011Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.840425315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
122502024-02-01T17:33:00.011Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.840964268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
122512024-02-01T17:33:00.012Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841045523Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
122522024-02-01T17:33:00.012Z{"msg":"Crucible 1bb5dd78-015d-4e89-a54a-e61816c9c8a9 has session id: 90c831d6-648e-4f9a-b285-f9017f1c5b0b","v":0,"name":"crucible","level":30Crucible runtime is spawned
122532024-02-01T17:33:00.012Z,"time":"2024-02-01T17:32:58.841101605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122542024-02-01T17:33:00.013Z{"msg":"Upstairs opts: Upstairs UUID: 1bb5dd78-015d-4e89-a54a-e61816c9c8a9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841234511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122552024-02-01T17:33:00.013Z{"msg":"Crucible stats registered with UUID: 1bb5dd78-015d-4e89-a54a-e61816c9c8a9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841262883Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122562024-02-01T17:33:00.013Z{"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841303903Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
122572024-02-01T17:33:00.013Z{"msg":"1bb5dd78-015d-4e89-a54a-e61816c9c8a9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841472585Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122582024-02-01T17:33:00.013Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841522669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122592024-02-01T17:33:00.013Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841590923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122602024-02-01T17:33:00.013Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841617192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122612024-02-01T17:33:00.013Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841685773Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122622024-02-01T17:33:00.013Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841762972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122632024-02-01T17:33:00.014Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841811058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122642024-02-01T17:33:00.014Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841849288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122652024-02-01T17:33:00.014Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841889835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122662024-02-01T17:33:00.014Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.841959925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122672024-02-01T17:33:00.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.842120115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122682024-02-01T17:33:00.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.842171026Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122692024-02-01T17:33:00.014Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.842246549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"io task","client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122702024-02-01T17:33:00.014Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843411588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122712024-02-01T17:33:00.015Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843478598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122722024-02-01T17:33:00.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843505155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122732024-02-01T17:33:00.015Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843528257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122742024-02-01T17:33:00.015Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843857257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122752024-02-01T17:33:00.015Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.843937655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122762024-02-01T17:33:00.016Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.84431661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122772024-02-01T17:33:00.025Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.844481494Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122782024-02-01T17:33:00.025Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.844642801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122792024-02-01T17:33:00.026Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.84509272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122802024-02-01T17:33:00.026Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845178923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122812024-02-01T17:33:00.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845226815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122822024-02-01T17:33:00.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845283656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122832024-02-01T17:33:00.027Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845309681Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122842024-02-01T17:33:00.027Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845331725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122852024-02-01T17:33:00.027Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845483602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122862024-02-01T17:33:00.027Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845535226Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122872024-02-01T17:33:00.027Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 167, 165, 168, 167, 167, 169, 169, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845561757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122882024-02-01T17:33:00.027Z{"msg":"[0]R generation[0..12]: [108, 110, 110, 108, 112, 110, 110, 112, 112, 113, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845591537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122892024-02-01T17:33:00.027Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845648279Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122902024-02-01T17:33:00.027Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 167, 165, 168, 167, 167, 169, 169, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845676185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122912024-02-01T17:33:00.028Z{"msg":"[1]R generation[0..12]: [108, 110, 110, 108, 112, 110, 110, 112, 112, 113, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845733766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122922024-02-01T17:33:00.028Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845762187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122932024-02-01T17:33:00.028Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 167, 165, 168, 167, 167, 169, 169, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845820568Z"Wait for a query_work_queue command to finish before sending IO
122942024-02-01T17:33:00.028Z,"hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122952024-02-01T17:33:00.028Z{"msg":"[2]R generation[0..12]: [108, 110, 110, 108, 112, 110, 110, 112, 112, 113, 113, 113]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845853178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122962024-02-01T17:33:00.029Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845911662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122972024-02-01T17:33:00.029Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.845938312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122982024-02-01T17:33:00.029Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846013401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
122992024-02-01T17:33:00.029Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846078119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123002024-02-01T17:33:00.029Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.8461043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"":"downstairs","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123012024-02-01T17:33:00.030Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846131078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123022024-02-01T17:33:00.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846183925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"0","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123032024-02-01T17:33:00.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846209662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"1","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123042024-02-01T17:33:00.030Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846267107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"client":"2","":"downstairs","session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123052024-02-01T17:33:00.031Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846293898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123062024-02-01T17:33:00.031Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846314275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123072024-02-01T17:33:00.031Z{"msg":"1bb5dd78-015d-4e89-a54a-e61816c9c8a9 is now active with session: 90c831d6-648e-4f9a-b285-f9017f1c5b0b","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846382864Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123082024-02-01T17:33:00.031Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846410448Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585,"session_id":"90c831d6-648e-4f9a-b285-f9017f1c5b0b"}
123092024-02-01T17:33:00.031Z{"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:32:58.846432317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
123102024-02-01T17:33:00.031ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123112024-02-01T17:33:00.032ZLoading write count information from file "/tmp/test_repair_verify.data"
123122024-02-01T17:33:00.033ZRun Repair workload
123132024-02-01T17:33:00.033Z01/30 Write block 457 len 8192 data: 12 12
123142024-02-01T17:33:00.033Z02/30 Flush
123152024-02-01T17:33:00.033Z03/30 Write block 352 len 4096 data: 10
123162024-02-01T17:33:00.033Z04/30 Flush
123172024-02-01T17:33:00.034Z05/30 Write block 109 len 16384 data: 8 5 6 6
123182024-02-01T17:33:00.034Z06/30 Write block 500 len 20480 data: 8 8 9 7 5
123192024-02-01T17:33:00.034Z07/30 Read block 20 len 20480
123202024-02-01T17:33:00.034Z08/30 Write block 259 len 36864 data: 9 7 6 5 7 7 8 9 10
123212024-02-01T17:33:00.034Z09/30 Flush
123222024-02-01T17:33:00.035Z10/30 Write block 541 len 16384 data: 14 15 12 14
123232024-02-01T17:33:00.035Z11/30 Flush
123242024-02-01T17:33:00.035Z12/30 Write block 236 len 20480 data: 6 6 7 8 7
123252024-02-01T17:33:00.035Z13/30 Write block 191 len 24576 data: 11 8 9 11 11 12
123262024-02-01T17:33:00.035Z14/30 Write block 289 len 4096 data: 11
123272024-02-01T17:33:00.036Z15/30 Write block 57 len 4096 data: 8
123282024-02-01T17:33:00.036Z16/30 Read block 59 len 24576
123292024-02-01T17:33:00.036Z17/30 Flush
123302024-02-01T17:33:00.036Z18/30 Write block 343 len 36864 data: 7 7 7 8 9 10 8 6 7
123312024-02-01T17:33:00.036Z19/30 Read block 69 len 24576
123322024-02-01T17:33:00.036Z20/30 Write block 384 len 24576 data: 8 7 5 5 6 7
123332024-02-01T17:33:00.036Z21/30 Flush
123342024-02-01T17:33:00.036Z22/30 Write block 385 len 12288 data: 8 6 6
123352024-02-01T17:33:00.036Z23/30 Write block 175 len 40960 data: 11 14 12 10 10 8 9 12 13 12
123362024-02-01T17:33:00.036Z24/30 Write block 258 len 12288 data: 10 10 8
123372024-02-01T17:33:00.037Z25/30 Read block 36 len 20480
123382024-02-01T17:33:00.037Z26/30 Read block 177 len 24576
123392024-02-01T17:33:00.037Z27/30 Read block 524 len 4096
123402024-02-01T17:33:00.037Z28/30 Write block 60 len 20480 data: 8 7 7 8 8
123412024-02-01T17:33:00.037Z29/30 Write block 265 len 20480 data: 9 10 11 10 9
123422024-02-01T17:33:00.037Z30/30 Read block 194 len 32768
123432024-02-01T17:33:00.037Z----------------------------------------------------------------
123442024-02-01T17:33:00.037Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
123452024-02-01T17:33:00.038ZStates: Active Active Active
123462024-02-01T17:33:00.038ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123472024-02-01T17:33:00.038Z 1 Acked 1000 Write 2 Done Done Sent false
123482024-02-01T17:33:00.038Z 2 Acked 1001 Flush 0 Done Done Sent false
123492024-02-01T17:33:00.038Z 3 Acked 1002 Write 1 Done Done Sent false
123502024-02-01T17:33:00.038Z 4 Acked 1003 Flush 0 Done Done Sent false
123512024-02-01T17:33:00.038Z 5 Acked 1004 Write 4 Done Done Sent false
123522024-02-01T17:33:00.038Z 6 Acked 1005 Write 5 Done Done Sent false
123532024-02-01T17:33:00.038Z 7 Acked 1006 Read 5 Done Done Sent false
123542024-02-01T17:33:00.038Z 8 Acked 1007 Write 9 Done Done Sent false
123552024-02-01T17:33:00.038Z 9 Acked 1008 Flush 0 Done Done Sent false
123562024-02-01T17:33:00.039Z 10 Acked 1009 Write 4 Done Done Sent false
123572024-02-01T17:33:00.039Z 11 Acked 1010 Flush 0 Done Done Sent false
123582024-02-01T17:33:00.039Z 12 Acked 1011 Write 5 Done Done Sent false
123592024-02-01T17:33:00.039Z 13 Acked 1012 Write 6 Done Done Sent false
123602024-02-01T17:33:00.039Z 14 Acked 1013 Write 1 Done Done Sent false
123612024-02-01T17:33:00.039Z 15 Acked 1014 Write 1 Done Done Sent false
123622024-02-01T17:33:00.039Z 16 Acked 1015 Read 6 Done Done Sent false
123632024-02-01T17:33:00.040Z 17 Acked 1016 Flush 0 Done Done Sent false
123642024-02-01T17:33:00.040Z 18 Acked 1017 Write 9 Done Done Sent false
123652024-02-01T17:33:00.040Z 19 Acked 1018 Read 6 Done Done Sent false
123662024-02-01T17:33:00.040Z 20 Acked 1019 Write 6 Done Done Sent false
123672024-02-01T17:33:00.040Z 21 Acked 1020 Flush 0 Done Done Sent false
123682024-02-01T17:33:00.040Z 22 Acked 1021 Write 3 Done Done Sent false
123692024-02-01T17:33:00.040Z 23 Acked 1022 Write 10 Done Done Sent false
123702024-02-01T17:33:00.040Z 24 Acked 1023 Write 3 Done Done Sent false
123712024-02-01T17:33:00.040Z 25 Acked 1024 Read 5 Done Done Sent false
123722024-02-01T17:33:00.040Z 26 Acked 1025 Read 6 Done Done Sent false
123732024-02-01T17:33:00.040Z 27 Acked 1026 Read 1 Done Done Sent false
123742024-02-01T17:33:00.041Z 28 Acked 1027 Write 5 Done Done Sent false
123752024-02-01T17:33:00.041Z 29 Acked 1028 Write 5 Done Done Sent false
123762024-02-01T17:33:00.041Z 30 Acked 1029 Read 8 Done Sent Sent false
123772024-02-01T17:33:00.050Z STATES DS:0 DS:1 DS:2 TOTAL
123782024-02-01T17:33:00.051Z New 0 0 0 0
123792024-02-01T17:33:00.051Z Sent 0 1 30 31
123802024-02-01T17:33:00.051Z Done 30 29 0 59
123812024-02-01T17:33:00.051Z Skipped 0 0 0 0
123822024-02-01T17:33:00.051Z Error 0 0 0 0
123832024-02-01T17:33:00.051ZLast Flush: 1020 1020 0
123842024-02-01T17:33:00.051ZDownstairs last five completed:
123852024-02-01T17:33:00.051ZUpstairs last five completed: 30 29 28 27 26
123862024-02-01T17:33:00.051Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:32:58.894255272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1585}
123872024-02-01T17:33:00.051ZWrote out file "/tmp/test_repair_verify.data"
123882024-02-01T17:33:00.051Z
123892024-02-01T17:33:00.051ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
123902024-02-01T17:33:00.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123912024-02-01T17:33:00.051Z 0 000-019 108 108 108 163 163 163 F F F
123922024-02-01T17:33:00.051Z 1 020-039 110 110 110 167 167 167 F F F
123932024-02-01T17:33:00.051Z 2 040-059 114 114 110 175 175 167 F F F <---
123942024-02-01T17:33:00.051Z 3 060-079 108 108 108 165 165 165 T T F <---
123952024-02-01T17:33:00.051Z 4 080-099 112 112 112 168 168 168 F F F
123962024-02-01T17:33:00.051Z 5 100-119 114 114 110 173 173 167 F F F <---
123972024-02-01T17:33:00.051Z 6 120-139 110 110 110 167 167 167 F F F
123982024-02-01T17:33:00.052Z 7 140-159 112 112 112 169 169 169 F F F
123992024-02-01T17:33:00.052Z 8 160-179 112 112 112 169 169 169 T T F <---
124002024-02-01T17:33:00.052Z 9 180-199 114 114 113 175 175 170 T T F <---
124012024-02-01T17:33:00.052Z 10 200-219 113 113 113 170 170 170 F F F
124022024-02-01T17:33:00.052Z 11 220-239 114 114 113 175 175 170 F F F <---
124032024-02-01T17:33:00.052Z 12 240-259 114 114 112 175 175 169 T T F <---
124042024-02-01T17:33:00.052Z 13 260-279 114 114 104 173 173 156 T T F <---
124052024-02-01T17:33:00.052Z 14 280-299 114 114 108 175 175 163 F F F <---
124062024-02-01T17:33:00.052Z 15 300-319 110 110 110 167 167 167 F F F
124072024-02-01T17:33:00.053Z 16 320-339 110 110 110 167 167 167 F F F
124082024-02-01T17:33:00.053Z 17 340-359 114 114 110 176 176 167 F F F <---
124092024-02-01T17:33:00.053Z 18 360-379 108 108 108 164 164 164 F F F
124102024-02-01T17:33:00.053Z 19 380-399 114 114 112 176 176 169 T T F <---
124112024-02-01T17:33:00.054Z 20 400-419 112 112 112 168 168 168 F F F
124122024-02-01T17:33:00.054Z 21 420-439 110 110 110 167 167 167 F F F
124132024-02-01T17:33:00.054Z 22 440-459 114 114 110 171 171 167 F F F <---
124142024-02-01T17:33:00.054Z 23 460-479 110 110 110 167 167 167 F F F
124152024-02-01T17:33:00.054Z 24 480-499 110 110 110 167 167 167 F F F
124162024-02-01T17:33:00.054Z 25 500-519 114 114 110 173 173 167 F F F <---
124172024-02-01T17:33:00.054Z 26 520-539 107 107 107 161 161 161 F F F
124182024-02-01T17:33:00.054Z 27 540-559 114 114 106 174 174 158 F F F <---
124192024-02-01T17:33:00.054Z 28 560-579 108 108 108 163 163 163 F F F
124202024-02-01T17:33:00.054Z 29 580-599 108 108 108 162 162 162 F F F
124212024-02-01T17:33:00.054ZMax gen: 114, Max flush: 176
124222024-02-01T17:33:00.054ZError: Difference in extent metadata found!
124232024-02-01T17:33:00.054ZA Difference in extent metadata is expected here
124242024-02-01T17:33:00.054ZOn loop 57
124252024-02-01T17:33:00.054Z
124262024-02-01T17:33:00.055ZVerifying data now
124272024-02-01T17:33:04.856ZNo change after verify
124282024-02-01T17:33:04.856ZLoop: 57 Downstairs dump after verify (and repair):
124292024-02-01T17:33:04.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124302024-02-01T17:33:04.873Z 0 000-019 108 108 108 163 163 163 F F F
124312024-02-01T17:33:04.935Z 1 020-039 110 110 110 167 167 167 F F F
124322024-02-01T17:33:04.935Z 2 040-059 114 114 114 175 175 175 F F F
124332024-02-01T17:33:04.935Z 3 060-079 115 115 115 177 177 177 F F F
124342024-02-01T17:33:04.935Z 4 080-099 112 112 112 168 168 168 F F F
124352024-02-01T17:33:04.935Z 5 100-119 114 114 114 173 173 173 F F F
124362024-02-01T17:33:04.935Z 6 120-139 110 110 110 167 167 167 F F F
124372024-02-01T17:33:04.935Z 7 140-159 112 112 112 169 169 169 F F F
124382024-02-01T17:33:04.936Z 8 160-179 115 115 115 177 177 177 F F F
124392024-02-01T17:33:04.936Z 9 180-199 115 115 115 177 177 177 F F F
124402024-02-01T17:33:04.936Z 10 200-219 113 113 113 170 170 170 F F F
124412024-02-01T17:33:04.936Z 11 220-239 114 114 114 175 175 175 F F F
124422024-02-01T17:33:04.936Z 12 240-259 115 115 115 177 177 177 F F F
124432024-02-01T17:33:04.936Z 13 260-279 115 115 115 177 177 177 F F F
124442024-02-01T17:33:04.936Z 14 280-299 114 114 114 175 175 175 F F F
124452024-02-01T17:33:04.936Z 15 300-319 110 110 110 167 167 167 F F F
124462024-02-01T17:33:04.941Z 16 320-339 110 110 110 167 167 167 F F F
124472024-02-01T17:33:04.941Z 17 340-359 114 114 114 176 176 176 F F F
124482024-02-01T17:33:04.941Z 18 360-379 108 108 108 164 164 164 F F F
124492024-02-01T17:33:04.942Z 19 380-399 115 115 115 177 177 177 F F F
124502024-02-01T17:33:04.942Z 20 400-419 112 112 112 168 168 168 F F F
124512024-02-01T17:33:04.942Z 21 420-439 110 110 110 167 167 167 F F F
124522024-02-01T17:33:04.942Z 22 440-459 114 114 114 171 171 171 F F F
124532024-02-01T17:33:04.942Z 23 460-479 110 110 110 167 167 167 F F F
124542024-02-01T17:33:04.942Z 24 480-499 110 110 110 167 167 167 F F F
124552024-02-01T17:33:04.942Z 25 500-519 114 114 114 173 173 173 F F F
124562024-02-01T17:33:04.943Z 26 520-539 107 107 107 161 161 161 F F F
124572024-02-01T17:33:04.943Z 27 540-559 114 114 114 174 174 174 F F F
124582024-02-01T17:33:04.943Z 28 560-579 108 108 108 163 163 163 F F F
124592024-02-01T17:33:04.943Z 29 580-599 108 108 108 162 162 162 F F F
124602024-02-01T17:33:04.943ZMax gen: 115, Max flush: 177
124612024-02-01T17:33:04.943Z
124622024-02-01T17:33:04.943ZBegin loop 58
124632024-02-01T17:33:04.943ZDownstairs to restart: 2
124642024-02-01T17:33:04.981Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.177715584Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}
124652024-02-01T17:33:04.981Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178259343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}
124662024-02-01T17:33:04.981Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178300789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}
124672024-02-01T17:33:04.982Z{"msg":"Crucible 8abc3914-dec5-4c15-a464-91625dc891ca has session id: 77a7adaa-a087-4e36-a4e1-328e4fc3a72e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178369064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124682024-02-01T17:33:04.982Z{"msg":"Upstairs opts: Upstairs UUID: 8abc3914-dec5-4c15-a464-91625dc891ca, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178424075Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124692024-02-01T17:33:04.982Z{"msg":"Crucible stats registered with UUID: 8abc3914-dec5-4c15-a464-91625dc891ca","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.17849429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"Crucible runtime is spawned
124702024-02-01T17:33:04.983Z}
124712024-02-01T17:33:04.983Z{"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178798862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}
124722024-02-01T17:33:04.983Z{"msg":"8abc3914-dec5-4c15-a464-91625dc891ca active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178882186Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124732024-02-01T17:33:04.984Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.178956052Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124742024-02-01T17:33:04.984Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179129947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124752024-02-01T17:33:04.984Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179186538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124762024-02-01T17:33:04.984Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179343424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124772024-02-01T17:33:04.984Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179693041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124782024-02-01T17:33:04.984Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179752345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124792024-02-01T17:33:04.984Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179822738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124802024-02-01T17:33:04.984Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179869718Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124812024-02-01T17:33:04.985Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179914377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124822024-02-01T17:33:04.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.179984894Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124832024-02-01T17:33:04.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.180032256Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124842024-02-01T17:33:04.985Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.180094282Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"io task","client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124852024-02-01T17:33:04.985Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.180924802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124862024-02-01T17:33:04.985Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.181057928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124872024-02-01T17:33:05.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.181145941Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124882024-02-01T17:33:05.040Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.18117853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124892024-02-01T17:33:05.040Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.181205308Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124902024-02-01T17:33:05.040Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.181227711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124912024-02-01T17:33:05.040Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.18201518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124922024-02-01T17:33:05.040Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.182087755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124932024-02-01T17:33:05.040Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.18244204Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124942024-02-01T17:33:05.041Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.182923311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124952024-02-01T17:33:05.041Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.182984843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124962024-02-01T17:33:05.041Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183012336Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124972024-02-01T17:33:05.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183072827Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124982024-02-01T17:33:05.042Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183105516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
124992024-02-01T17:33:05.042Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183141322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125002024-02-01T17:33:05.042Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183255933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125012024-02-01T17:33:05.042Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183319658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125022024-02-01T17:33:05.042Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 167, 169, 177, 177, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183416593Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125032024-02-01T17:33:05.043Z{"msg":"[0]R generation[0..12]: [108, 110, 114, 115, 112, 114, 110, 112, 115, 115, 113, 114]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
125042024-02-01T17:33:05.043Z,"time":"2024-02-01T17:33:04.183510967Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125052024-02-01T17:33:05.043Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183609895Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125062024-02-01T17:33:05.043Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 167, 169, 177, 177, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183635688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125072024-02-01T17:33:05.043Z{"msg":"[1]R generation[0..12]: [108, 110, 114, 115, 112, 114, 110, 112, 115, 115, 113, 114]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183660777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125082024-02-01T17:33:05.043Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183707066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125092024-02-01T17:33:05.044Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 167, 169, 177, 177, 170, 175]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183733605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125102024-02-01T17:33:05.044Z{"msg":"[2]R generation[0..12]: [108, 110, 114, 115, 112, 114, 110, 112, 115, 115, 113, 114]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183757004Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125112024-02-01T17:33:05.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183788103Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125122024-02-01T17:33:05.044Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183840328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125132024-02-01T17:33:05.044Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183865073Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125142024-02-01T17:33:05.044Z{"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183886934Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125152024-02-01T17:33:05.044Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.183939465Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"":"downstairs","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125162024-02-01T17:33:05.044Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184012356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125172024-02-01T17:33:05.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184038197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"0","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125182024-02-01T17:33:05.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.18409636Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"1","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125192024-02-01T17:33:05.045Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184122877Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"client":"2","":"downstairs","session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125202024-02-01T17:33:05.045Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184145334Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125212024-02-01T17:33:05.045Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184193882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125222024-02-01T17:33:05.045Z{"msg":"8abc3914-dec5-4c15-a464-91625dc891ca is now active with session: 77a7adaa-a087-4e36-a4e1-328e4fc3a72e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184218106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125232024-02-01T17:33:05.045Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.18423857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593,"session_id":"77a7adaa-a087-4e36-a4e1-328e4fc3a72e"}
125242024-02-01T17:33:05.045Z{"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:04.184283962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125252024-02-01T17:33:05.045Z
125262024-02-01T17:33:05.045ZLoading write count information from file "/tmp/test_repair_verify.data"
125272024-02-01T17:33:05.046ZRun Repair workload
125282024-02-01T17:33:05.046Z01/30 Flush
125292024-02-01T17:33:05.046Z02/30 Write block 223 len 16384 data: 12 11 8 7
125302024-02-01T17:33:05.046Z03/30 Read block 396 len 24576
125312024-02-01T17:33:05.046Z04/30 Read block 292 len 20480
125322024-02-01T17:33:05.046Z05/30 Read block 274 len 12288
125332024-02-01T17:33:05.046Z06/30 Read block 226 len 40960
125342024-02-01T17:33:05.046Z07/30 Write block 444 len 12288 data: 10 11 9
125352024-02-01T17:33:05.047Z08/30 Write block 133 len 24576 data: 8 11 12 14 14 15
125362024-02-01T17:33:05.047Z09/30 Write block 236 len 40960 data: 7 7 8 9 8 6 8 7 7 7
125372024-02-01T17:33:05.047Z10/30 Read block 435 len 4096
125382024-02-01T17:33:05.047Z11/30 Read block 505 len 36864
125392024-02-01T17:33:05.047Z12/30 Read block 522 len 16384
125402024-02-01T17:33:05.047Z13/30 Write block 404 len 28672 data: 9 7 8 9 11 8 5
125412024-02-01T17:33:05.047Z14/30 Write block 147 len 28672 data: 8 11 11 7 7 10 10
125422024-02-01T17:33:05.047Z15/30 Read block 237 len 16384
125432024-02-01T17:33:05.047Z16/30 Read block 571 len 12288
125442024-02-01T17:33:05.047Z17/30 Read block 544 len 28672
125452024-02-01T17:33:05.047Z18/30 Flush
125462024-02-01T17:33:05.047Z19/30 Write block 125 len 4096 data: 7
125472024-02-01T17:33:05.048Z20/30 Read block 586 len 4096
125482024-02-01T17:33:05.048Z21/30 Write block 209 len 28672 data: 14 10 11 8 9 9 9
125492024-02-01T17:33:05.048Z22/30 Read block 90 len 36864
125502024-02-01T17:33:05.048Z23/30 Write block 236 len 8192 data: 8 8
125512024-02-01T17:33:05.048Z24/30 Write block 575 len 28672 data: 8 8 8 8 8 7 7
125522024-02-01T17:33:05.049Z25/30 Write block 486 len 12288 data: 9 10 10
125532024-02-01T17:33:05.049Z26/30 Write block 540 len 4096 data: 10
125542024-02-01T17:33:05.049Z27/30 Read block 258 len 4096
125552024-02-01T17:33:05.049Z28/30 Write block 397 len 32768 data: 7 6 4 5 6 7 10 10
125562024-02-01T17:33:05.049Z29/30 Read block 535 len 8192
125572024-02-01T17:33:05.049Z30/30 Read block 527 len 36864
125582024-02-01T17:33:05.049Z----------------------------------------------------------------
125592024-02-01T17:33:05.049Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
125602024-02-01T17:33:05.049ZStates: Active Active Active
125612024-02-01T17:33:05.049ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
125622024-02-01T17:33:05.049Z 1 Acked 1000 Flush 0 Done Done Sent false
125632024-02-01T17:33:05.049Z 2 Acked 1001 Write 4 Done Done Sent false
125642024-02-01T17:33:05.060Z 3 Acked 1002 Read 6 Done Done Sent false
125652024-02-01T17:33:05.060Z 4 Acked 1003 Read 5 Done Done Sent false
125662024-02-01T17:33:05.060Z 5 Acked 1004 Read 3 Done Done Sent false
125672024-02-01T17:33:05.060Z 6 Acked 1005 Read 10 Done Done Sent false
125682024-02-01T17:33:05.060Z 7 Acked 1006 Write 3 Done Done Sent false
125692024-02-01T17:33:05.060Z 8 Acked 1007 Write 6 Done Done Sent false
125702024-02-01T17:33:05.060Z 9 Acked 1008 Write 10 Done Done Sent false
125712024-02-01T17:33:05.060Z 10 Acked 1009 Read 1 Done Done Sent false
125722024-02-01T17:33:05.060Z 11 Acked 1010 Read 9 Done Done Sent false
125732024-02-01T17:33:05.060Z 12 Acked 1011 Read 4 Done Done Sent false
125742024-02-01T17:33:05.060Z 13 Acked 1012 Write 7 Done Done Sent false
125752024-02-01T17:33:05.061Z 14 Acked 1013 Write 7 Done Done Sent false
125762024-02-01T17:33:05.061Z 15 Acked 1014 Read 4 Done Done Sent false
125772024-02-01T17:33:05.061Z 16 Acked 1015 Read 3 Done Done Sent false
125782024-02-01T17:33:05.062Z 17 Acked 1016 Read 7 Done Done Sent false
125792024-02-01T17:33:05.062Z 18 Acked 1017 Flush 0 Done Done Sent false
125802024-02-01T17:33:05.063Z 19 Acked 1018 Write 1 Done Done Sent false
125812024-02-01T17:33:05.063Z 20 Acked 1019 Read 1 Done Done Sent false
125822024-02-01T17:33:05.063Z 21 Acked 1020 Write 7 Done Done Sent false
125832024-02-01T17:33:05.063Z 22 Acked 1021 Read 9 Done Done Sent false
125842024-02-01T17:33:05.063Z 23 Acked 1022 Write 2 Done Done Sent false
125852024-02-01T17:33:05.063Z 24 Acked 1023 Write 7 Done Done Sent false
125862024-02-01T17:33:05.063Z 25 Acked 1024 Write 3 Done Done Sent false
125872024-02-01T17:33:05.063Z 26 Acked 1025 Write 1 Done Done Sent false
125882024-02-01T17:33:05.063Z 27 Acked 1026 Read 1 Done Done Sent false
125892024-02-01T17:33:05.063Z 28 Acked 1027 Write 8 Done Done Sent false
125902024-02-01T17:33:05.063Z 29 Acked 1028 Read 2 Done Done Sent false
125912024-02-01T17:33:05.063Z 30 Acked 1029 Read 9 Done Done Sent false
125922024-02-01T17:33:05.063Z STATES DS:0 DS:1 DS:2 TOTAL
125932024-02-01T17:33:05.064Z New 0 0 0 0
125942024-02-01T17:33:05.064Z Sent 0 0 30 30
125952024-02-01T17:33:05.064Z Done 30 30 0 60
125962024-02-01T17:33:05.064Z Skipped 0 0 0 0
125972024-02-01T17:33:05.064Z Error 0 0 0 0
125982024-02-01T17:33:05.064ZLast Flush: 1017 1017 0
125992024-02-01T17:33:05.064ZDownstairs last five completed:
126002024-02-01T17:33:05.064ZUpstairs last five completed: 30 29 28 27 26
126012024-02-01T17:33:05.064Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:04.227789383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1593}
126022024-02-01T17:33:05.065ZWrote out file "/tmp/test_repair_verify.data"
126032024-02-01T17:33:05.065Z
126042024-02-01T17:33:05.065ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126052024-02-01T17:33:05.065ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126062024-02-01T17:33:05.065Z 0 000-019 108 108 108 163 163 163 F F F
126072024-02-01T17:33:05.065Z 1 020-039 110 110 110 167 167 167 F F F
126082024-02-01T17:33:05.065Z 2 040-059 114 114 114 175 175 175 F F F
126092024-02-01T17:33:05.066Z 3 060-079 115 115 115 177 177 177 F F F
126102024-02-01T17:33:05.066Z 4 080-099 112 112 112 168 168 168 F F F
126112024-02-01T17:33:05.066Z 5 100-119 114 114 114 173 173 173 F F F
126122024-02-01T17:33:05.066Z 6 120-139 116 116 110 179 179 167 T T F <---
126132024-02-01T17:33:05.066Z 7 140-159 116 116 112 179 179 169 F F F <---
126142024-02-01T17:33:05.066Z 8 160-179 115 115 115 177 177 177 F F F
126152024-02-01T17:33:05.066Z 9 180-199 115 115 115 177 177 177 F F F
126162024-02-01T17:33:05.066Z 10 200-219 113 113 113 170 170 170 T T F <---
126172024-02-01T17:33:05.066Z 11 220-239 116 116 114 179 179 175 T T F <---
126182024-02-01T17:33:05.066Z 12 240-259 116 116 115 179 179 177 F F F <---
126192024-02-01T17:33:05.066Z 13 260-279 115 115 115 177 177 177 F F F
126202024-02-01T17:33:05.066Z 14 280-299 114 114 114 175 175 175 F F F
126212024-02-01T17:33:05.066Z 15 300-319 110 110 110 167 167 167 F F F
126222024-02-01T17:33:05.067Z 16 320-339 110 110 110 167 167 167 F F F
126232024-02-01T17:33:05.067Z 17 340-359 114 114 114 176 176 176 F F F
126242024-02-01T17:33:05.067Z 18 360-379 108 108 108 164 164 164 F F F
126252024-02-01T17:33:05.067Z 19 380-399 115 115 115 177 177 177 T T F <---
126262024-02-01T17:33:05.067Z 20 400-419 116 116 112 179 179 168 T T F <---
126272024-02-01T17:33:05.067Z 21 420-439 110 110 110 167 167 167 F F F
126282024-02-01T17:33:05.068Z 22 440-459 116 116 114 179 179 171 F F F <---
126292024-02-01T17:33:05.068Z 23 460-479 110 110 110 167 167 167 F F F
126302024-02-01T17:33:05.068Z 24 480-499 110 110 110 167 167 167 T T F <---
126312024-02-01T17:33:05.068Z 25 500-519 114 114 114 173 173 173 F F F
126322024-02-01T17:33:05.068Z 26 520-539 107 107 107 161 161 161 F F F
126332024-02-01T17:33:05.068Z 27 540-559 114 114 114 174 174 174 T T F <---
126342024-02-01T17:33:05.068Z 28 560-579 108 108 108 163 163 163 T T F <---
126352024-02-01T17:33:05.068Z 29 580-599 108 108 108 162 162 162 T T F <---
126362024-02-01T17:33:05.068ZMax gen: 116, Max flush: 179
126372024-02-01T17:33:05.068ZError: Difference in extent metadata found!
126382024-02-01T17:33:05.068ZA Difference in extent metadata is expected here
126392024-02-01T17:33:05.068ZOn loop 58
126402024-02-01T17:33:05.068Z
126412024-02-01T17:33:05.069ZVerifying data now
126422024-02-01T17:33:06.054ZNo change after verify
126432024-02-01T17:33:06.054ZLoop: 58 Downstairs dump after verify (and repair):
126442024-02-01T17:33:06.087ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126452024-02-01T17:33:06.088Z 0 000-019 108 108 108 163 163 163 F F F
126462024-02-01T17:33:06.088Z 1 020-039 110 110 110 167 167 167 F F F
126472024-02-01T17:33:06.088Z 2 040-059 114 114 114 175 175 175 F F F
126482024-02-01T17:33:06.089Z 3 060-079 115 115 115 177 177 177 F F F
126492024-02-01T17:33:06.089Z 4 080-099 112 112 112 168 168 168 F F F
126502024-02-01T17:33:06.112Z 5 100-119 114 114 114 173 173 173 F F F
126512024-02-01T17:33:06.112Z 6 120-139 117 117 117 180 180 180 F F F
126522024-02-01T17:33:06.112Z 7 140-159 116 116 116 179 179 179 F F F
126532024-02-01T17:33:06.112Z 8 160-179 115 115 115 177 177 177 F F F
126542024-02-01T17:33:06.112Z 9 180-199 115 115 115 177 177 177 F F F
126552024-02-01T17:33:06.112Z 10 200-219 117 117 117 180 180 180 F F F
126562024-02-01T17:33:06.112Z 11 220-239 117 117 117 180 180 180 F F F
126572024-02-01T17:33:06.113Z 12 240-259 116 116 116 179 179 179 F F F
126582024-02-01T17:33:06.113Z 13 260-279 115 115 115 177 177 177 F F F
126592024-02-01T17:33:06.113Z 14 280-299 114 114 114 175 175 175 F F F
126602024-02-01T17:33:06.113Z 15 300-319 110 110 110 167 167 167 F F F
126612024-02-01T17:33:06.113Z 16 320-339 110 110 110 167 167 167 F F F
126622024-02-01T17:33:06.113Z 17 340-359 114 114 114 176 176 176 F F F
126632024-02-01T17:33:06.113Z 18 360-379 108 108 108 164 164 164 F F F
126642024-02-01T17:33:06.113Z 19 380-399 117 117 117 180 180 180 F F F
126652024-02-01T17:33:06.113Z 20 400-419 117 117 117 180 180 180 F F F
126662024-02-01T17:33:06.114Z 21 420-439 110 110 110 167 167 167 F F F
126672024-02-01T17:33:06.114Z 22 440-459 116 116 116 179 179 179 F F F
126682024-02-01T17:33:06.114Z 23 460-479 110 110 110 167 167 167 F F F
126692024-02-01T17:33:06.114Z 24 480-499 117 117 117 180 180 180 F F F
126702024-02-01T17:33:06.114Z 25 500-519 114 114 114 173 173 173 F F F
126712024-02-01T17:33:06.114Z 26 520-539 107 107 107 161 161 161 F F F
126722024-02-01T17:33:06.114Z 27 540-559 117 117 117 180 180 180 F F F
126732024-02-01T17:33:06.114Z 28 560-579 117 117 117 180 180 180 F F F
126742024-02-01T17:33:06.114Z 29 580-599 117 117 117 180 180 180 F F F
126752024-02-01T17:33:06.114ZMax gen: 117, Max flush: 180
126762024-02-01T17:33:06.117Z
126772024-02-01T17:33:06.117ZBegin loop 59
126782024-02-01T17:33:06.118ZDownstairs to restart: 0
126792024-02-01T17:33:06.140Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.384445098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
126802024-02-01T17:33:06.141Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.384954219Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
126812024-02-01T17:33:06.141Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385077157Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
126822024-02-01T17:33:06.142Z{"msg":"Crucible runtime is spawned
126832024-02-01T17:33:06.142ZCrucible bac5559d-48e5-4589-9ce8-855b3e5c6125 has session id: 2b3b94aa-1631-4520-bd6d-4d9cc3121aef","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385168801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126842024-02-01T17:33:06.142Z{"msg":"Upstairs opts: Upstairs UUID: bac5559d-48e5-4589-9ce8-855b3e5c6125, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385250872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126852024-02-01T17:33:06.142Z{"msg":"Crucible stats registered with UUID: bac5559d-48e5-4589-9ce8-855b3e5c6125","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385314847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126862024-02-01T17:33:06.142Z{"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.38534127Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
126872024-02-01T17:33:06.142Z{"msg":"bac5559d-48e5-4589-9ce8-855b3e5c6125 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385395408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126882024-02-01T17:33:06.142Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385420433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126892024-02-01T17:33:06.143Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385456112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126902024-02-01T17:33:06.143Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385498627Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126912024-02-01T17:33:06.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385539203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126922024-02-01T17:33:06.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385581086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126932024-02-01T17:33:06.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385625839Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126942024-02-01T17:33:06.145Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385748792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126952024-02-01T17:33:06.145Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385796896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126962024-02-01T17:33:06.145Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385837296Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126972024-02-01T17:33:06.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.385961449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126982024-02-01T17:33:06.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.386039143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
126992024-02-01T17:33:06.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.38616589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"io task","client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127002024-02-01T17:33:06.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.387323799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127012024-02-01T17:33:06.149Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.38739504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127022024-02-01T17:33:06.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.387572015Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127032024-02-01T17:33:06.149Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.387671669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127042024-02-01T17:33:06.149Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.38782612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127052024-02-01T17:33:06.149Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.387898798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127062024-02-01T17:33:06.152Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.388373272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127072024-02-01T17:33:06.152Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.388641846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127082024-02-01T17:33:06.152Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389268224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127092024-02-01T17:33:06.153Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.3893307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127102024-02-01T17:33:06.153Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389488834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127112024-02-01T17:33:06.155Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389571646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127122024-02-01T17:33:06.155Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389644531Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127132024-02-01T17:33:06.155Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389694092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127142024-02-01T17:33:06.155Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.389718646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127152024-02-01T17:33:06.155Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.39012777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127162024-02-01T17:33:06.156Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390207333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127172024-02-01T17:33:06.156Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 180, 179, 177, 177, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390238493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127182024-02-01T17:33:06.156Z{"msg":"[0]R generation[0..12]: [108, 110, 114, 115, 112, 114, 117, 116, 115, 115, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390262766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127192024-02-01T17:33:06.156Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390313322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127202024-02-01T17:33:06.156Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 180, 179, 177, 177, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390340875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127212024-02-01T17:33:06.156Z{"msg":"[1]R generation[0..12]: [108, 110, 114, 115, 112, 114, 117, 116, 115, 115, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390387981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127222024-02-01T17:33:06.156Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390413975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127232024-02-01T17:33:06.156Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 175, 177, 168, 173, 180, 179, 177, 177, 180, 180]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
127242024-02-01T17:33:06.156Z,"time":"2024-02-01T17:33:05.390472356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127252024-02-01T17:33:06.157Z{"msg":"[2]R generation[0..12]: [108, 110, 114, 115, 112, 114, 117, 116, 115, 115, 117, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390498353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127262024-02-01T17:33:06.157Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390544972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127272024-02-01T17:33:06.157Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390633899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127282024-02-01T17:33:06.157Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390689035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127292024-02-01T17:33:06.157Z{"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390712986Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127302024-02-01T17:33:06.157Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390752911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"":"downstairs","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127312024-02-01T17:33:06.160Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390805228Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127322024-02-01T17:33:06.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390830396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"0","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127332024-02-01T17:33:06.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390852304Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"1","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127342024-02-01T17:33:06.160Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390893469Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"client":"2","":"downstairs","session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127352024-02-01T17:33:06.160Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390917996Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127362024-02-01T17:33:06.160Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390957264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127372024-02-01T17:33:06.160Z{"msg":"bac5559d-48e5-4589-9ce8-855b3e5c6125 is now active with session: 2b3b94aa-1631-4520-bd6d-4d9cc3121aef","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.390980567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127382024-02-01T17:33:06.160Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.391019709Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601,"session_id":"2b3b94aa-1631-4520-bd6d-4d9cc3121aef"}
127392024-02-01T17:33:06.161Z{"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:05.391043137Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
127402024-02-01T17:33:06.161ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
127412024-02-01T17:33:06.161ZLoading write count information from file "/tmp/test_repair_verify.data"
127422024-02-01T17:33:06.161ZRun Repair workload
127432024-02-01T17:33:06.162Z01/30 Flush
127442024-02-01T17:33:06.162Z02/30 Write block 478 len 4096 data: 7
127452024-02-01T17:33:06.163Z03/30 Read block 362 len 32768
127462024-02-01T17:33:06.164Z04/30 Write block 212 len 4096 data: 9
127472024-02-01T17:33:06.164Z05/30 Write block 143 len 8192 data: 9 8
127482024-02-01T17:33:06.164Z06/30 Read block 147 len 8192
127492024-02-01T17:33:06.164Z07/30 Flush
127502024-02-01T17:33:06.172Z08/30 Write block 261 len 40960 data: 7 6 8 8 10 11 12 11 10 10
127512024-02-01T17:33:06.172Z09/30 Flush
127522024-02-01T17:33:06.172Z10/30 Write block 578 len 40960 data: 9 9 8 8 8 7 8 9 9 8
127532024-02-01T17:33:06.172Z11/30 Write block 74 len 32768 data: 13 12 7 5 4 4 5 4
127542024-02-01T17:33:06.172Z12/30 Flush
127552024-02-01T17:33:06.172Z13/30 Write block 582 len 40960 data: 9 8 9 10 10 9 10 9 9 8
127562024-02-01T17:33:06.172Z14/30 Write block 123 len 36864 data: 6 7 8 6 6 7 7 7 6
127572024-02-01T17:33:06.172Z15/30 Read block 420 len 36864
127582024-02-01T17:33:06.172Z16/30 Write block 503 len 28672 data: 8 6 5 5 5 8 9
127592024-02-01T17:33:06.172Z17/30 Read block 162 len 8192
127602024-02-01T17:33:06.173Z18/30 Read block 145 len 12288
127612024-02-01T17:33:06.173Z19/30 Write block 148 len 16384 data: 12 12 8 8
127622024-02-01T17:33:06.173Z20/30 Read block 128 len 40960
127632024-02-01T17:33:06.173Z21/30 Write block 162 len 40960 data: 11 11 11 8 7 7 10 13 15 16
127642024-02-01T17:33:06.173Z22/30 Read block 37 len 40960
127652024-02-01T17:33:06.173Z23/30 Read block 522 len 16384
127662024-02-01T17:33:06.173Z24/30 Read block 515 len 36864
127672024-02-01T17:33:06.173Z25/30 Read block 274 len 4096
127682024-02-01T17:33:06.174Z26/30 Read block 587 len 16384
127692024-02-01T17:33:06.175Z27/30 Read block 173 len 24576
127702024-02-01T17:33:06.175Z28/30 Read block 125 len 16384
127712024-02-01T17:33:06.175Z29/30 Write block 261 len 32768 data: 8 7 9 9 11 12 13 12
127722024-02-01T17:33:06.175Z30/30 Read block 360 len 36864
127732024-02-01T17:33:06.175Z----------------------------------------------------------------
127742024-02-01T17:33:06.175Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:29
127752024-02-01T17:33:06.175ZStates: Active Active Active
127762024-02-01T17:33:06.175ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127772024-02-01T17:33:06.175Z 2 Acked 1001 Write 1 Sent Done Done false
127782024-02-01T17:33:06.176Z 3 Acked 1002 Read 8 Sent Done Done false
127792024-02-01T17:33:06.176Z 4 Acked 1003 Write 1 Sent Done Done false
127802024-02-01T17:33:06.176Z 5 Acked 1004 Write 2 Sent Done Done false
127812024-02-01T17:33:06.176Z 6 Acked 1005 Read 2 Sent Done Done false
127822024-02-01T17:33:06.176Z 7 Acked 1006 Flush 0 Sent Done Done false
127832024-02-01T17:33:06.176Z 8 Acked 1007 Write 10 Sent Done Done false
127842024-02-01T17:33:06.176Z 9 Acked 1008 Flush 0 Sent Done Done false
127852024-02-01T17:33:06.177Z 10 Acked 1009 Write 10 Sent Done Done false
127862024-02-01T17:33:06.177Z 11 Acked 1010 Write 8 Sent Done Done false
127872024-02-01T17:33:06.177Z 12 Acked 1011 Flush 0 Sent Done Done false
127882024-02-01T17:33:06.177Z 13 Acked 1012 Write 10 Sent Done Done false
127892024-02-01T17:33:06.177Z 14 Acked 1013 Write 9 Sent Done Done false
127902024-02-01T17:33:06.177Z 15 Acked 1014 Read 9 Sent Done Done false
127912024-02-01T17:33:06.199Z 16 Acked 1015 Write 7 Sent Done Done false
127922024-02-01T17:33:06.199Z 17 Acked 1016 Read 2 Sent Done Done false
127932024-02-01T17:33:06.200Z 18 Acked 1017 Read 3 Sent Done Done false
127942024-02-01T17:33:06.200Z 19 Acked 1018 Write 4 Sent Done Done false
127952024-02-01T17:33:06.200Z 20 Acked 1019 Read 10 Sent Done Done false
127962024-02-01T17:33:06.200Z 21 Acked 1020 Write 10 Sent Done Done false
127972024-02-01T17:33:06.200Z 22 Acked 1021 Read 10 Sent Done Done false
127982024-02-01T17:33:06.200Z 23 Acked 1022 Read 4 Sent Done Done false
127992024-02-01T17:33:06.200Z 24 Acked 1023 Read 9 Sent Done Done false
128002024-02-01T17:33:06.200Z 25 Acked 1024 Read 1 Sent Done Done false
128012024-02-01T17:33:06.200Z 26 Acked 1025 Read 4 Sent Done Done false
128022024-02-01T17:33:06.200Z 27 Acked 1026 Read 6 Sent Done Done false
128032024-02-01T17:33:06.200Z 28 Acked 1027 Read 4 Sent Done Done false
128042024-02-01T17:33:06.201Z 29 Acked 1028 Write 8 Sent Done Done false
128052024-02-01T17:33:06.201Z 30 Acked 1029 Read 9 Sent Sent Done false
128062024-02-01T17:33:06.201Z STATES DS:0 DS:1 DS:2 TOTAL
128072024-02-01T17:33:06.201Z New 0 0 0 0
128082024-02-01T17:33:06.201Z Sent 29 1 0 30
128092024-02-01T17:33:06.201Z Done 0 28 29 57
128102024-02-01T17:33:06.203Z Skipped 0 0 0 0
128112024-02-01T17:33:06.203Z Error 0 0 0 0
128122024-02-01T17:33:06.203ZLast Flush: 1000 1011 1011
128132024-02-01T17:33:06.203ZDownstairs last five completed: 1000
128142024-02-01T17:33:06.203ZUpstairs last five completed: 30 29 28 27 26
128152024-02-01T17:33:06.204Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:05.441275687Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1601}
128162024-02-01T17:33:06.222ZWrote out file "/tmp/test_repair_verify.data"
128172024-02-01T17:33:06.252Z
128182024-02-01T17:33:06.264ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128192024-02-01T17:33:06.264ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128202024-02-01T17:33:06.265Z 0 000-019 108 108 108 163 163 163 F F F
128212024-02-01T17:33:06.265Z 1 020-039 110 110 110 167 167 167 F F F
128222024-02-01T17:33:06.265Z 2 040-059 114 114 114 175 175 175 F F F
128232024-02-01T17:33:06.265Z 3 060-079 115 118 118 177 184 184 F F F <---
128242024-02-01T17:33:06.265Z 4 080-099 112 118 118 168 184 184 F F F <---
128252024-02-01T17:33:06.266Z 5 100-119 114 114 114 173 173 173 F F F
128262024-02-01T17:33:06.266Z 6 120-139 117 117 117 180 180 180 F T T <---
128272024-02-01T17:33:06.266Z 7 140-159 116 118 118 179 182 182 F T T <---
128282024-02-01T17:33:06.266Z 8 160-179 115 115 115 177 177 177 F T T <---
128292024-02-01T17:33:06.280Z 9 180-199 115 115 115 177 177 177 F F F
128302024-02-01T17:33:06.280Z 10 200-219 117 118 118 180 182 182 F F F <---
128312024-02-01T17:33:06.280Z 11 220-239 117 117 117 180 180 180 F F F
128322024-02-01T17:33:06.280Z 12 240-259 116 116 116 179 179 179 F F F
128332024-02-01T17:33:06.281Z 13 260-279 115 118 118 177 183 183 F T T <---
128342024-02-01T17:33:06.281Z 14 280-299 114 114 114 175 175 175 F F F
128352024-02-01T17:33:06.281Z 15 300-319 110 110 110 167 167 167 F F F
128362024-02-01T17:33:06.281Z 16 320-339 110 110 110 167 167 167 F F F
128372024-02-01T17:33:06.281Z 17 340-359 114 114 114 176 176 176 F F F
128382024-02-01T17:33:06.281Z 18 360-379 108 108 108 164 164 164 F F F
128392024-02-01T17:33:06.281Z 19 380-399 117 117 117 180 180 180 F F F
128402024-02-01T17:33:06.281Z 20 400-419 117 117 117 180 180 180 F F F
128412024-02-01T17:33:06.281Z 21 420-439 110 110 110 167 167 167 F F F
128422024-02-01T17:33:06.281Z 22 440-459 116 116 116 179 179 179 F F F
128432024-02-01T17:33:06.282Z 23 460-479 110 118 118 167 182 182 F F F <---
128442024-02-01T17:33:06.282Z 24 480-499 117 117 117 180 180 180 F F F
128452024-02-01T17:33:06.282Z 25 500-519 114 114 114 173 173 173 F T T <---
128462024-02-01T17:33:06.282Z 26 520-539 107 107 107 161 161 161 F F F
128472024-02-01T17:33:06.282Z 27 540-559 117 117 117 180 180 180 F F F
128482024-02-01T17:33:06.282Z 28 560-579 117 118 118 180 184 184 F F F <---
128492024-02-01T17:33:06.282Z 29 580-599 117 118 118 180 184 184 F T T <---
128502024-02-01T17:33:06.282ZMax gen: 118, Max flush: 184
128512024-02-01T17:33:06.282ZError: Difference in extent metadata found!
128522024-02-01T17:33:06.283ZA Difference in extent metadata is expected here
128532024-02-01T17:33:06.283ZOn loop 59
128542024-02-01T17:33:06.283Z
128552024-02-01T17:33:06.283ZVerifying data now
128562024-02-01T17:33:22.168ZNo change after verify
128572024-02-01T17:33:22.168ZLoop: 59 Downstairs dump after verify (and repair):
128582024-02-01T17:33:22.347ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128592024-02-01T17:33:22.347Z 0 000-019 108 108 108 163 163 163 F F F
128602024-02-01T17:33:22.347Z 1 020-039 110 110 110 167 167 167 F F F
128612024-02-01T17:33:22.941Z 2 040-059 114 114 114 175 175 175 F F F
128622024-02-01T17:33:22.942Z 3 060-079 118 118 118 184 184 184 F F F
128632024-02-01T17:33:22.942Z 4 080-099 118 118 118 184 184 184 F F F
128642024-02-01T17:33:22.942Z 5 100-119 114 114 114 173 173 173 F F F
128652024-02-01T17:33:22.942Z 6 120-139 119 119 119 185 185 185 F F F
128662024-02-01T17:33:22.942Z 7 140-159 119 119 119 185 185 185 F F F
128672024-02-01T17:33:22.942Z 8 160-179 119 119 119 185 185 185 F F F
128682024-02-01T17:33:22.942Z 9 180-199 115 115 115 177 177 177 F F F
128692024-02-01T17:33:22.942Z 10 200-219 118 118 118 182 182 182 F F F
128702024-02-01T17:33:22.942Z 11 220-239 117 117 117 180 180 180 F F F
128712024-02-01T17:33:22.942Z 12 240-259 116 116 116 179 179 179 F F F
128722024-02-01T17:33:22.942Z 13 260-279 119 119 119 185 185 185 F F F
128732024-02-01T17:33:22.942Z 14 280-299 114 114 114 175 175 175 F F F
128742024-02-01T17:33:22.942Z 15 300-319 110 110 110 167 167 167 F F F
128752024-02-01T17:33:22.942Z 16 320-339 110 110 110 167 167 167 F F F
128762024-02-01T17:33:22.942Z 17 340-359 114 114 114 176 176 176 F F F
128772024-02-01T17:33:22.942Z 18 360-379 108 108 108 164 164 164 F F F
128782024-02-01T17:33:22.942Z 19 380-399 117 117 117 180 180 180 F F F
128792024-02-01T17:33:22.943Z 20 400-419 117 117 117 180 180 180 F F F
128802024-02-01T17:33:22.943Z 21 420-439 110 110 110 167 167 167 F F F
128812024-02-01T17:33:22.943Z 22 440-459 116 116 116 179 179 179 F F F
128822024-02-01T17:33:22.943Z 23 460-479 118 118 118 182 182 182 F F F
128832024-02-01T17:33:22.943Z 24 480-499 117 117 117 180 180 180 F F F
128842024-02-01T17:33:22.943Z 25 500-519 119 119 119 185 185 185 F F F
128852024-02-01T17:33:22.943Z 26 520-539 107 107 107 161 161 161 F F F
128862024-02-01T17:33:22.943Z 27 540-559 117 117 117 180 180 180 F F F
128872024-02-01T17:33:22.953Z 28 560-579 118 118 118 184 184 184 F F F
128882024-02-01T17:33:22.953Z 29 580-599 119 119 119 185 185 185 F F F
128892024-02-01T17:33:22.953ZMax gen: 119, Max flush: 185
128902024-02-01T17:33:22.953Z
128912024-02-01T17:33:22.953ZBegin loop 60
128922024-02-01T17:33:22.953ZDownstairs to restart: 0
128932024-02-01T17:33:22.953Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.50839261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
128942024-02-01T17:33:22.953Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.508890801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
128952024-02-01T17:33:22.953Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.508989788Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
128962024-02-01T17:33:22.954Z{"msg":"Crucible bab0e4b9-d5b0-462b-9197-144369f44bc9 has session id: 23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0","v":0,"name":"crucible","level":30Crucible runtime is spawned
128972024-02-01T17:33:22.954Z,"time":"2024-02-01T17:33:10.509028264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
128982024-02-01T17:33:22.954Z{"msg":"Upstairs opts: Upstairs UUID: bab0e4b9-d5b0-462b-9197-144369f44bc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509164789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
128992024-02-01T17:33:22.954Z{"msg":"Crucible stats registered with UUID: bab0e4b9-d5b0-462b-9197-144369f44bc9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509216714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129002024-02-01T17:33:22.954Z{"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509313212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
129012024-02-01T17:33:22.954Z{"msg":"bab0e4b9-d5b0-462b-9197-144369f44bc9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509361476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129022024-02-01T17:33:22.954Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509480498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129032024-02-01T17:33:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509559077Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129042024-02-01T17:33:22.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509634939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129052024-02-01T17:33:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509710288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129062024-02-01T17:33:22.955Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509814411Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129072024-02-01T17:33:22.955Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.509898418Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129082024-02-01T17:33:22.955Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510085522Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129092024-02-01T17:33:22.955Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510187443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129102024-02-01T17:33:22.956Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510251632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129112024-02-01T17:33:22.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510294111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129122024-02-01T17:33:22.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510370746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129132024-02-01T17:33:22.956Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.510450728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"io task","client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129142024-02-01T17:33:22.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.511281562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129152024-02-01T17:33:22.956Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.51134447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129162024-02-01T17:33:22.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.51139377Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129172024-02-01T17:33:22.956Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.511440938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129182024-02-01T17:33:22.956Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.511540652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129192024-02-01T17:33:22.956Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.511599384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129202024-02-01T17:33:22.956Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.51240842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129212024-02-01T17:33:22.956Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.512522769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129222024-02-01T17:33:22.956Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.51261358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129232024-02-01T17:33:22.956Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513237403Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129242024-02-01T17:33:22.956Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513317303Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129252024-02-01T17:33:22.956Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513439012Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129262024-02-01T17:33:22.959Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513484694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129272024-02-01T17:33:22.959Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513527749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129282024-02-01T17:33:22.959Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513553548Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129292024-02-01T17:33:22.959Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.51359228Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129302024-02-01T17:33:22.959Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513617006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129312024-02-01T17:33:22.959Z{"msg":"[0]R flush_numbers[0..12]: [163, 167, 175, 184, 184, 173, 185, 185, 185, 177, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513658126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129322024-02-01T17:33:22.959Z{"msg":"[0]R generation[0..12]: [108, 110, 114, 118, 118, 114, 119, 119, 119, 115, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513701297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129332024-02-01T17:33:22.959Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513744929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129342024-02-01T17:33:22.959Z{"msg":"[1]R flush_numbers[0..12]: [163, 167, 175, 184, 184, 173, 185, 185, 185, 177, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513787848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129352024-02-01T17:33:22.959Z{"msg":"[1]R generation[0..12]: [108, 110, 114, 118, 118, 114, 119, 119, 119, 115, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513830243Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129362024-02-01T17:33:22.961Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513855658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129372024-02-01T17:33:22.961Z{"msg":"[2]R flush_numbers[0..12]: [163, 167, 175, 184, 184, 173, 185, 185, 185, 177, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513879278Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
129382024-02-01T17:33:22.961Z,"":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129392024-02-01T17:33:22.961Z{"msg":"[2]R generation[0..12]: [108, 110, 114, 118, 118, 114, 119, 119, 119, 115, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.513989696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129402024-02-01T17:33:22.961Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514047581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129412024-02-01T17:33:22.961Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514090649Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129422024-02-01T17:33:22.961Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514136515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129432024-02-01T17:33:22.961Z{"msg":"Next flush: 186","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514177351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129442024-02-01T17:33:22.961Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514202445Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"":"downstairs","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129452024-02-01T17:33:22.961Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514243031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129462024-02-01T17:33:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514266516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"0","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129472024-02-01T17:33:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514306464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"1","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129482024-02-01T17:33:22.963Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514349571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"client":"2","":"downstairs","session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129492024-02-01T17:33:22.963Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514395158Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129502024-02-01T17:33:22.963Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514419099Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129512024-02-01T17:33:22.963Z{"msg":"bab0e4b9-d5b0-462b-9197-144369f44bc9 is now active with session: 23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514459708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129522024-02-01T17:33:22.963Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514481573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609,"session_id":"23d31ce9-2b3a-4fa0-8b0c-de46919d8ec0"}
129532024-02-01T17:33:22.963Z{"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:10.514520664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
129542024-02-01T17:33:22.963ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129552024-02-01T17:33:22.963ZLoading write count information from file "/tmp/test_repair_verify.data"
129562024-02-01T17:33:22.963ZRun Repair workload
129572024-02-01T17:33:22.963Z01/30 Write block 183 len 20480 data: 14 13 14 12 12
129582024-02-01T17:33:22.963Z02/30 Write block 467 len 32768 data: 8 8 7 8 9 9 7 6
129592024-02-01T17:33:22.963ZError: Difference in extent metadata found!
129602024-02-01T17:33:22.964Z03/30 Read block 35 len 16384
129612024-02-01T17:33:22.979Z04/30 Write block 288 len 12288 data: 10 12 9
129622024-02-01T17:33:22.979Z05/30 Write block 580 len 20480 data: 9 9 10 9 10
129632024-02-01T17:33:22.979Z06/30 Read block 159 len 36864
129642024-02-01T17:33:22.979Z07/30 Read block 504 len 8192
129652024-02-01T17:33:22.979Z08/30 Write block 459 len 36864 data: 14 12 13 11 9 8 7 9 9
129662024-02-01T17:33:22.979Z09/30 Read block 538 len 40960
129672024-02-01T17:33:22.979Z10/30 Read block 379 len 4096
129682024-02-01T17:33:22.979Z11/30 Read block 252 len 4096
129692024-02-01T17:33:22.979Z12/30 Read block 448 len 40960
129702024-02-01T17:33:22.979Z13/30 Read block 439 len 20480
129712024-02-01T17:33:22.979Z14/30 Read block 52 len 4096
129722024-02-01T17:33:22.979Z15/30 Write block 104 len 16384 data: 7 10 10 11
129732024-02-01T17:33:22.980Z16/30 Write block 10 len 12288 data: 9 6 5
129742024-02-01T17:33:22.980Z17/30 Read block 55 len 40960
129752024-02-01T17:33:22.980Z18/30 Read block 332 len 24576
129762024-02-01T17:33:22.980Z19/30 Read block 162 len 28672
129772024-02-01T17:33:22.980Z20/30 Flush
129782024-02-01T17:33:22.980Z21/30 Write block 561 len 40960 data: 9 9 8 7 7 6 5 5 6 7
129792024-02-01T17:33:22.980Z22/30 Read block 489 len 4096
129802024-02-01T17:33:22.980Z23/30 Write block 435 len 12288 data: 13 12 13
129812024-02-01T17:33:22.980Z24/30 Write block 489 len 12288 data: 10 11 11
129822024-02-01T17:33:22.980Z25/30 Flush
129832024-02-01T17:33:22.982Z26/30 Write block 499 len 4096 data: 9
129842024-02-01T17:33:22.982Z27/30 Read block 67 len 12288
129852024-02-01T17:33:22.982Z28/30 Write block 276 len 12288 data: 12 12 13
129862024-02-01T17:33:22.982Z29/30 Write block 262 len 36864 data: 8 10 10 12 13 14 13 11 11
129872024-02-01T17:33:22.982Z30/30 Read block 107 len 24576
129882024-02-01T17:33:22.982Z----------------------------------------------------------------
129892024-02-01T17:33:22.982Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30
129902024-02-01T17:33:22.982ZStates: Active Active Active
129912024-02-01T17:33:22.982ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129922024-02-01T17:33:22.982Z 1 Acked 1000 Write 5 Done Done Done false
129932024-02-01T17:33:22.982Z 2 Acked 1001 Write 8 Done Done Done false
129942024-02-01T17:33:22.982Z 3 Acked 1002 Read 4 Done Done Done false
129952024-02-01T17:33:22.982Z 4 Acked 1003 Write 3 Done Done Done false
129962024-02-01T17:33:22.982Z 5 Acked 1004 Write 5 Done Done Done false
129972024-02-01T17:33:22.982Z 6 Acked 1005 Read 9 Done Done Done false
129982024-02-01T17:33:22.982Z 7 Acked 1006 Read 2 Done Done Done false
129992024-02-01T17:33:22.983Z 8 Acked 1007 Write 9 Sent Done Done false
130002024-02-01T17:33:22.983Z 9 Acked 1008 Read 10 Sent Done Done false
130012024-02-01T17:33:22.983Z 10 Acked 1009 Read 1 Sent Done Done false
130022024-02-01T17:33:22.983Z 11 Acked 1010 Read 1 Sent Done Done false
130032024-02-01T17:33:22.983Z 12 Acked 1011 Read 10 Sent Done Done false
130042024-02-01T17:33:22.983Z 13 Acked 1012 Read 5 Sent Done Done false
130052024-02-01T17:33:22.983Z 14 Acked 1013 Read 1 Sent Done Done false
130062024-02-01T17:33:22.983Z 15 Acked 1014 Write 4 Sent Done Done false
130072024-02-01T17:33:22.983Z 16 Acked 1015 Write 3 Sent Done Done false
130082024-02-01T17:33:22.983Z 17 Acked 1016 Read 10 Sent Done Done false
130092024-02-01T17:33:22.983Z 18 Acked 1017 Read 6 Sent Done Done false
130102024-02-01T17:33:22.983Z 19 Acked 1018 Read 7 Sent Done Done false
130112024-02-01T17:33:22.983Z 20 Acked 1019 Flush 0 Sent Done Done false
130122024-02-01T17:33:22.983Z 21 Acked 1020 Write 10 Sent Done Done false
130132024-02-01T17:33:22.983Z 22 Acked 1021 Read 1 Sent Done Done false
130142024-02-01T17:33:22.983Z 23 Acked 1022 Write 3 Sent Done Done false
130152024-02-01T17:33:22.983Z 24 Acked 1023 Write 3 Sent Done Done false
130162024-02-01T17:33:22.983Z 25 Acked 1024 Flush 0 Sent Done Done false
130172024-02-01T17:33:22.983Z 26 Acked 1025 Write 1 Sent Done Done false
130182024-02-01T17:33:22.983Z 27 Acked 1026 Read 3 Sent Done Done false
130192024-02-01T17:33:22.983Z 28 Acked 1027 Write 3 Sent Done Done false
130202024-02-01T17:33:22.984Z 29 Acked 1028 Write 9 Sent Done Done false
130212024-02-01T17:33:22.984Z 30 Acked 1029 Read 6 Sent Done Sent false
130222024-02-01T17:33:22.984Z STATES DS:0 DS:1 DS:2 TOTAL
130232024-02-01T17:33:22.984Z New 0 0 0 0
130242024-02-01T17:33:22.984Z Sent 23 0 1 24
130252024-02-01T17:33:22.984Z Done 7 30 29 66
130262024-02-01T17:33:22.984Z Skipped 0 0 0 0
130272024-02-01T17:33:22.984Z Error 0 0 0 0
130282024-02-01T17:33:22.984ZLast Flush: 0 1024 1024
130292024-02-01T17:33:22.984ZDownstairs last five completed:
130302024-02-01T17:33:22.984ZUpstairs last five completed: 30 29 28 27 26
130312024-02-01T17:33:22.984Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:10.559324352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1609}
130322024-02-01T17:33:22.984ZWrote out file "/tmp/test_repair_verify.data"
130332024-02-01T17:33:22.984Z
130342024-02-01T17:33:22.984ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130352024-02-01T17:33:22.984ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130362024-02-01T17:33:22.984Z 0 000-019 108 120 120 163 186 186 F F F <---
130372024-02-01T17:33:22.984Z 1 020-039 110 110 110 167 167 167 F F F
130382024-02-01T17:33:22.984Z 2 040-059 114 114 114 175 175 175 F F F
130392024-02-01T17:33:22.984Z 3 060-079 118 118 118 184 184 184 F F F
130402024-02-01T17:33:22.985Z 4 080-099 118 118 118 184 184 184 F F F
130412024-02-01T17:33:22.985Z 5 100-119 114 120 120 173 186 186 F F F <---
130422024-02-01T17:33:22.985Z 6 120-139 119 119 119 185 185 185 F F F
130432024-02-01T17:33:22.985Z 7 140-159 119 119 119 185 185 185 F F F
130442024-02-01T17:33:22.985Z 8 160-179 119 119 119 185 185 185 F F F
130452024-02-01T17:33:22.985Z 9 180-199 115 120 120 177 186 186 T F F <---
130462024-02-01T17:33:22.985Z 10 200-219 118 118 118 182 182 182 F F F
130472024-02-01T17:33:22.985Z 11 220-239 117 117 117 180 180 180 F F F
130482024-02-01T17:33:22.985Z 12 240-259 116 116 116 179 179 179 F F F
130492024-02-01T17:33:22.985Z 13 260-279 119 119 119 185 185 185 F T T <---
130502024-02-01T17:33:22.985Z 14 280-299 114 120 120 175 186 186 T F F <---
130512024-02-01T17:33:22.985Z 15 300-319 110 110 110 167 167 167 F F F
130522024-02-01T17:33:22.985Z 16 320-339 110 110 110 167 167 167 F F F
130532024-02-01T17:33:22.986Z 17 340-359 114 114 114 176 176 176 F F F
130542024-02-01T17:33:22.986Z 18 360-379 108 108 108 164 164 164 F F F
130552024-02-01T17:33:22.986Z 19 380-399 117 117 117 180 180 180 F F F
130562024-02-01T17:33:22.986Z 20 400-419 117 117 117 180 180 180 F F F
130572024-02-01T17:33:22.986Z 21 420-439 110 120 120 167 187 187 F F F <---
130582024-02-01T17:33:22.986Z 22 440-459 116 120 120 179 186 186 F F F <---
130592024-02-01T17:33:22.986Z 23 460-479 118 120 120 182 186 186 T F F <---
130602024-02-01T17:33:22.986Z 24 480-499 117 120 120 180 187 187 F T T <---
130612024-02-01T17:33:23.417Z 25 500-519 119 119 119 185 185 185 F F F
130622024-02-01T17:33:23.417Z 26 520-539 107 107 107 161 161 161 F F F
130632024-02-01T17:33:23.422Z 27 540-559 117 117 117 180 180 180 F F F
130642024-02-01T17:33:23.422Z 28 560-579 118 120 120 184 187 187 F F F <---
130652024-02-01T17:33:23.422Z 29 580-599 119 120 120 185 186 186 T F F <---
130662024-02-01T17:33:23.422ZMax gen: 120, Max flush: 187
130672024-02-01T17:33:23.422ZA Difference in extent metadata is expected here
130682024-02-01T17:33:23.422ZOn loop 60
130692024-02-01T17:33:23.422Z
130702024-02-01T17:33:23.422ZVerifying data now
130712024-02-01T17:33:23.422ZNo change after verify
130722024-02-01T17:33:23.422ZLoop: 60 Downstairs dump after verify (and repair):
130732024-02-01T17:33:23.422ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130742024-02-01T17:33:23.422Z 0 000-019 120 120 120 186 186 186 F F F
130752024-02-01T17:33:23.422Z 1 020-039 110 110 110 167 167 167 F F F
130762024-02-01T17:33:23.422Z 2 040-059 114 114 114 175 175 175 F F F
130772024-02-01T17:33:23.422Z 3 060-079 118 118 118 184 184 184 F F F
130782024-02-01T17:33:23.422Z 4 080-099 118 118 118 184 184 184 F F F
130792024-02-01T17:33:23.422Z 5 100-119 120 120 120 186 186 186 F F F
130802024-02-01T17:33:23.422Z 6 120-139 119 119 119 185 185 185 F F F
130812024-02-01T17:33:23.423Z 7 140-159 119 119 119 185 185 185 F F F
130822024-02-01T17:33:23.423Z 8 160-179 119 119 119 185 185 185 F F F
130832024-02-01T17:33:23.423Z 9 180-199 120 120 120 186 186 186 F F F
130842024-02-01T17:33:23.423Z 10 200-219 118 118 118 182 182 182 F F F
130852024-02-01T17:33:23.423Z 11 220-239 117 117 117 180 180 180 F F F
130862024-02-01T17:33:23.423Z 12 240-259 116 116 116 179 179 179 F F F
130872024-02-01T17:33:23.423Z 13 260-279 121 121 121 188 188 188 F F F
130882024-02-01T17:33:23.423Z 14 280-299 120 120 120 186 186 186 F F F
130892024-02-01T17:33:23.423Z 15 300-319 110 110 110 167 167 167 F F F
130902024-02-01T17:33:23.423Z 16 320-339 110 110 110 167 167 167 F F F
130912024-02-01T17:33:23.424Z 17 340-359 114 114 114 176 176 176 F F F
130922024-02-01T17:33:23.424Z 18 360-379 108 108 108 164 164 164 F F F
130932024-02-01T17:33:23.424Z 19 380-399 117 117 117 180 180 180 F F F
130942024-02-01T17:33:23.424Z 20 400-419 117 117 117 180 180 180 F F F
130952024-02-01T17:33:23.424Z 21 420-439 120 120 120 187 187 187 F F F
130962024-02-01T17:33:23.424Z 22 440-459 120 120 120 186 186 186 F F F
130972024-02-01T17:33:23.424Z 23 460-479 120 120 120 186 186 186 F F F
130982024-02-01T17:33:23.424Z 24 480-499 121 121 121 188 188 188 F F F
130992024-02-01T17:33:23.424Z 25 500-519 119 119 119 185 185 185 F F F
131002024-02-01T17:33:23.424Z 26 520-539 107 107 107 161 161 161 F F F
131012024-02-01T17:33:23.424Z 27 540-559 117 117 117 180 180 180 F F F
131022024-02-01T17:33:23.424Z 28 560-579 120 120 120 187 187 187 F F F
131032024-02-01T17:33:23.424Z 29 580-599 120 120 120 186 186 186 F F F
131042024-02-01T17:33:23.424ZMax gen: 121, Max flush: 188
131052024-02-01T17:33:23.424Z
131062024-02-01T17:33:23.424ZBegin loop 61
131072024-02-01T17:33:23.424ZDownstairs to restart: 0
131082024-02-01T17:33:23.424Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612099835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
131092024-02-01T17:33:23.424Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612600414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
131102024-02-01T17:33:23.424Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612644208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
131112024-02-01T17:33:23.424Z{"msg":"Crucible eab2e8dd-0ff7-4e88-8fc5-2bb1f2e096f0 has session id: 4dc2e9f5-034b-40f3-ada5-daa6a6486a9c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612685903Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131122024-02-01T17:33:23.424Z{"msg":"Upstairs opts: Upstairs UUID: eab2e8dd-0ff7-4e88-8fc5-2bb1f2e096f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612737173Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131132024-02-01T17:33:23.424Z{"msg":"Crucible stats registered with UUID: eab2e8dd-0ff7-4e88-8fc5-2bb1f2e096f0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612778981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131142024-02-01T17:33:23.424ZCrucible runtime is spawned
131152024-02-01T17:33:23.425Z{"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.612966746Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
131162024-02-01T17:33:23.425Z{"msg":"eab2e8dd-0ff7-4e88-8fc5-2bb1f2e096f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613149876Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131172024-02-01T17:33:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613178058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131182024-02-01T17:33:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613209939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131192024-02-01T17:33:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613233619Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131202024-02-01T17:33:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613256655Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131212024-02-01T17:33:23.425Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613278998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131222024-02-01T17:33:23.425Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613301341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131232024-02-01T17:33:23.425Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613344241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131242024-02-01T17:33:23.425Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613378345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131252024-02-01T17:33:23.425Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613474561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131262024-02-01T17:33:23.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613772395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131272024-02-01T17:33:23.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.613823172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131282024-02-01T17:33:23.425Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.614174426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"io task","client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131292024-02-01T17:33:23.425Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.615358637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131302024-02-01T17:33:23.427Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.615406801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131312024-02-01T17:33:23.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.615432802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131322024-02-01T17:33:23.427Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.615455501Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131332024-02-01T17:33:23.427Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.616410652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131342024-02-01T17:33:23.427Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.616575711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131352024-02-01T17:33:23.427Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.616677371Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131362024-02-01T17:33:23.428Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.616720738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131372024-02-01T17:33:23.428Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.61747629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131382024-02-01T17:33:23.428Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.617539424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131392024-02-01T17:33:23.428Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.61758292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131402024-02-01T17:33:23.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:16.617608906Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131412024-02-01T17:33:23.431Z{Wait for a query_work_queue command to finish before sending IO
131422024-02-01T17:33:23.431Z"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523039146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131432024-02-01T17:33:23.431Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523123678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131442024-02-01T17:33:23.431Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523150971Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131452024-02-01T17:33:23.431Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.52317361Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131462024-02-01T17:33:23.431Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523196451Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131472024-02-01T17:33:23.431Z{"msg":"[0]R flush_numbers[0..12]: [186, 167, 175, 184, 184, 186, 185, 185, 185, 186, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.52322072Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131482024-02-01T17:33:23.431Z{"msg":"[0]R generation[0..12]: [120, 110, 114, 118, 118, 120, 119, 119, 119, 120, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523244578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131492024-02-01T17:33:23.431Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523268917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131502024-02-01T17:33:23.433Z{"msg":"[1]R flush_numbers[0..12]: [186, 167, 175, 184, 184, 186, 185, 185, 185, 186, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523292851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131512024-02-01T17:33:23.433Z{"msg":"[1]R generation[0..12]: [120, 110, 114, 118, 118, 120, 119, 119, 119, 120, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523317588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131522024-02-01T17:33:23.433Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523353989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131532024-02-01T17:33:23.433Z{"msg":"[2]R flush_numbers[0..12]: [186, 167, 175, 184, 184, 186, 185, 185, 185, 186, 182, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523379019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131542024-02-01T17:33:23.433Z{"msg":"[2]R generation[0..12]: [120, 110, 114, 118, 118, 120, 119, 119, 119, 120, 118, 117]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523402356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131552024-02-01T17:33:23.433Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523425918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131562024-02-01T17:33:23.433Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523447942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131572024-02-01T17:33:23.433Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523470091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131582024-02-01T17:33:23.433Z{"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523491441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131592024-02-01T17:33:23.433Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523513759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"":"downstairs","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131602024-02-01T17:33:23.433Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523536022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131612024-02-01T17:33:23.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523556252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"0","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131622024-02-01T17:33:23.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523578264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"1","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131632024-02-01T17:33:23.446Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523600065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"client":"2","":"downstairs","session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131642024-02-01T17:33:23.446Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523622485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131652024-02-01T17:33:23.446Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523643209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131662024-02-01T17:33:23.446Z{"msg":"eab2e8dd-0ff7-4e88-8fc5-2bb1f2e096f0 is now active with session: 4dc2e9f5-034b-40f3-ada5-daa6a6486a9c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523665009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131672024-02-01T17:33:23.446Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523685091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617,"session_id":"4dc2e9f5-034b-40f3-ada5-daa6a6486a9c"}
131682024-02-01T17:33:23.446Z{"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:21.523705642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
131692024-02-01T17:33:23.446ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
131702024-02-01T17:33:23.447ZLoading write count information from file "/tmp/test_repair_verify.data"
131712024-02-01T17:33:23.447ZRun Repair workload
131722024-02-01T17:33:23.447Z01/30 Flush
131732024-02-01T17:33:23.447Z02/30 Write block 303 len 24576 data: 7 8 8 5 4 8
131742024-02-01T17:33:23.447Z03/30 Write block 252 len 36864 data: 7 6 8 9 8 8 11 11 9
131752024-02-01T17:33:23.447Z04/30 Read block 143 len 40960
131762024-02-01T17:33:23.447Z05/30 Read block 112 len 20480
131772024-02-01T17:33:23.447Z06/30 Write block 229 len 40960 data: 9 7 8 7 7 7 6 9 9 9
131782024-02-01T17:33:23.448Z07/30 Read block 93 len 24576
131792024-02-01T17:33:23.448Z08/30 Read block 492 len 24576
131802024-02-01T17:33:23.448Z09/30 Write block 8 len 16384 data: 10 8 10 7
131812024-02-01T17:33:23.448Z10/30 Read block 229 len 40960
131822024-02-01T17:33:23.448Z11/30 Write block 361 len 16384 data: 15 16 13 10
131832024-02-01T17:33:23.448Z12/30 Read block 12 len 28672
131842024-02-01T17:33:23.448Z13/30 Flush
131852024-02-01T17:33:23.448Z14/30 Write block 262 len 20480 data: 9 11 11 13 14
131862024-02-01T17:33:23.448Z15/30 Read block 425 len 12288
131872024-02-01T17:33:23.448Z16/30 Write block 92 len 12288 data: 7 6 6
131882024-02-01T17:33:23.448Z17/30 Write block 139 len 8192 data: 12 12
131892024-02-01T17:33:23.448Z18/30 Read block 131 len 12288
131902024-02-01T17:33:23.448Z19/30 Write block 104 len 28672 data: 8 11 11 12 10 9 6
131912024-02-01T17:33:23.448Z20/30 Write block 242 len 28672 data: 9 8 8 8 8 8 9
131922024-02-01T17:33:23.448Z21/30 Read block 271 len 28672
131932024-02-01T17:33:23.448Z22/30 Flush
131942024-02-01T17:33:23.448Z23/30 Write block 121 len 32768 data: 6 7 7 8 9 7 7 8
131952024-02-01T17:33:23.448Z24/30 Read block 461 len 40960
131962024-02-01T17:33:23.449Z25/30 Read block 444 len 40960
131972024-02-01T17:33:23.449Z26/30 Write block 553 len 24576 data: 9 7 10 9 10 11
131982024-02-01T17:33:23.449Z27/30 Read block 379 len 4096
131992024-02-01T17:33:23.449Z28/30 Read block 294 len 12288
132002024-02-01T17:33:23.449Z29/30 Write block 78 len 12288 data: 5 5 6
132012024-02-01T17:33:23.449Z30/30 Write block 365 len 32768 data: 9 10 9 7 7 10 12 11
132022024-02-01T17:33:23.449Z----------------------------------------------------------------
132032024-02-01T17:33:23.449Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
132042024-02-01T17:33:23.449ZStates: Active Active Active
132052024-02-01T17:33:23.449ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132062024-02-01T17:33:23.450Z 1 Acked 1000 Flush 0 Sent Done Done false
132072024-02-01T17:33:23.450Z 2 Acked 1001 Write 6 Sent Done Done false
132082024-02-01T17:33:23.450Z 3 Acked 1002 Write 9 Sent Done Done false
132092024-02-01T17:33:23.450Z 4 Acked 1003 Read 10 Sent Done Done false
132102024-02-01T17:33:23.450Z 5 Acked 1004 Read 5 Sent Done Done false
132112024-02-01T17:33:23.450Z 6 Acked 1005 Write 10 Sent Done Done false
132122024-02-01T17:33:23.450Z 7 Acked 1006 Read 6 Sent Done Done false
132132024-02-01T17:33:23.450Z 8 Acked 1007 Read 6 Sent Done Done false
132142024-02-01T17:33:23.450Z 9 Acked 1008 Write 4 Sent Done Done false
132152024-02-01T17:33:23.450Z 10 Acked 1009 Read 10 Sent Done Done false
132162024-02-01T17:33:23.450Z 11 Acked 1010 Write 4 Sent Done Done false
132172024-02-01T17:33:23.450Z 12 Acked 1011 Read 7 Sent Done Done false
132182024-02-01T17:33:23.450Z 13 Acked 1012 Flush 0 Sent Done Done false
132192024-02-01T17:33:23.450Z 14 Acked 1013 Write 5 Sent Done Done false
132202024-02-01T17:33:23.450Z 15 Acked 1014 Read 3 Sent Done Done false
132212024-02-01T17:33:23.450Z 16 Acked 1015 Write 3 Sent Done Done false
132222024-02-01T17:33:23.450Z 17 Acked 1016 Write 2 Sent Done Done false
132232024-02-01T17:33:23.450Z 18 Acked 1017 Read 3 Sent Done Done false
132242024-02-01T17:33:23.450Z 19 Acked 1018 Write 7 Sent Done Done false
132252024-02-01T17:33:23.450Z 20 Acked 1019 Write 7 Sent Done Done false
132262024-02-01T17:33:23.450Z 21 Acked 1020 Read 7 Sent Done Done false
132272024-02-01T17:33:23.450Z 22 Acked 1021 Flush 0 Sent Done Done false
132282024-02-01T17:33:23.451Z 23 Acked 1022 Write 8 Sent Done Done false
132292024-02-01T17:33:23.451Z 24 Acked 1023 Read 10 Sent Done Done false
132302024-02-01T17:33:23.451Z 25 Acked 1024 Read 10 Sent Done Done false
132312024-02-01T17:33:23.451Z 26 Acked 1025 Write 6 Sent Done Done false
132322024-02-01T17:33:23.451Z 27 Acked 1026 Read 1 Sent Done Done false
132332024-02-01T17:33:23.451Z 28 Acked 1027 Read 3 Sent Done Done false
132342024-02-01T17:33:23.451Z 29 Acked 1028 Write 3 Sent Done Done false
132352024-02-01T17:33:23.451Z 30 Acked 1029 Write 8 Sent Sent Sent false
132362024-02-01T17:33:23.451Z STATES DS:0 DS:1 DS:2 TOTAL
132372024-02-01T17:33:23.451Z New 0 0 0 0
132382024-02-01T17:33:23.451Z Sent 30 1 1 32
132392024-02-01T17:33:23.451Z Done 0 29 29 58
132402024-02-01T17:33:23.451Z Skipped 0 0 0 0
132412024-02-01T17:33:23.451Z Error 0 0 0 0
132422024-02-01T17:33:23.451ZLast Flush: 0 1021 1021
132432024-02-01T17:33:23.451ZDownstairs last five completed:
132442024-02-01T17:33:23.451ZUpstairs last five completed: 30 29 28 27 26
132452024-02-01T17:33:23.451Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:22.25353497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1617}
132462024-02-01T17:33:23.451ZWrote out file "/tmp/test_repair_verify.data"
132472024-02-01T17:33:23.451Z
132482024-02-01T17:33:23.451ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
132492024-02-01T17:33:23.451ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132502024-02-01T17:33:23.451Z 0 000-019 120 122 122 186 190 190 F F F <---
132512024-02-01T17:33:23.451Z 1 020-039 110 110 110 167 167 167 F F F
132522024-02-01T17:33:23.452Z 2 040-059 114 114 114 175 175 175 F F F
132532024-02-01T17:33:23.452Z 3 060-079 118 118 118 184 184 184 F T T <---
132542024-02-01T17:33:23.452Z 4 080-099 118 122 122 184 191 191 F T T <---
132552024-02-01T17:33:23.452Z 5 100-119 120 122 122 186 191 191 F F F <---
132562024-02-01T17:33:23.452Z 6 120-139 119 122 122 185 191 191 F T T <---
132572024-02-01T17:33:23.452Z 7 140-159 119 122 122 185 191 191 F F F <---
132582024-02-01T17:33:23.452Z 8 160-179 119 119 119 185 185 185 F F F
132592024-02-01T17:33:23.452Z 9 180-199 120 120 120 186 186 186 F F F
132602024-02-01T17:33:23.452Z 10 200-219 118 118 118 182 182 182 F F F
132612024-02-01T17:33:23.458Z 11 220-239 117 122 122 180 190 190 F F F <---
132622024-02-01T17:33:23.458Z 12 240-259 116 122 122 179 191 191 F F F <---
132632024-02-01T17:33:23.458Z 13 260-279 121 122 122 188 191 191 F F F <---
132642024-02-01T17:33:23.458Z 14 280-299 120 120 120 186 186 186 F F F
132652024-02-01T17:33:23.458Z 15 300-319 110 122 122 167 190 190 F F F <---
132662024-02-01T17:33:23.458Z 16 320-339 110 110 110 167 167 167 F F F
132672024-02-01T17:33:23.458Z 17 340-359 114 114 114 176 176 176 F F F
132682024-02-01T17:33:23.458Z 18 360-379 108 122 122 164 190 190 F T T <---
132692024-02-01T17:33:23.458Z 19 380-399 117 117 117 180 180 180 F F F
132702024-02-01T17:33:23.458Z 20 400-419 117 117 117 180 180 180 F F F
132712024-02-01T17:33:23.460Z 21 420-439 120 120 120 187 187 187 F F F
132722024-02-01T17:33:23.460Z 22 440-459 120 120 120 186 186 186 F F F
132732024-02-01T17:33:23.460Z 23 460-479 120 120 120 186 186 186 F F F
132742024-02-01T17:33:23.460Z 24 480-499 121 121 121 188 188 188 F F F
132752024-02-01T17:33:23.460Z 25 500-519 119 119 119 185 185 185 F F F
132762024-02-01T17:33:23.460Z 26 520-539 107 107 107 161 161 161 F F F
132772024-02-01T17:33:23.461Z 27 540-559 117 117 117 180 180 180 F T T <---
132782024-02-01T17:33:23.461Z 28 560-579 120 120 120 187 187 187 F F F
132792024-02-01T17:33:23.461Z 29 580-599 120 120 120 186 186 186 F F F
132802024-02-01T17:33:23.461ZMax gen: 122, Max flush: 191
132812024-02-01T17:33:23.461ZError: Difference in extent metadata found!
132822024-02-01T17:33:23.461ZA Difference in extent metadata is expected here
132832024-02-01T17:33:23.461ZOn loop 61
132842024-02-01T17:33:23.461Z
132852024-02-01T17:33:23.461ZVerifying data now
132862024-02-01T17:33:24.071ZNo change after verify
132872024-02-01T17:33:24.071ZLoop: 61 Downstairs dump after verify (and repair):
132882024-02-01T17:33:24.093ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
132892024-02-01T17:33:24.093Z 0 000-019 122 122 122 190 190 190 F F F
132902024-02-01T17:33:24.093Z 1 020-039 110 110 110 167 167 167 F F F
132912024-02-01T17:33:24.093Z 2 040-059 114 114 114 175 175 175 F F F
132922024-02-01T17:33:24.093Z 3 060-079 123 123 123 192 192 192 F F F
132932024-02-01T17:33:24.093Z 4 080-099 123 123 123 192 192 192 F F F
132942024-02-01T17:33:24.116Z 5 100-119 122 122 122 191 191 191 F F F
132952024-02-01T17:33:24.116Z 6 120-139 123 123 123 192 192 192 F F F
132962024-02-01T17:33:24.116Z 7 140-159 122 122 122 191 191 191 F F F
132972024-02-01T17:33:24.118Z 8 160-179 119 119 119 185 185 185 F F F
132982024-02-01T17:33:24.118Z 9 180-199 120 120 120 186 186 186 F F F
132992024-02-01T17:33:24.118Z 10 200-219 118 118 118 182 182 182 F F F
133002024-02-01T17:33:24.118Z 11 220-239 122 122 122 190 190 190 F F F
133012024-02-01T17:33:24.118Z 12 240-259 122 122 122 191 191 191 F F F
133022024-02-01T17:33:24.118Z 13 260-279 122 122 122 191 191 191 F F F
133032024-02-01T17:33:24.118Z 14 280-299 120 120 120 186 186 186 F F F
133042024-02-01T17:33:24.118Z 15 300-319 122 122 122 190 190 190 F F F
133052024-02-01T17:33:24.118Z 16 320-339 110 110 110 167 167 167 F F F
133062024-02-01T17:33:24.118Z 17 340-359 114 114 114 176 176 176 F F F
133072024-02-01T17:33:24.118Z 18 360-379 123 123 123 192 192 192 F F F
133082024-02-01T17:33:24.118Z 19 380-399 117 117 117 180 180 180 F F F
133092024-02-01T17:33:24.118Z 20 400-419 117 117 117 180 180 180 F F F
133102024-02-01T17:33:24.118Z 21 420-439 120 120 120 187 187 187 F F F
133112024-02-01T17:33:24.118Z 22 440-459 120 120 120 186 186 186 F F F
133122024-02-01T17:33:24.118Z 23 460-479 120 120 120 186 186 186 F F F
133132024-02-01T17:33:24.118Z 24 480-499 121 121 121 188 188 188 F F F
133142024-02-01T17:33:24.118Z 25 500-519 119 119 119 185 185 185 F F F
133152024-02-01T17:33:24.118Z 26 520-539 107 107 107 161 161 161 F F F
133162024-02-01T17:33:24.118Z 27 540-559 123 123 123 192 192 192 F F F
133172024-02-01T17:33:24.118Z 28 560-579 120 120 120 187 187 187 F F F
133182024-02-01T17:33:24.118Z 29 580-599 120 120 120 186 186 186 F F F
133192024-02-01T17:33:24.118ZMax gen: 123, Max flush: 192
133202024-02-01T17:33:24.119Z
133212024-02-01T17:33:24.119ZBegin loop 62
133222024-02-01T17:33:24.121ZDownstairs to restart: 2
133232024-02-01T17:33:24.142Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.402860751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
133242024-02-01T17:33:24.142Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403369504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
133252024-02-01T17:33:24.142Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403428992Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
133262024-02-01T17:33:24.142Z{"msg":"Crucible fbb0b009-40c2-45a0-80b9-d7d3d81ecb0d has session id: 32fd6df1-30d6-42b6-a76f-6b13de5c7683","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403469427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133272024-02-01T17:33:24.142ZCrucible runtime is spawned
133282024-02-01T17:33:24.142Z{"msg":"Upstairs opts: Upstairs UUID: fbb0b009-40c2-45a0-80b9-d7d3d81ecb0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403577168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133292024-02-01T17:33:24.142Z{"msg":"Crucible stats registered with UUID: fbb0b009-40c2-45a0-80b9-d7d3d81ecb0d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403637803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133302024-02-01T17:33:24.142Z{"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40367881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
133312024-02-01T17:33:24.142Z{"msg":"fbb0b009-40c2-45a0-80b9-d7d3d81ecb0d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40374468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133322024-02-01T17:33:24.142Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403786143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133332024-02-01T17:33:24.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403852988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133342024-02-01T17:33:24.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403897337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133352024-02-01T17:33:24.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.403962214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133362024-02-01T17:33:24.145Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40399526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133372024-02-01T17:33:24.145Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40405074Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133382024-02-01T17:33:24.145Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.404268099Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133392024-02-01T17:33:24.145Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.404348822Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133402024-02-01T17:33:24.145Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40442995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133412024-02-01T17:33:24.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.404478483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133422024-02-01T17:33:24.145Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.404521758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133432024-02-01T17:33:24.146Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.404590096Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"io task","client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133442024-02-01T17:33:24.146Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.405642409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133452024-02-01T17:33:24.147Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.405753707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133462024-02-01T17:33:24.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.405864833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133472024-02-01T17:33:24.147Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.405952166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133482024-02-01T17:33:24.147Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.406258715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133492024-02-01T17:33:24.147Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.406345612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133502024-02-01T17:33:24.147Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.4066037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133512024-02-01T17:33:24.147Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.406799613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133522024-02-01T17:33:24.147Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407441748Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133532024-02-01T17:33:24.147Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407534353Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133542024-02-01T17:33:24.147Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407579427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133552024-02-01T17:33:24.148Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407667091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133562024-02-01T17:33:24.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407743118Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133572024-02-01T17:33:24.148Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.407821776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133582024-02-01T17:33:24.148Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.40789618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133592024-02-01T17:33:24.148Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408386994Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133602024-02-01T17:33:24.148Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408451575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133612024-02-01T17:33:24.148Z{"msg":"[0]R flush_numbers[0..12]: [190, 167, 175, 192, 192, 191, 192, 191, 185, 186, 182, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408480316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133622024-02-01T17:33:24.148Z{"msg":"[0]R generation[0..12]: [122, 110, 114, 123, 123, 122, 123, 122, 119, 120, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408524294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133632024-02-01T17:33:24.151Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408550795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133642024-02-01T17:33:24.151Z{"msg":"[1]R flush_numbers[0..12]: [190, 167, 175, 192, 192, 191, 192, 191, 185, 186, 182, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408593387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133652024-02-01T17:33:24.151Z{"msg":"[1]R generation[0..12]: [122, 110, 114, 123, 123, 122, 123, 122, 119, 120, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408618913Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133662024-02-01T17:33:24.151Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408666774Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133672024-02-01T17:33:24.153Z{"msg":"[2]R flush_numbers[0..12]: [190, 167, 175, 192, 192, 191, 192, 191, 185, 186, 182, 190]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
133682024-02-01T17:33:24.153Z,"time":"2024-02-01T17:33:23.408692404Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133692024-02-01T17:33:24.154Z{"msg":"[2]R generation[0..12]: [122, 110, 114, 123, 123, 122, 123, 122, 119, 120, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408812408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133702024-02-01T17:33:24.154Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408862171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133712024-02-01T17:33:24.154Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408886983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133722024-02-01T17:33:24.154Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408928781Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133732024-02-01T17:33:24.154Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408952476Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133742024-02-01T17:33:24.154Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.408974788Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"":"downstairs","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133752024-02-01T17:33:24.154Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409050151Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133762024-02-01T17:33:24.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409093959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"0","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133772024-02-01T17:33:24.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409118405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"1","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133782024-02-01T17:33:24.155Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409142043Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"client":"2","":"downstairs","session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133792024-02-01T17:33:24.155Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409191217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133802024-02-01T17:33:24.155Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409214525Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133812024-02-01T17:33:24.155Z{"msg":"fbb0b009-40c2-45a0-80b9-d7d3d81ecb0d is now active with session: 32fd6df1-30d6-42b6-a76f-6b13de5c7683","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409236284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133822024-02-01T17:33:24.155Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409279955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625,"session_id":"32fd6df1-30d6-42b6-a76f-6b13de5c7683"}
133832024-02-01T17:33:24.156Z{"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:23.409302578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
133842024-02-01T17:33:24.156ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133852024-02-01T17:33:24.156ZLoading write count information from file "/tmp/test_repair_verify.data"
133862024-02-01T17:33:24.157ZRun Repair workload
133872024-02-01T17:33:24.157Z01/30 Write block 72 len 32768 data: 14 13 14 13 8 6 6 6
133882024-02-01T17:33:24.157Z02/30 Read block 552 len 36864
133892024-02-01T17:33:24.180Z03/30 Read block 197 len 16384
133902024-02-01T17:33:24.180Z04/30 Read block 546 len 12288
133912024-02-01T17:33:24.180Z05/30 Read block 102 len 24576
133922024-02-01T17:33:24.181Z06/30 Flush
133932024-02-01T17:33:24.181Z07/30 Write block 181 len 4096 data: 10
133942024-02-01T17:33:24.181Z08/30 Write block 110 len 36864 data: 7 7 7 8 8 8 8 8 7
133952024-02-01T17:33:24.181Z09/30 Read block 486 len 24576
133962024-02-01T17:33:24.181Z10/30 Write block 170 len 16384 data: 16 17 15 14
133972024-02-01T17:33:24.181Z11/30 Write block 113 len 24576 data: 9 9 9 9 9 8
133982024-02-01T17:33:24.181Z12/30 Write block 9 len 32768 data: 9 11 8 6 5 5 6 6
133992024-02-01T17:33:24.181Z13/30 Flush
134002024-02-01T17:33:24.181Z14/30 Write block 101 len 16384 data: 9 10 8 9
134012024-02-01T17:33:24.182Z15/30 Write block 72 len 28672 data: 15 14 15 14 9 7 7
134022024-02-01T17:33:24.182Z16/30 Write block 570 len 16384 data: 8 5 4 6
134032024-02-01T17:33:24.182Z17/30 Write block 105 len 24576 data: 12 12 13 11 10 8
134042024-02-01T17:33:24.182Z18/30 Write block 161 len 28672 data: 12 12 12 12 9 8 8
134052024-02-01T17:33:24.182Z19/30 Flush
134062024-02-01T17:33:24.183Z20/30 Write block 430 len 24576 data: 8 8 12 15 13 14
134072024-02-01T17:33:24.183Z21/30 Read block 480 len 12288
134082024-02-01T17:33:24.183Z22/30 Read block 579 len 4096
134092024-02-01T17:33:24.184Z23/30 Read block 589 len 16384
134102024-02-01T17:33:24.184Z24/30 Flush
134112024-02-01T17:33:24.184Z25/30 Write block 4 len 20480 data: 9 9 10 9 11
134122024-02-01T17:33:24.184Z26/30 Write block 207 len 40960 data: 14 15 15 11 12 10 10 10 10 5
134132024-02-01T17:33:24.207Z27/30 Write block 449 len 40960 data: 9 7 5 6 6 6 6 10 13 13
134142024-02-01T17:33:24.207Z28/30 Read block 297 len 8192
134152024-02-01T17:33:24.207Z29/30 Write block 549 len 4096 data: 8
134162024-02-01T17:33:24.207Z30/30 Read block 147 len 24576
134172024-02-01T17:33:24.207Z----------------------------------------------------------------
134182024-02-01T17:33:24.207Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30
134192024-02-01T17:33:24.207ZStates: Active Active Active
134202024-02-01T17:33:24.207ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
134212024-02-01T17:33:24.207Z 1 Acked 1000 Write 8 Done Done Done false
134222024-02-01T17:33:24.207Z 2 Acked 1001 Read 9 Done Done Done false
134232024-02-01T17:33:24.207Z 3 Acked 1002 Read 4 Done Done Sent false
134242024-02-01T17:33:24.207Z 4 Acked 1003 Read 3 Done Done Sent false
134252024-02-01T17:33:24.207Z 5 Acked 1004 Read 6 Done Done Sent false
134262024-02-01T17:33:24.207Z 6 Acked 1005 Flush 0 Done Done Sent false
134272024-02-01T17:33:24.207Z 7 Acked 1006 Write 1 Done Done Sent false
134282024-02-01T17:33:24.208Z 8 Acked 1007 Write 9 Done Done Sent false
134292024-02-01T17:33:24.208Z 9 Acked 1008 Read 6 Done Done Sent false
134302024-02-01T17:33:24.208Z 10 Acked 1009 Write 4 Done Done Sent false
134312024-02-01T17:33:24.212Z 11 Acked 1010 Write 6 Done Done Sent false
134322024-02-01T17:33:24.212Z 12 Acked 1011 Write 8 Done Done Sent false
134332024-02-01T17:33:24.212Z 13 Acked 1012 Flush 0 Done Done Sent false
134342024-02-01T17:33:24.212Z 14 Acked 1013 Write 4 Done Done Sent false
134352024-02-01T17:33:24.212Z 15 Acked 1014 Write 7 Done Done Sent false
134362024-02-01T17:33:24.212Z 16 Acked 1015 Write 4 Done Done Sent false
134372024-02-01T17:33:24.212Z 17 Acked 1016 Write 6 Done Done Sent false
134382024-02-01T17:33:24.212Z 18 Acked 1017 Write 7 Done Done Sent false
134392024-02-01T17:33:24.212Z 19 Acked 1018 Flush 0 Done Done Sent false
134402024-02-01T17:33:24.212Z 20 Acked 1019 Write 6 Done Done Sent false
134412024-02-01T17:33:24.212Z 21 Acked 1020 Read 3 Done Done Sent false
134422024-02-01T17:33:24.212Z 22 Acked 1021 Read 1 Done Done Sent false
134432024-02-01T17:33:24.212Z 23 Acked 1022 Read 4 Done Done Sent false
134442024-02-01T17:33:24.212Z 24 Acked 1023 Flush 0 Done Done Sent false
134452024-02-01T17:33:24.212Z 25 Acked 1024 Write 5 Done Done Sent false
134462024-02-01T17:33:24.212Z 26 Acked 1025 Write 10 Done Done Sent false
134472024-02-01T17:33:24.212Z 27 Acked 1026 Write 10 Done Done Sent false
134482024-02-01T17:33:24.212Z 28 Acked 1027 Read 2 Done Done Sent false
134492024-02-01T17:33:24.212Z 29 Acked 1028 Write 1 Done Done Sent false
134502024-02-01T17:33:24.212Z 30 Acked 1029 Read 6 Done Done Sent false
134512024-02-01T17:33:24.212Z STATES DS:0 DS:1 DS:2 TOTAL
134522024-02-01T17:33:24.213Z New 0 0 0 0
134532024-02-01T17:33:24.213Z Sent 0 0 28 28
134542024-02-01T17:33:24.213Z Done 30 30 2 62
134552024-02-01T17:33:24.213Z Skipped 0 0 0 0
134562024-02-01T17:33:24.213Z Error 0 0 0 0
134572024-02-01T17:33:24.213ZLast Flush: 1023 1023 0
134582024-02-01T17:33:24.213ZDownstairs last five completed:
134592024-02-01T17:33:24.213ZUpstairs last five completed: 30 29 28 27 26
134602024-02-01T17:33:24.213Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:23.45717191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1625}
134612024-02-01T17:33:24.213ZWrote out file "/tmp/test_repair_verify.data"
134622024-02-01T17:33:24.213Z
134632024-02-01T17:33:24.214ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
134642024-02-01T17:33:24.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134652024-02-01T17:33:24.224Z 0 000-019 124 124 122 194 194 190 T T F <---
134662024-02-01T17:33:24.224Z 1 020-039 110 110 110 167 167 167 F F F
134672024-02-01T17:33:24.224Z 2 040-059 114 114 114 175 175 175 F F F
134682024-02-01T17:33:24.224Z 3 060-079 124 124 123 195 195 192 F F T <---
134692024-02-01T17:33:24.224Z 4 080-099 123 123 123 192 192 192 F F F
134702024-02-01T17:33:24.224Z 5 100-119 124 124 122 195 195 191 F F F <---
134712024-02-01T17:33:24.224Z 6 120-139 123 123 123 192 192 192 F F F
134722024-02-01T17:33:24.224Z 7 140-159 122 122 122 191 191 191 F F F
134732024-02-01T17:33:24.224Z 8 160-179 124 124 119 195 195 185 F F F <---
134742024-02-01T17:33:24.224Z 9 180-199 124 124 120 194 194 186 F F F <---
134752024-02-01T17:33:24.224Z 10 200-219 118 118 118 182 182 182 T T F <---
134762024-02-01T17:33:24.224Z 11 220-239 122 122 122 190 190 190 F F F
134772024-02-01T17:33:24.224Z 12 240-259 122 122 122 191 191 191 F F F
134782024-02-01T17:33:24.225Z 13 260-279 122 122 122 191 191 191 F F F
134792024-02-01T17:33:24.225Z 14 280-299 120 120 120 186 186 186 F F F
134802024-02-01T17:33:24.225Z 15 300-319 122 122 122 190 190 190 F F F
134812024-02-01T17:33:24.226Z 16 320-339 110 110 110 167 167 167 F F F
134822024-02-01T17:33:24.226Z 17 340-359 114 114 114 176 176 176 F F F
134832024-02-01T17:33:24.226Z 18 360-379 123 123 123 192 192 192 F F F
134842024-02-01T17:33:24.226Z 19 380-399 117 117 117 180 180 180 F F F
134852024-02-01T17:33:24.226Z 20 400-419 117 117 117 180 180 180 F F F
134862024-02-01T17:33:24.226Z 21 420-439 124 124 120 196 196 187 F F F <---
134872024-02-01T17:33:24.226Z 22 440-459 120 120 120 186 186 186 T T F <---
134882024-02-01T17:33:24.226Z 23 460-479 120 120 120 186 186 186 F F F
134892024-02-01T17:33:24.226Z 24 480-499 121 121 121 188 188 188 F F F
134902024-02-01T17:33:24.226Z 25 500-519 119 119 119 185 185 185 F F F
134912024-02-01T17:33:24.226Z 26 520-539 107 107 107 161 161 161 F F F
134922024-02-01T17:33:24.226Z 27 540-559 123 123 123 192 192 192 T T F <---
134932024-02-01T17:33:24.226Z 28 560-579 124 124 120 195 195 187 F F F <---
134942024-02-01T17:33:24.226Z 29 580-599 120 120 120 186 186 186 F F F
134952024-02-01T17:33:24.226ZMax gen: 124, Max flush: 196
134962024-02-01T17:33:24.226ZError: Difference in extent metadata found!
134972024-02-01T17:33:24.226ZA Difference in extent metadata is expected here
134982024-02-01T17:33:24.226ZOn loop 62
134992024-02-01T17:33:24.226Z
135002024-02-01T17:33:24.226ZVerifying data now
135012024-02-01T17:33:29.110ZNo change after verify
135022024-02-01T17:33:29.110ZLoop: 62 Downstairs dump after verify (and repair):
135032024-02-01T17:33:29.133ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135042024-02-01T17:33:29.133Z 0 000-019 125 125 125 197 197 197 F F F
135052024-02-01T17:33:29.133Z 1 020-039 110 110 110 167 167 167 F F F
135062024-02-01T17:33:29.133Z 2 040-059 114 114 114 175 175 175 F F F
135072024-02-01T17:33:29.133Z 3 060-079 124 124 124 195 195 195 F F F
135082024-02-01T17:33:29.133Z 4 080-099 123 123 123 192 192 192 F F F
135092024-02-01T17:33:29.133Z 5 100-119 124 124 124 195 195 195 F F F
135102024-02-01T17:33:29.133Z 6 120-139 123 123 123 192 192 192 F F F
135112024-02-01T17:33:29.133Z 7 140-159 122 122 122 191 191 191 F F F
135122024-02-01T17:33:29.136Z 8 160-179 124 124 124 195 195 195 F F F
135132024-02-01T17:33:29.136Z 9 180-199 124 124 124 194 194 194 F F F
135142024-02-01T17:33:29.137Z 10 200-219 125 125 125 197 197 197 F F F
135152024-02-01T17:33:29.137Z 11 220-239 122 122 122 190 190 190 F F F
135162024-02-01T17:33:29.137Z 12 240-259 122 122 122 191 191 191 F F F
135172024-02-01T17:33:29.137Z 13 260-279 122 122 122 191 191 191 F F F
135182024-02-01T17:33:29.137Z 14 280-299 120 120 120 186 186 186 F F F
135192024-02-01T17:33:29.137Z 15 300-319 122 122 122 190 190 190 F F F
135202024-02-01T17:33:29.137Z 16 320-339 110 110 110 167 167 167 F F F
135212024-02-01T17:33:29.137Z 17 340-359 114 114 114 176 176 176 F F F
135222024-02-01T17:33:29.137Z 18 360-379 123 123 123 192 192 192 F F F
135232024-02-01T17:33:29.137Z 19 380-399 117 117 117 180 180 180 F F F
135242024-02-01T17:33:29.137Z 20 400-419 117 117 117 180 180 180 F F F
135252024-02-01T17:33:29.137Z 21 420-439 124 124 124 196 196 196 F F F
135262024-02-01T17:33:29.137Z 22 440-459 125 125 125 197 197 197 F F F
135272024-02-01T17:33:29.137Z 23 460-479 120 120 120 186 186 186 F F F
135282024-02-01T17:33:29.137Z 24 480-499 121 121 121 188 188 188 F F F
135292024-02-01T17:33:29.137Z 25 500-519 119 119 119 185 185 185 F F F
135302024-02-01T17:33:29.137Z 26 520-539 107 107 107 161 161 161 F F F
135312024-02-01T17:33:29.142Z 27 540-559 125 125 125 197 197 197 F F F
135322024-02-01T17:33:29.142Z 28 560-579 124 124 124 195 195 195 F F F
135332024-02-01T17:33:29.142Z 29 580-599 120 120 120 186 186 186 F F F
135342024-02-01T17:33:29.142ZMax gen: 125, Max flush: 197
135352024-02-01T17:33:29.148Z
135362024-02-01T17:33:29.148ZBegin loop 63
135372024-02-01T17:33:29.148ZDownstairs to restart: 0
135382024-02-01T17:33:29.180Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.447838443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
135392024-02-01T17:33:29.180Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
135402024-02-01T17:33:29.187Z,"time":"2024-02-01T17:33:28.448413676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
135412024-02-01T17:33:29.187Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.44864719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
135422024-02-01T17:33:29.188Z{"msg":"Crucible 0ecac49a-9a84-4693-b114-4b7d3f293c97 has session id: 2cf53532-4489-4fa5-85ad-c7997f276082","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.44874195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135432024-02-01T17:33:29.188Z{"msg":"Upstairs opts: Upstairs UUID: 0ecac49a-9a84-4693-b114-4b7d3f293c97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.448827638Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135442024-02-01T17:33:29.188Z{"msg":"Crucible stats registered with UUID: 0ecac49a-9a84-4693-b114-4b7d3f293c97","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.44893826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135452024-02-01T17:33:29.188Z{"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.448962498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
135462024-02-01T17:33:29.188Z{"msg":"0ecac49a-9a84-4693-b114-4b7d3f293c97 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449040433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135472024-02-01T17:33:29.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449101028Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135482024-02-01T17:33:29.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449166507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135492024-02-01T17:33:29.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449211246Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135502024-02-01T17:33:29.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449410831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135512024-02-01T17:33:29.188Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.44954888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135522024-02-01T17:33:29.188Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449637563Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135532024-02-01T17:33:29.188Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449716552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135542024-02-01T17:33:29.188Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.4497672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135552024-02-01T17:33:29.188Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449846197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135562024-02-01T17:33:29.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449894968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135572024-02-01T17:33:29.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.449935596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135582024-02-01T17:33:29.189Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.450009564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"io task","client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135592024-02-01T17:33:29.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.450873429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135602024-02-01T17:33:29.189Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.450958798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135612024-02-01T17:33:29.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.451004661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135622024-02-01T17:33:29.189Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.451052513Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135632024-02-01T17:33:29.189Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.451980437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135642024-02-01T17:33:29.189Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45207672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135652024-02-01T17:33:29.189Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.452124945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135662024-02-01T17:33:29.190Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.452197085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135672024-02-01T17:33:29.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453013166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135682024-02-01T17:33:29.190Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453077891Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135692024-02-01T17:33:29.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453104938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135702024-02-01T17:33:29.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.4531277Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135712024-02-01T17:33:29.190Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453151037Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135722024-02-01T17:33:29.190Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45318582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135732024-02-01T17:33:29.190Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453251434Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135742024-02-01T17:33:29.190Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453859704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135752024-02-01T17:33:29.190Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453892091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135762024-02-01T17:33:29.193Z{"msg":"[0]R flush_numbers[0..12]: [197, 167, 175, 195, 192, 195, 192, 191, 195, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453938123Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135772024-02-01T17:33:29.193Z{"msg":"[0]R generation[0..12]: [125, 110, 114, 124, 123, 124, 123, 122, 124, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.453994143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135782024-02-01T17:33:29.193Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454028484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135792024-02-01T17:33:29.193Z{"msg":"[1]R flush_numbers[0..12]: [197, 167, 175, 195, 192, 195, 192, 191, 195, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454084647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135802024-02-01T17:33:29.193Z{"msg":"[1]R generation[0..12]: [125, 110, 114, 124, 123, 124, 123, 122, 124, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45415152Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135812024-02-01T17:33:29.193Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
135822024-02-01T17:33:29.193Z30,"time":"2024-02-01T17:33:28.454233828Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135832024-02-01T17:33:29.193Z{"msg":"[2]R flush_numbers[0..12]: [197, 167, 175, 195, 192, 195, 192, 191, 195, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454295875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135842024-02-01T17:33:29.193Z{"msg":"[2]R generation[0..12]: [125, 110, 114, 124, 123, 124, 123, 122, 124, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454329165Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135852024-02-01T17:33:29.193Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454384408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135862024-02-01T17:33:29.195Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454410207Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135872024-02-01T17:33:29.195Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454468316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135882024-02-01T17:33:29.195Z{"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454494206Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135892024-02-01T17:33:29.195Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45455433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"":"downstairs","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135902024-02-01T17:33:29.195Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454581395Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135912024-02-01T17:33:29.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454642125Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"0","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135922024-02-01T17:33:29.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454668505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"1","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135932024-02-01T17:33:29.195Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45473057Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"client":"2","":"downstairs","session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135942024-02-01T17:33:29.195Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454757223Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135952024-02-01T17:33:29.195Zip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135962024-02-01T17:33:29.195Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.45488103Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135972024-02-01T17:33:29.195Z{"msg":"0ecac49a-9a84-4693-b114-4b7d3f293c97 is now active with session: 2cf53532-4489-4fa5-85ad-c7997f276082","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454917164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135982024-02-01T17:33:29.196Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454967342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633,"session_id":"2cf53532-4489-4fa5-85ad-c7997f276082"}
135992024-02-01T17:33:29.196Z{"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:28.454995527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
136002024-02-01T17:33:29.196ZLoading write count information from file "/tmp/test_repair_verify.data"
136012024-02-01T17:33:29.196ZRun Repair workload
136022024-02-01T17:33:29.196Z01/30 Write block 122 len 28672 data: 8 8 9 10 8 8 9
136032024-02-01T17:33:29.196Z02/30 Write block 321 len 28672 data: 10 11 8 9 5 6 8
136042024-02-01T17:33:29.196Z03/30 Read block 317 len 8192
136052024-02-01T17:33:29.196Z04/30 Flush
136062024-02-01T17:33:29.199Z05/30 Write block 534 len 40960 data: 15 14 10 12 9 7 11 15 16 13
136072024-02-01T17:33:29.199Z06/30 Write block 99 len 12288 data: 9 9 10
136082024-02-01T17:33:29.199Z07/30 Flush
136092024-02-01T17:33:29.199Z08/30 Write block 371 len 24576 data: 13 12 11 10 11 13
136102024-02-01T17:33:29.199Z09/30 Read block 238 len 36864
136112024-02-01T17:33:29.199Z10/30 Read block 116 len 16384
136122024-02-01T17:33:29.199Z11/30 Flush
136132024-02-01T17:33:29.199Z12/30 Write block 257 len 36864 data: 9 12 12 10 9 10 12 12 14
136142024-02-01T17:33:29.204Z13/30 Read block 271 len 40960
136152024-02-01T17:33:29.204Z14/30 Write block 303 len 4096 data: 8
136162024-02-01T17:33:29.207Z15/30 Flush
136172024-02-01T17:33:29.207Z16/30 Write block 328 len 24576 data: 7 8 9 12 11 11
136182024-02-01T17:33:29.208Z17/30 Read block 410 len 20480
136192024-02-01T17:33:29.211Z18/30 Read block 140 len 40960
136202024-02-01T17:33:29.211Z19/30 Write block 126 len 24576 data: 9 9 10 8 8 7
136212024-02-01T17:33:29.214Z20/30 Read block 229 len 36864
136222024-02-01T17:33:29.214Z21/30 Read block 239 len 12288
136232024-02-01T17:33:29.217Z22/30 Write block 526 len 16384 data: 7 5 5 5
136242024-02-01T17:33:29.217Z23/30 Read block 6 len 24576
136252024-02-01T17:33:29.221Z24/30 Write block 168 len 8192 data: 11 14
136262024-02-01T17:33:29.221Z25/30 Read block 266 len 28672
136272024-02-01T17:33:29.221Z26/30 Read block 401 len 8192
136282024-02-01T17:33:29.228Z27/30 Write block 285 len 4096 data: 11
136292024-02-01T17:33:29.228Z28/30 Write block 305 len 4096 data: 9
136302024-02-01T17:33:29.228Z29/30 Write block 330 len 20480 data: 10 13 12 12 13
136312024-02-01T17:33:29.228Z30/30 Read block 6 len 16384
136322024-02-01T17:33:29.228Z----------------------------------------------------------------
136332024-02-01T17:33:29.228Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
136342024-02-01T17:33:29.228ZStates: Active Active Active
136352024-02-01T17:33:29.228ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
136362024-02-01T17:33:29.228Z 1 Acked 1000 Write 7 Done Done Done false
136372024-02-01T17:33:29.228Z 2 Acked 1001 Write 7 Sent Done Done false
136382024-02-01T17:33:29.228Z 3 Acked 1002 Read 2 Sent Done Done false
136392024-02-01T17:33:29.229Z 4 Acked 1003 Flush 0 Sent Done Done false
136402024-02-01T17:33:29.229Z 5 Acked 1004 Write 10 Sent Done Done false
136412024-02-01T17:33:29.229Z 6 Acked 1005 Write 3 Sent Done Done false
136422024-02-01T17:33:29.229Z 7 Acked 1006 Flush 0 Sent Done Done false
136432024-02-01T17:33:29.229Z 8 Acked 1007 Write 6 Sent Done Done false
136442024-02-01T17:33:29.229Z 9 Acked 1008 Read 9 Sent Done Done false
136452024-02-01T17:33:29.229Z 10 Acked 1009 Read 4 Sent Done Done false
136462024-02-01T17:33:29.229Z 11 Acked 1010 Flush 0 Sent Done Done false
136472024-02-01T17:33:29.230Z 12 Acked 1011 Write 9 Sent Done Done false
136482024-02-01T17:33:29.230Z 13 Acked 1012 Read 10 Sent Done Done false
136492024-02-01T17:33:29.230Z 14 Acked 1013 Write 1 Sent Done Done false
136502024-02-01T17:33:29.230Z 15 Acked 1014 Flush 0 Sent Done Done false
136512024-02-01T17:33:29.230Z 16 Acked 1015 Write 6 Sent Done Done false
136522024-02-01T17:33:29.230Z 17 Acked 1016 Read 5 Sent Done Done false
136532024-02-01T17:33:29.230Z 18 Acked 1017 Read 10 Sent Done Done false
136542024-02-01T17:33:29.230Z 19 Acked 1018 Write 6 Sent Done Done false
136552024-02-01T17:33:29.230Z 20 Acked 1019 Read 9 Sent Done Done false
136562024-02-01T17:33:29.230Z 21 Acked 1020 Read 3 Sent Done Done false
136572024-02-01T17:33:29.230Z 22 Acked 1021 Write 4 Sent Done Done false
136582024-02-01T17:33:29.230Z 23 Acked 1022 Read 6 Sent Done Done false
136592024-02-01T17:33:29.230Z 24 Acked 1023 Write 2 Sent Done Done false
136602024-02-01T17:33:29.230Z 25 Acked 1024 Read 7 Sent Done Done false
136612024-02-01T17:33:29.230Z 26 Acked 1025 Read 2 Sent Done Done false
136622024-02-01T17:33:29.230Z 27 Acked 1026 Write 1 Sent Done Done false
136632024-02-01T17:33:29.230Z 28 Acked 1027 Write 1 Sent Done Done false
136642024-02-01T17:33:29.230Z 29 Acked 1028 Write 5 Sent Done Done false
136652024-02-01T17:33:29.230Z 30 Acked 1029 Read 4 Sent Done Done false
136662024-02-01T17:33:29.231Z STATES DS:0 DS:1 DS:2 TOTAL
136672024-02-01T17:33:29.231Z New 0 0 0 0
136682024-02-01T17:33:29.231Z Sent 29 0 0 29
136692024-02-01T17:33:29.231Z Done 1 30 30 61
136702024-02-01T17:33:29.231Z Skipped 0 0 0 0
136712024-02-01T17:33:29.231Z Error 0 0 0 0
136722024-02-01T17:33:29.231ZLast Flush: 0 1014 1014
136732024-02-01T17:33:29.231ZDownstairs last five completed:
136742024-02-01T17:33:29.232ZUpstairs last five completed: 30 29 28 27 26
136752024-02-01T17:33:29.232Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:28.504858923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1633}
136762024-02-01T17:33:29.232ZWrote out file "/tmp/test_repair_verify.data"
136772024-02-01T17:33:29.236Z
136782024-02-01T17:33:29.241ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136792024-02-01T17:33:30.257ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136802024-02-01T17:33:30.257Z 0 000-019 125 125 125 197 197 197 F F F
136812024-02-01T17:33:30.286Z 1 020-039 110 110 110 167 167 167 F F F
136822024-02-01T17:33:30.286Z 2 040-059 114 114 114 175 175 175 F F F
136832024-02-01T17:33:30.286Z 3 060-079 124 124 124 195 195 195 F F F
136842024-02-01T17:33:30.286Z 4 080-099 123 126 126 192 199 199 F F F <---
136852024-02-01T17:33:30.286Z 5 100-119 124 126 126 195 199 199 F F F <---
136862024-02-01T17:33:30.287Z 6 120-139 123 126 126 192 198 198 T T T <---
136872024-02-01T17:33:30.287Z 7 140-159 122 122 122 191 191 191 F F F
136882024-02-01T17:33:30.287Z 8 160-179 124 124 124 195 195 195 F T T <---
136892024-02-01T17:33:30.287Z 9 180-199 124 124 124 194 194 194 F F F
136902024-02-01T17:33:30.287Z 10 200-219 125 125 125 197 197 197 F F F
136912024-02-01T17:33:30.287Z 11 220-239 122 122 122 190 190 190 F F F
136922024-02-01T17:33:30.287Z 12 240-259 122 126 126 191 201 201 F F F <---
136932024-02-01T17:33:30.287Z 13 260-279 122 126 126 191 201 201 F F F <---
136942024-02-01T17:33:30.287Z 14 280-299 120 120 120 186 186 186 F T T <---
136952024-02-01T17:33:30.287Z 15 300-319 122 126 126 190 201 201 F T T <---
136962024-02-01T17:33:30.287Z 16 320-339 110 126 126 167 198 198 F T T <---
136972024-02-01T17:33:30.287Z 17 340-359 114 114 114 176 176 176 F F F
136982024-02-01T17:33:30.287Z 18 360-379 123 126 126 192 200 200 F F F <---
136992024-02-01T17:33:30.287Z 19 380-399 117 117 117 180 180 180 F F F
137002024-02-01T17:33:30.287Z 20 400-419 117 117 117 180 180 180 F F F
137012024-02-01T17:33:30.288Z 21 420-439 124 124 124 196 196 196 F F F
137022024-02-01T17:33:30.288Z 22 440-459 125 125 125 197 197 197 F F F
137032024-02-01T17:33:30.288Z 23 460-479 120 120 120 186 186 186 F F F
137042024-02-01T17:33:30.288Z 24 480-499 121 121 121 188 188 188 F F F
137052024-02-01T17:33:30.288Z 25 500-519 119 119 119 185 185 185 F F F
137062024-02-01T17:33:30.288Z 26 520-539 107 126 126 161 199 199 F T T <---
137072024-02-01T17:33:30.288Z 27 540-559 125 126 126 197 199 199 F F F <---
137082024-02-01T17:33:30.288Z 28 560-579 124 124 124 195 195 195 F F F
137092024-02-01T17:33:30.288Z 29 580-599 120 120 120 186 186 186 F F F
137102024-02-01T17:33:30.288ZMax gen: 126, Max flush: 201
137112024-02-01T17:33:30.288ZError: Difference in extent metadata found!
137122024-02-01T17:33:30.288ZA Difference in extent metadata is expected here
137132024-02-01T17:33:30.288ZOn loop 63
137142024-02-01T17:33:30.288Z
137152024-02-01T17:33:30.288ZVerifying data now
137162024-02-01T17:33:34.310ZNo change after verify
137172024-02-01T17:33:34.310ZLoop: 63 Downstairs dump after verify (and repair):
137182024-02-01T17:33:34.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
137192024-02-01T17:33:34.342Z 0 000-019 125 125 125 197 197 197 F F F
137202024-02-01T17:33:34.369Z 1 020-039 110 110 110 167 167 167 F F F
137212024-02-01T17:33:34.369Z 2 040-059 114 114 114 175 175 175 F F F
137222024-02-01T17:33:34.369Z 3 060-079 124 124 124 195 195 195 F F F
137232024-02-01T17:33:34.370Z 4 080-099 126 126 126 199 199 199 F F F
137242024-02-01T17:33:34.370Z 5 100-119 126 126 126 199 199 199 F F F
137252024-02-01T17:33:34.371Z 6 120-139 127 127 127 202 202 202 F F F
137262024-02-01T17:33:34.371Z 7 140-159 122 122 122 191 191 191 F F F
137272024-02-01T17:33:34.371Z 8 160-179 127 127 127 202 202 202 F F F
137282024-02-01T17:33:34.371Z 9 180-199 124 124 124 194 194 194 F F F
137292024-02-01T17:33:34.371Z 10 200-219 125 125 125 197 197 197 F F F
137302024-02-01T17:33:34.372Z 11 220-239 122 122 122 190 190 190 F F F
137312024-02-01T17:33:34.372Z 12 240-259 126 126 126 201 201 201 F F F
137322024-02-01T17:33:34.372Z 13 260-279 126 126 126 201 201 201 F F F
137332024-02-01T17:33:34.372Z 14 280-299 127 127 127 202 202 202 F F F
137342024-02-01T17:33:34.372Z 15 300-319 127 127 127 202 202 202 F F F
137352024-02-01T17:33:34.372Z 16 320-339 127 127 127 202 202 202 F F F
137362024-02-01T17:33:34.373Z 17 340-359 114 114 114 176 176 176 F F F
137372024-02-01T17:33:34.373Z 18 360-379 126 126 126 200 200 200 F F F
137382024-02-01T17:33:34.373Z 19 380-399 117 117 117 180 180 180 F F F
137392024-02-01T17:33:34.373Z 20 400-419 117 117 117 180 180 180 F F F
137402024-02-01T17:33:34.373Z 21 420-439 124 124 124 196 196 196 F F F
137412024-02-01T17:33:34.373Z 22 440-459 125 125 125 197 197 197 F F F
137422024-02-01T17:33:34.374Z 23 460-479 120 120 120 186 186 186 F F F
137432024-02-01T17:33:34.374Z 24 480-499 121 121 121 188 188 188 F F F
137442024-02-01T17:33:34.374Z 25 500-519 119 119 119 185 185 185 F F F
137452024-02-01T17:33:34.374Z 26 520-539 127 127 127 202 202 202 F F F
137462024-02-01T17:33:34.374Z 27 540-559 126 126 126 199 199 199 F F F
137472024-02-01T17:33:34.374Z 28 560-579 124 124 124 195 195 195 F F F
137482024-02-01T17:33:34.375Z 29 580-599 120 120 120 186 186 186 F F F
137492024-02-01T17:33:34.375ZMax gen: 127, Max flush: 202
137502024-02-01T17:33:34.375Z
137512024-02-01T17:33:34.375ZBegin loop 64
137522024-02-01T17:33:34.375ZDownstairs to restart: 0
137532024-02-01T17:33:34.405Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.643382374Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
137542024-02-01T17:33:34.405Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.643970654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
137552024-02-01T17:33:34.405Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644007195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
137562024-02-01T17:33:34.406Z{"msg":"Crucible 25f4801e-e692-49de-8bc2-d095960ac1d2 has session id: 198400f6-f490-48e4-ba4f-9f5764a11bbc","v":0,"name":"crucible","level":30Crucible runtime is spawned
137572024-02-01T17:33:34.406Z,"time":"2024-02-01T17:33:33.64407712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137582024-02-01T17:33:34.406Z{"msg":"Upstairs opts: Upstairs UUID: 25f4801e-e692-49de-8bc2-d095960ac1d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64418388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137592024-02-01T17:33:34.406Z{"msg":"Crucible stats registered with UUID: 25f4801e-e692-49de-8bc2-d095960ac1d2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644271683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137602024-02-01T17:33:34.406Z{"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644349478Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
137612024-02-01T17:33:34.406Z{"msg":"25f4801e-e692-49de-8bc2-d095960ac1d2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644420804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137622024-02-01T17:33:34.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644495806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137632024-02-01T17:33:34.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644551097Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137642024-02-01T17:33:34.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644621021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137652024-02-01T17:33:34.406Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644664044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137662024-02-01T17:33:34.406Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644705856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137672024-02-01T17:33:34.407Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644748488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137682024-02-01T17:33:34.407Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.644891972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137692024-02-01T17:33:34.408Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.645085994Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137702024-02-01T17:33:34.408Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.645141632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137712024-02-01T17:33:34.408Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.645225943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137722024-02-01T17:33:34.408Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.645274652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137732024-02-01T17:33:34.408Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.645321064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"io task","client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137742024-02-01T17:33:34.408Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.646286612Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137752024-02-01T17:33:34.409Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64634816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137762024-02-01T17:33:34.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.646374806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137772024-02-01T17:33:34.409Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.646397528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137782024-02-01T17:33:34.409Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.646452383Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137792024-02-01T17:33:34.409Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64647703Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137802024-02-01T17:33:34.409Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.647354284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137812024-02-01T17:33:34.409Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.647631662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137822024-02-01T17:33:34.410Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64769758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137832024-02-01T17:33:34.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648132472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137842024-02-01T17:33:34.410Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648206836Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137852024-02-01T17:33:34.410Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64827132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137862024-02-01T17:33:34.410Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648632737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137872024-02-01T17:33:34.410Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648740728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137882024-02-01T17:33:34.410Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648805009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137892024-02-01T17:33:34.412Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648830505Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137902024-02-01T17:33:34.412Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648872116Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137912024-02-01T17:33:34.412Z{"msg":"[0]R flush_numbers[0..12]: [197, 167, 175, 195, 199, 199, 202, 191, 202, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648898358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137922024-02-01T17:33:34.412Z{"msg":"[0]R generation[0..12]: [125, 110, 114, 124, 126, 126, 127, 122, 127, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648942009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137932024-02-01T17:33:34.412Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0Wait for a query_work_queue command to finish before sending IO
137942024-02-01T17:33:34.412Z,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.648972459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137952024-02-01T17:33:34.412Z{"msg":"[1]R flush_numbers[0..12]: [197, 167, 175, 195, 199, 199, 202, 191, 202, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64901926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137962024-02-01T17:33:34.412Z{"msg":"[1]R generation[0..12]: [125, 110, 114, 124, 126, 126, 127, 122, 127, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649045218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137972024-02-01T17:33:34.413Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649069099Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137982024-02-01T17:33:34.413Z{"msg":"[2]R flush_numbers[0..12]: [197, 167, 175, 195, 199, 199, 202, 191, 202, 194, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649110659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
137992024-02-01T17:33:34.413Z{"msg":"[2]R generation[0..12]: [125, 110, 114, 124, 126, 126, 127, 122, 127, 124, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649136174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138002024-02-01T17:33:34.414Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649242674Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138012024-02-01T17:33:34.414Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649288316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138022024-02-01T17:33:34.414Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649312462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138032024-02-01T17:33:34.414Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649353591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138042024-02-01T17:33:34.414Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649377206Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"":"downstairs","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138052024-02-01T17:33:34.414Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649418064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138062024-02-01T17:33:34.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649440112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"0","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138072024-02-01T17:33:34.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649481256Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"1","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138082024-02-01T17:33:34.414Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649505128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"client":"2","":"downstairs","session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138092024-02-01T17:33:34.415Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649545523Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138102024-02-01T17:33:34.415Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649567926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138112024-02-01T17:33:34.415Z{"msg":"25f4801e-e692-49de-8bc2-d095960ac1d2 is now active with session: 198400f6-f490-48e4-ba4f-9f5764a11bbc","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.64960813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138122024-02-01T17:33:34.415Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649630219Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641,"session_id":"198400f6-f490-48e4-ba4f-9f5764a11bbc"}
138132024-02-01T17:33:34.415ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
138142024-02-01T17:33:34.415Z{"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:33.649673838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
138152024-02-01T17:33:34.415ZLoading write count information from file "/tmp/test_repair_verify.data"
138162024-02-01T17:33:34.415ZRun Repair workload
138172024-02-01T17:33:34.415Z01/30 Write block 103 len 24576 data: 9 10 13 13 14 12
138182024-02-01T17:33:34.415Z02/30 Write block 30 len 36864 data: 7 7 8 7 7 8 10 11 12
138192024-02-01T17:33:34.416Z03/30 Write block 415 len 12288 data: 11 14 8
138202024-02-01T17:33:34.416Z04/30 Read block 411 len 36864
138212024-02-01T17:33:34.416Z05/30 Write block 342 len 4096 data: 8
138222024-02-01T17:33:34.416Z06/30 Flush
138232024-02-01T17:33:34.439Z07/30 Write block 94 len 8192 data: 7 6
138242024-02-01T17:33:34.440Z08/30 Read block 42 len 20480
138252024-02-01T17:33:34.440Z09/30 Write block 438 len 8192 data: 11 11
138262024-02-01T17:33:34.440Z10/30 Write block 256 len 8192 data: 9 10
138272024-02-01T17:33:34.440Z11/30 Read block 457 len 36864
138282024-02-01T17:33:34.441Z12/30 Read block 163 len 16384
138292024-02-01T17:33:34.441Z13/30 Read block 103 len 32768
138302024-02-01T17:33:34.441Z14/30 Flush
138312024-02-01T17:33:34.441Z15/30 Write block 54 len 40960 data: 12 10 9 9 6 8 9 8 8 9
138322024-02-01T17:33:34.441Z16/30 Read block 329 len 24576
138332024-02-01T17:33:34.441Z17/30 Write block 435 len 28672 data: 15 13 14 12 12 11 11
138342024-02-01T17:33:34.442Z18/30 Read block 219 len 36864
138352024-02-01T17:33:34.442Z19/30 Write block 575 len 4096 data: 9
138362024-02-01T17:33:34.442Z20/30 Read block 37 len 16384
138372024-02-01T17:33:34.443Z21/30 Write block 306 len 8192 data: 6 5
138382024-02-01T17:33:34.443Z22/30 Write block 169 len 16384 data: 15 17 18 16
138392024-02-01T17:33:34.443Z23/30 Write block 129 len 28672 data: 9 9 8 5 9 12 13
138402024-02-01T17:33:34.444Z24/30 Read block 112 len 16384
138412024-02-01T17:33:34.444Z25/30 Read block 160 len 40960
138422024-02-01T17:33:34.444Z26/30 Read block 116 len 16384
138432024-02-01T17:33:34.452Z27/30 Write block 407 len 12288 data: 10 12 9
138442024-02-01T17:33:34.452Z28/30 Write block 175 len 24576 data: 12 15 13 11 11 9
138452024-02-01T17:33:34.452Z29/30 Read block 260 len 8192
138462024-02-01T17:33:34.453Z30/30 Write block 565 len 32768 data: 8 7 6 6 7 9 6 5
138472024-02-01T17:33:34.453Z----------------------------------------------------------------
138482024-02-01T17:33:34.453Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
138492024-02-01T17:33:34.454ZStates: Active Active Active
138502024-02-01T17:33:34.454ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138512024-02-01T17:33:34.456Z 1 Acked 1000 Write 6 Sent Done Done false
138522024-02-01T17:33:34.456Z 2 Acked 1001 Write 9 Sent Done Done false
138532024-02-01T17:33:34.456Z 3 Acked 1002 Write 3 Sent Done Done false
138542024-02-01T17:33:34.456Z 4 Acked 1003 Read 9 Sent Done Done false
138552024-02-01T17:33:34.457Z 5 Acked 1004 Write 1 Sent Done Done false
138562024-02-01T17:33:34.457Z 6 Acked 1005 Flush 0 Sent Done Done false
138572024-02-01T17:33:34.457Z 7 Acked 1006 Write 2 Sent Done Done false
138582024-02-01T17:33:34.458Z 8 Acked 1007 Read 5 Sent Done Done false
138592024-02-01T17:33:34.458Z 9 Acked 1008 Write 2 Sent Done Done false
138602024-02-01T17:33:34.458Z 10 Acked 1009 Write 2 Sent Done Done false
138612024-02-01T17:33:34.458Z 11 Acked 1010 Read 9 Sent Done Done false
138622024-02-01T17:33:34.458Z 12 Acked 1011 Read 4 Sent Done Done false
138632024-02-01T17:33:34.459Z 13 Acked 1012 Read 8 Sent Done Done false
138642024-02-01T17:33:34.459Z 14 Acked 1013 Flush 0 Sent Done Done false
138652024-02-01T17:33:34.459Z 15 Acked 1014 Write 10 Sent Done Done false
138662024-02-01T17:33:34.459Z 16 Acked 1015 Read 6 Sent Done Done false
138672024-02-01T17:33:34.459Z 17 Acked 1016 Write 7 Sent Done Done false
138682024-02-01T17:33:34.460Z 18 Acked 1017 Read 9 Sent Done Done false
138692024-02-01T17:33:34.460Z 19 Acked 1018 Write 1 Sent Done Done false
138702024-02-01T17:33:34.460Z 20 Acked 1019 Read 4 Sent Done Done false
138712024-02-01T17:33:34.460Z 21 Acked 1020 Write 2 Sent Done Done false
138722024-02-01T17:33:34.460Z 22 Acked 1021 Write 4 Sent Done Done false
138732024-02-01T17:33:34.461Z 23 Acked 1022 Write 7 Sent Done Done false
138742024-02-01T17:33:34.461Z 24 Acked 1023 Read 4 Sent Done Done false
138752024-02-01T17:33:34.461Z 25 Acked 1024 Read 10 Sent Done Done false
138762024-02-01T17:33:34.461Z 26 Acked 1025 Read 4 Sent Done Done false
138772024-02-01T17:33:34.461Z 27 Acked 1026 Write 3 Sent Done Done false
138782024-02-01T17:33:34.462Z 28 Acked 1027 Write 6 Sent Done Done false
138792024-02-01T17:33:34.462Z 29 Acked 1028 Read 2 Sent Done Done false
138802024-02-01T17:33:34.462Z 30 Acked 1029 Write 8 Sent Sent Sent false
138812024-02-01T17:33:34.462Z STATES DS:0 DS:1 DS:2 TOTAL
138822024-02-01T17:33:34.462Z New 0 0 0 0
138832024-02-01T17:33:34.463Z Sent 30 1 1 32
138842024-02-01T17:33:34.463Z Done 0 29 29 58
138852024-02-01T17:33:34.463Z Skipped 0 0 0 0
138862024-02-01T17:33:34.463Z Error 0 0 0 0
138872024-02-01T17:33:34.463ZLast Flush: 0 1013 1013
138882024-02-01T17:33:34.463ZDownstairs last five completed:
138892024-02-01T17:33:34.464ZUpstairs last five completed: 30 29 28 27 26
138902024-02-01T17:33:34.464Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:33.695929496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1641}
138912024-02-01T17:33:34.464ZWrote out file "/tmp/test_repair_verify.data"
138922024-02-01T17:33:34.478Z
138932024-02-01T17:33:34.479ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138942024-02-01T17:33:34.479ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138952024-02-01T17:33:34.479Z 0 000-019 125 125 125 197 197 197 F F F
138962024-02-01T17:33:34.479Z 1 020-039 110 128 128 167 203 203 F F F <---
138972024-02-01T17:33:34.479Z 2 040-059 114 114 114 175 175 175 F T T <---
138982024-02-01T17:33:34.479Z 3 060-079 124 124 124 195 195 195 F T T <---
138992024-02-01T17:33:34.479Z 4 080-099 126 128 128 199 204 204 F F F <---
139002024-02-01T17:33:34.480Z 5 100-119 126 128 128 199 203 203 F F F <---
139012024-02-01T17:33:34.480Z 6 120-139 127 127 127 202 202 202 F T T <---
139022024-02-01T17:33:34.480Z 7 140-159 122 122 122 191 191 191 F F F
139032024-02-01T17:33:34.480Z 8 160-179 127 127 127 202 202 202 F T T <---
139042024-02-01T17:33:34.480Z 9 180-199 124 124 124 194 194 194 F T T <---
139052024-02-01T17:33:34.480Z 10 200-219 125 125 125 197 197 197 F F F
139062024-02-01T17:33:34.480Z 11 220-239 122 122 122 190 190 190 F F F
139072024-02-01T17:33:34.481Z 12 240-259 126 128 128 201 204 204 F F F <---
139082024-02-01T17:33:34.481Z 13 260-279 126 126 126 201 201 201 F F F
139092024-02-01T17:33:34.481Z 14 280-299 127 127 127 202 202 202 F F F
139102024-02-01T17:33:34.481Z 15 300-319 127 127 127 202 202 202 F T T <---
139112024-02-01T17:33:34.481Z 16 320-339 127 127 127 202 202 202 F F F
139122024-02-01T17:33:34.481Z 17 340-359 114 128 128 176 203 203 F F F <---
139132024-02-01T17:33:34.482Z 18 360-379 126 126 126 200 200 200 F F F
139142024-02-01T17:33:34.482Z 19 380-399 117 117 117 180 180 180 F F F
139152024-02-01T17:33:34.482Z 20 400-419 117 128 128 180 203 203 F T T <---
139162024-02-01T17:33:34.482Z 21 420-439 124 128 128 196 204 204 F T T <---
139172024-02-01T17:33:34.482Z 22 440-459 125 125 125 197 197 197 F T T <---
139182024-02-01T17:33:34.482Z 23 460-479 120 120 120 186 186 186 F F F
139192024-02-01T17:33:34.482Z 24 480-499 121 121 121 188 188 188 F F F
139202024-02-01T17:33:34.482Z 25 500-519 119 119 119 185 185 185 F F F
139212024-02-01T17:33:34.483Z 26 520-539 127 127 127 202 202 202 F F F
139222024-02-01T17:33:34.483Z 27 540-559 126 126 126 199 199 199 F F F
139232024-02-01T17:33:34.483Z 28 560-579 124 124 124 195 195 195 F T T <---
139242024-02-01T17:33:34.483Z 29 580-599 120 120 120 186 186 186 F F F
139252024-02-01T17:33:34.483ZMax gen: 128, Max flush: 204
139262024-02-01T17:33:34.483ZError: Difference in extent metadata found!
139272024-02-01T17:33:34.483ZA Difference in extent metadata is expected here
139282024-02-01T17:33:34.483ZOn loop 64
139292024-02-01T17:33:34.483Z
139302024-02-01T17:33:34.484ZVerifying data now
139312024-02-01T17:33:39.744ZNo change after verify
139322024-02-01T17:33:39.744ZLoop: 64 Downstairs dump after verify (and repair):
139332024-02-01T17:33:39.767ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139342024-02-01T17:33:39.767Z 0 000-019 125 125 125 197 197 197 F F F
139352024-02-01T17:33:39.767Z 1 020-039 128 128 128 203 203 203 F F F
139362024-02-01T17:33:39.767Z 2 040-059 129 129 129 205 205 205 F F F
139372024-02-01T17:33:39.779Z 3 060-079 129 129 129 205 205 205 F F F
139382024-02-01T17:33:39.779Z 4 080-099 128 128 128 204 204 204 F F F
139392024-02-01T17:33:39.779Z 5 100-119 128 128 128 203 203 203 F F F
139402024-02-01T17:33:39.779Z 6 120-139 129 129 129 205 205 205 F F F
139412024-02-01T17:33:39.781Z 7 140-159 122 122 122 191 191 191 F F F
139422024-02-01T17:33:39.781Z 8 160-179 129 129 129 205 205 205 F F F
139432024-02-01T17:33:39.781Z 9 180-199 129 129 129 205 205 205 F F F
139442024-02-01T17:33:39.781Z 10 200-219 125 125 125 197 197 197 F F F
139452024-02-01T17:33:39.795Z 11 220-239 122 122 122 190 190 190 F F F
139462024-02-01T17:33:39.795Z 12 240-259 128 128 128 204 204 204 F F F
139472024-02-01T17:33:39.795Z 13 260-279 126 126 126 201 201 201 F F F
139482024-02-01T17:33:39.795Z 14 280-299 127 127 127 202 202 202 F F F
139492024-02-01T17:33:39.795Z 15 300-319 129 129 129 205 205 205 F F F
139502024-02-01T17:33:39.795Z 16 320-339 127 127 127 202 202 202 F F F
139512024-02-01T17:33:39.795Z 17 340-359 128 128 128 203 203 203 F F F
139522024-02-01T17:33:39.795Z 18 360-379 126 126 126 200 200 200 F F F
139532024-02-01T17:33:39.795Z 19 380-399 117 117 117 180 180 180 F F F
139542024-02-01T17:33:39.795Z 20 400-419 129 129 129 205 205 205 F F F
139552024-02-01T17:33:39.795Z 21 420-439 129 129 129 205 205 205 F F F
139562024-02-01T17:33:39.795Z 22 440-459 129 129 129 205 205 205 F F F
139572024-02-01T17:33:39.795Z 23 460-479 120 120 120 186 186 186 F F F
139582024-02-01T17:33:39.795Z 24 480-499 121 121 121 188 188 188 F F F
139592024-02-01T17:33:39.795Z 25 500-519 119 119 119 185 185 185 F F F
139602024-02-01T17:33:39.795Z 26 520-539 127 127 127 202 202 202 F F F
139612024-02-01T17:33:39.795Z 27 540-559 126 126 126 199 199 199 F F F
139622024-02-01T17:33:39.796Z 28 560-579 129 129 129 205 205 205 F F F
139632024-02-01T17:33:39.796Z 29 580-599 120 120 120 186 186 186 F F F
139642024-02-01T17:33:39.796ZMax gen: 129, Max flush: 205
139652024-02-01T17:33:39.796Z
139662024-02-01T17:33:39.796ZBegin loop 65
139672024-02-01T17:33:39.796ZDownstairs to restart: 2
139682024-02-01T17:33:39.801Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.076253516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
139692024-02-01T17:33:39.801Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.076697558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
139702024-02-01T17:33:39.801Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.076768307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
139712024-02-01T17:33:39.801Z{"msg":"Crucible 11ffcc5c-8bf4-45c5-bae2-e2e10d1ba9c3 has session id: 1c3ae214-2c7f-41ec-b13e-9523cdc80c81","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.076860014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139722024-02-01T17:33:39.801Z{"msg":"Upstairs opts: Upstairs UUID: 11ffcc5c-8bf4-45c5-bae2-e2e10d1ba9c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.076915415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81Crucible runtime is spawned
139732024-02-01T17:33:39.801Z"}
139742024-02-01T17:33:39.802Z{"msg":"Crucible stats registered with UUID: 11ffcc5c-8bf4-45c5-bae2-e2e10d1ba9c3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077035727Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139752024-02-01T17:33:39.802Z{"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077085227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
139762024-02-01T17:33:39.802Z{"msg":"11ffcc5c-8bf4-45c5-bae2-e2e10d1ba9c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077296479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139772024-02-01T17:33:39.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077366347Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139782024-02-01T17:33:39.802Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077434179Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139792024-02-01T17:33:39.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.07746251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139802024-02-01T17:33:39.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077526434Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139812024-02-01T17:33:39.809Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077561364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139822024-02-01T17:33:39.809Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077604564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139832024-02-01T17:33:39.809Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077676133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139842024-02-01T17:33:39.809Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077764401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139852024-02-01T17:33:39.809Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.077943654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139862024-02-01T17:33:39.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.078147021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139872024-02-01T17:33:39.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.078259791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139882024-02-01T17:33:39.809Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.078306233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"io task","client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139892024-02-01T17:33:39.814Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.079170845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139902024-02-01T17:33:39.815Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.079237153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139912024-02-01T17:33:39.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.07926433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139922024-02-01T17:33:39.815Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.079309009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139932024-02-01T17:33:39.815Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.079333686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139942024-02-01T17:33:39.816Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.079357026Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139952024-02-01T17:33:39.817Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.080143025Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139962024-02-01T17:33:39.817Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.080457456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139972024-02-01T17:33:39.817Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.080518496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139982024-02-01T17:33:39.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.08104863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
139992024-02-01T17:33:39.817Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081131618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140002024-02-01T17:33:39.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081177497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140012024-02-01T17:33:39.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081499633Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140022024-02-01T17:33:39.817Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.08155868Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140032024-02-01T17:33:39.817Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081585152Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140042024-02-01T17:33:39.817Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081625493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140052024-02-01T17:33:39.817Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081650154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140062024-02-01T17:33:39.817Z{"msg":"[0]R flush_numbers[0..12]: [197, 203, 205, 205, 204, 203, 205, 191, 205, 205, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081691862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140072024-02-01T17:33:39.817Z{"msg":"[0]R generation[0..12]: [125, 128, 129, 129, 128, 128, 129, 122, 129, 129, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081717316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140082024-02-01T17:33:39.817Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081759298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140092024-02-01T17:33:39.817Z{"msg":"[1]R flush_numbers[0..12]: [197, 203, 205, 205, 204, 203, 205, 191, 205, 205, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081784491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140102024-02-01T17:33:39.817Z{"msg":"[1]R generation[0..12]: [125, 128, 129, 129, 128, 128, 129, 122, 129, 129, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081826001Z","hostnameWait for a query_work_queue command to finish before sending IO
140112024-02-01T17:33:39.817Z":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140122024-02-01T17:33:39.817Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.08185614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140132024-02-01T17:33:39.817Z{"msg":"[2]R flush_numbers[0..12]: [197, 203, 205, 205, 204, 203, 205, 191, 205, 205, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081881757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140142024-02-01T17:33:39.817Z{"msg":"[2]R generation[0..12]: [125, 128, 129, 129, 128, 128, 129, 122, 129, 129, 125, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081905084Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140152024-02-01T17:33:39.817Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081928978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140162024-02-01T17:33:39.817Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081950624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140172024-02-01T17:33:39.817Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.081990936Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140182024-02-01T17:33:39.817Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.08201502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140192024-02-01T17:33:39.818Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082038769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"":"downstairs","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140202024-02-01T17:33:39.821Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082062723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140212024-02-01T17:33:39.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082083415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"0","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140222024-02-01T17:33:39.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082105083Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"1","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140232024-02-01T17:33:39.821Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082126528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"client":"2","":"downstairs","session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140242024-02-01T17:33:39.821Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082166541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140252024-02-01T17:33:39.821Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082189547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140262024-02-01T17:33:39.821Z{"msg":"11ffcc5c-8bf4-45c5-bae2-e2e10d1ba9c3 is now active with session: 1c3ae214-2c7f-41ec-b13e-9523cdc80c81","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082213229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140272024-02-01T17:33:39.821Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082233484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649,"session_id":"1c3ae214-2c7f-41ec-b13e-9523cdc80c81"}
140282024-02-01T17:33:39.821Z{"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:39.082254105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
140292024-02-01T17:33:39.821ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140302024-02-01T17:33:39.821ZLoading write count information from file "/tmp/test_repair_verify.data"
140312024-02-01T17:33:39.821ZRun Repair workload
140322024-02-01T17:33:39.821Z01/30 Flush
140332024-02-01T17:33:39.821Z02/30 Write block 284 len 20480 data: 8 12 12 10 11
140342024-02-01T17:33:39.821Z03/30 Write block 106 len 24576 data: 14 15 13 11 9 8
140352024-02-01T17:33:39.822Z04/30 Write block 312 len 36864 data: 9 12 10 12 12 13 11 11 12
140362024-02-01T17:33:39.822Z05/30 Read block 477 len 32768
140372024-02-01T17:33:39.826Z06/30 Read block 363 len 36864
140382024-02-01T17:33:39.827Z07/30 Read block 176 len 16384
140392024-02-01T17:33:39.827Z08/30 Read block 466 len 16384
140402024-02-01T17:33:39.827Z09/30 Read block 140 len 4096
140412024-02-01T17:33:39.827Z10/30 Write block 181 len 28672 data: 11 13 15 14 15 13 13
140422024-02-01T17:33:39.827Z11/30 Write block 138 len 32768 data: 16 13 13 12 10 10 9 8
140432024-02-01T17:33:39.827Z12/30 Write block 348 len 28672 data: 11 9 7 8 11 9 15
140442024-02-01T17:33:39.827Z13/30 Read block 405 len 12288
140452024-02-01T17:33:39.828Z14/30 Read block 449 len 28672
140462024-02-01T17:33:39.844Z15/30 Write block 381 len 12288 data: 13 9 9
140472024-02-01T17:33:39.845Z16/30 Write block 37 len 40960 data: 12 13 12 8 11 10 12 13 12 12
140482024-02-01T17:33:39.845Z17/30 Read block 484 len 28672
140492024-02-01T17:33:39.845Z18/30 Write block 545 len 16384 data: 12 13 13 9
140502024-02-01T17:33:39.845Z19/30 Read block 296 len 12288
140512024-02-01T17:33:39.845Z20/30 Read block 348 len 12288
140522024-02-01T17:33:39.858Z21/30 Read block 120 len 4096
140532024-02-01T17:33:39.858Z22/30 Write block 274 len 16384 data: 7 9 13 13
140542024-02-01T17:33:39.858Z23/30 Write block 58 len 20480 data: 7 9 10 9 9
140552024-02-01T17:33:39.858Z24/30 Read block 486 len 32768
140562024-02-01T17:33:39.858Z25/30 Read block 203 len 24576
140572024-02-01T17:33:39.858Z26/30 Write block 20 len 8192 data: 11 10
140582024-02-01T17:33:39.858Z27/30 Read block 593 len 12288
140592024-02-01T17:33:39.858Z28/30 Read block 261 len 32768
140602024-02-01T17:33:39.858Z29/30 Write block 319 len 24576 data: 12 13 11 12 9 10
140612024-02-01T17:33:39.858Z30/30 Write block 210 len 12288 data: 12 13 11
140622024-02-01T17:33:39.858Z----------------------------------------------------------------
140632024-02-01T17:33:39.858Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:29
140642024-02-01T17:33:39.859ZStates: Active Active Active
140652024-02-01T17:33:39.859ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140662024-02-01T17:33:39.859Z 2 Acked 1001 Write 5 Done Done Done false
140672024-02-01T17:33:39.859Z 3 Acked 1002 Write 6 Done Done Done false
140682024-02-01T17:33:39.859Z 4 Acked 1003 Write 9 Done Done Done false
140692024-02-01T17:33:39.859Z 5 Acked 1004 Read 8 Done Done Done false
140702024-02-01T17:33:39.859Z 6 Acked 1005 Read 9 Done Done Done false
140712024-02-01T17:33:39.859Z 7 Acked 1006 Read 4 Done Done Done false
140722024-02-01T17:33:39.859Z 8 Acked 1007 Read 4 Done Done Sent false
140732024-02-01T17:33:39.859Z 9 Acked 1008 Read 1 Done Done Sent false
140742024-02-01T17:33:39.859Z 10 Acked 1009 Write 7 Done Done Sent false
140752024-02-01T17:33:39.859Z 11 Acked 1010 Write 8 Done Done Sent false
140762024-02-01T17:33:39.859Z 12 Acked 1011 Write 7 Done Done Sent false
140772024-02-01T17:33:39.859Z 13 Acked 1012 Read 3 Done Done Sent false
140782024-02-01T17:33:39.859Z 14 Acked 1013 Read 7 Done Done Sent false
140792024-02-01T17:33:39.859Z 15 Acked 1014 Write 3 Done Done Sent false
140802024-02-01T17:33:39.859Z 16 Acked 1015 Write 10 Done Done Sent false
140812024-02-01T17:33:39.859Z 17 Acked 1016 Read 7 Done Done Sent false
140822024-02-01T17:33:39.859Z 18 Acked 1017 Write 4 Done Done Sent false
140832024-02-01T17:33:39.859Z 19 Acked 1018 Read 3 Done Done Sent false
140842024-02-01T17:33:39.859Z 20 Acked 1019 Read 3 Done Done Sent false
140852024-02-01T17:33:39.859Z 21 Acked 1020 Read 1 Done Done Sent false
140862024-02-01T17:33:39.859Z 22 Acked 1021 Write 4 Done Done Sent false
140872024-02-01T17:33:39.860Z 23 Acked 1022 Write 5 Done Done Sent false
140882024-02-01T17:33:39.860Z 24 Acked 1023 Read 8 Done Done Sent false
140892024-02-01T17:33:39.860Z 25 Acked 1024 Read 6 Done Done Sent false
140902024-02-01T17:33:39.860Z 26 Acked 1025 Write 2 Done Done Sent false
140912024-02-01T17:33:39.860Z 27 Acked 1026 Read 3 Done Done Sent false
140922024-02-01T17:33:39.860Z 28 Acked 1027 Read 8 Done Done Sent false
140932024-02-01T17:33:39.860Z 29 Acked 1028 Write 6 Sent Sent Sent false
140942024-02-01T17:33:39.860Z 30 Acked 1029 Write 3 Sent Sent Sent false
140952024-02-01T17:33:39.860Z STATES DS:0 DS:1 DS:2 TOTAL
140962024-02-01T17:33:39.860Z New 0 0 0 0
140972024-02-01T17:33:39.860Z Sent 2 2 23 27
140982024-02-01T17:33:39.861Z Done 27 27 6 60
140992024-02-01T17:33:39.861Z Skipped 0 0 0 0
141002024-02-01T17:33:39.861Z Error 0 0 0 0
141012024-02-01T17:33:39.861ZLast Flush: 1000 1000 1000
141022024-02-01T17:33:39.861ZDownstairs last five completed: 1000
141032024-02-01T17:33:39.861ZUpstairs last five completed: 30 29 28 27 26
141042024-02-01T17:33:39.861Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:39.125742079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1649}
141052024-02-01T17:33:39.861ZWrote out file "/tmp/test_repair_verify.data"
141062024-02-01T17:33:39.862Z
141072024-02-01T17:33:39.870ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141082024-02-01T17:33:39.895ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141092024-02-01T17:33:39.895Z 0 000-019 125 125 125 197 197 197 F F F
141102024-02-01T17:33:39.895Z 1 020-039 128 128 128 203 203 203 T T F <---
141112024-02-01T17:33:39.895Z 2 040-059 129 129 129 205 205 205 T T F <---
141122024-02-01T17:33:39.895Z 3 060-079 129 129 129 205 205 205 T T F <---
141132024-02-01T17:33:39.906Z 4 080-099 128 128 128 204 204 204 F F F
141142024-02-01T17:33:39.906Z 5 100-119 128 128 128 203 203 203 T T T
141152024-02-01T17:33:39.906Z 6 120-139 129 129 129 205 205 205 T T F <---
141162024-02-01T17:33:39.906Z 7 140-159 122 122 122 191 191 191 T T F <---
141172024-02-01T17:33:39.906Z 8 160-179 129 129 129 205 205 205 F F F
141182024-02-01T17:33:39.906Z 9 180-199 129 129 129 205 205 205 T T F <---
141192024-02-01T17:33:39.906Z 10 200-219 125 125 125 197 197 197 T T F <---
141202024-02-01T17:33:39.906Z 11 220-239 122 122 122 190 190 190 F F F
141212024-02-01T17:33:39.906Z 12 240-259 128 128 128 204 204 204 F F F
141222024-02-01T17:33:39.906Z 13 260-279 126 126 126 201 201 201 T T F <---
141232024-02-01T17:33:39.906Z 14 280-299 127 127 127 202 202 202 T T T
141242024-02-01T17:33:39.906Z 15 300-319 129 129 129 205 205 205 T T T
141252024-02-01T17:33:39.906Z 16 320-339 127 127 127 202 202 202 T T T
141262024-02-01T17:33:39.906Z 17 340-359 128 128 128 203 203 203 T T F <---
141272024-02-01T17:33:39.906Z 18 360-379 126 126 126 200 200 200 F F F
141282024-02-01T17:33:39.907Z 19 380-399 117 117 117 180 180 180 T T F <---
141292024-02-01T17:33:39.907Z 20 400-419 129 129 129 205 205 205 F F F
141302024-02-01T17:33:39.907Z 21 420-439 129 129 129 205 205 205 F F F
141312024-02-01T17:33:39.907Z 22 440-459 129 129 129 205 205 205 F F F
141322024-02-01T17:33:39.907Z 23 460-479 120 120 120 186 186 186 F F F
141332024-02-01T17:33:39.907Z 24 480-499 121 121 121 188 188 188 F F F
141342024-02-01T17:33:39.907Z 25 500-519 119 119 119 185 185 185 F F F
141352024-02-01T17:33:39.907Z 26 520-539 127 127 127 202 202 202 F F F
141362024-02-01T17:33:39.907Z 27 540-559 126 126 126 199 199 199 T T F <---
141372024-02-01T17:33:39.907Z 28 560-579 129 129 129 205 205 205 F F F
141382024-02-01T17:33:39.907Z 29 580-599 120 120 120 186 186 186 F F F
141392024-02-01T17:33:39.907ZMax gen: 129, Max flush: 205
141402024-02-01T17:33:39.908ZError: Difference in extent metadata found!
141412024-02-01T17:33:39.908ZA Difference in extent metadata is expected here
141422024-02-01T17:33:39.908ZOn loop 65
141432024-02-01T17:33:39.908Z
141442024-02-01T17:33:39.908ZVerifying data now
141452024-02-01T17:33:46.194ZNo change after verify
141462024-02-01T17:33:46.194ZLoop: 65 Downstairs dump after verify (and repair):
141472024-02-01T17:33:46.221ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141482024-02-01T17:33:46.222Z 0 000-019 125 125 125 197 197 197 F F F
141492024-02-01T17:33:46.692Z 1 020-039 130 130 130 206 206 206 F F F
141502024-02-01T17:33:46.692Z 2 040-059 130 130 130 206 206 206 F F F
141512024-02-01T17:33:46.692Z 3 060-079 130 130 130 206 206 206 F F F
141522024-02-01T17:33:46.693Z 4 080-099 128 128 128 204 204 204 F F F
141532024-02-01T17:33:46.694Z 5 100-119 130 130 130 206 206 206 F F F
141542024-02-01T17:33:46.694Z 6 120-139 130 130 130 206 206 206 F F F
141552024-02-01T17:33:46.694Z 7 140-159 130 130 130 206 206 206 F F F
141562024-02-01T17:33:46.695Z 8 160-179 129 129 129 205 205 205 F F F
141572024-02-01T17:33:46.695Z 9 180-199 130 130 130 206 206 206 F F F
141582024-02-01T17:33:46.695Z 10 200-219 130 130 130 206 206 206 F F F
141592024-02-01T17:33:46.695Z 11 220-239 122 122 122 190 190 190 F F F
141602024-02-01T17:33:46.695Z 12 240-259 128 128 128 204 204 204 F F F
141612024-02-01T17:33:46.695Z 13 260-279 130 130 130 206 206 206 F F F
141622024-02-01T17:33:46.695Z 14 280-299 130 130 130 206 206 206 F F F
141632024-02-01T17:33:46.696Z 15 300-319 130 130 130 206 206 206 F F F
141642024-02-01T17:33:46.696Z 16 320-339 130 130 130 206 206 206 F F F
141652024-02-01T17:33:46.696Z 17 340-359 130 130 130 206 206 206 F F F
141662024-02-01T17:33:46.696Z 18 360-379 126 126 126 200 200 200 F F F
141672024-02-01T17:33:46.696Z 19 380-399 130 130 130 206 206 206 F F F
141682024-02-01T17:33:46.696Z 20 400-419 129 129 129 205 205 205 F F F
141692024-02-01T17:33:46.696Z 21 420-439 129 129 129 205 205 205 F F F
141702024-02-01T17:33:46.696Z 22 440-459 129 129 129 205 205 205 F F F
141712024-02-01T17:33:46.696Z 23 460-479 120 120 120 186 186 186 F F F
141722024-02-01T17:33:46.697Z 24 480-499 121 121 121 188 188 188 F F F
141732024-02-01T17:33:46.697Z 25 500-519 119 119 119 185 185 185 F F F
141742024-02-01T17:33:46.697Z 26 520-539 127 127 127 202 202 202 F F F
141752024-02-01T17:33:46.697Z 27 540-559 130 130 130 206 206 206 F F F
141762024-02-01T17:33:46.697Z 28 560-579 129 129 129 205 205 205 F F F
141772024-02-01T17:33:46.697Z 29 580-599 120 120 120 186 186 186 F F F
141782024-02-01T17:33:46.698ZMax gen: 130, Max flush: 206
141792024-02-01T17:33:46.698Z
141802024-02-01T17:33:46.698ZBegin loop 66
141812024-02-01T17:33:46.698ZDownstairs to restart: 2
141822024-02-01T17:33:46.698Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.527658656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
141832024-02-01T17:33:46.698Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528180558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
141842024-02-01T17:33:46.698Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528245786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
141852024-02-01T17:33:46.698Z{"msg":"Crucible 3f76f492-0c77-4963-b78c-9b0bc381c92d has session id: 082ce4f1-f833-48fa-a49b-9c9e5db38df3","v":0,"name":"crucible","level":30Crucible runtime is spawned
141862024-02-01T17:33:46.699Z,"time":"2024-02-01T17:33:45.528314123Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141872024-02-01T17:33:46.699Z{"msg":"Upstairs opts: Upstairs UUID: 3f76f492-0c77-4963-b78c-9b0bc381c92d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528385588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141882024-02-01T17:33:46.699Z{"msg":"Crucible stats registered with UUID: 3f76f492-0c77-4963-b78c-9b0bc381c92d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528431185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141892024-02-01T17:33:46.699Z{"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528454607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
141902024-02-01T17:33:46.699Z{"msg":"3f76f492-0c77-4963-b78c-9b0bc381c92d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528561887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141912024-02-01T17:33:46.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528623104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141922024-02-01T17:33:46.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528721292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141932024-02-01T17:33:46.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528767439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141942024-02-01T17:33:46.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528810073Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141952024-02-01T17:33:46.699Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528849893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141962024-02-01T17:33:46.699Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.528930264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141972024-02-01T17:33:46.700Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529009422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141982024-02-01T17:33:46.700Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529083399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
141992024-02-01T17:33:46.700Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529128866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142002024-02-01T17:33:46.700Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529170233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142012024-02-01T17:33:46.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529404975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142022024-02-01T17:33:46.702Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.529558141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"io task","client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142032024-02-01T17:33:46.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.530195769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142042024-02-01T17:33:46.702Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.530271215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142052024-02-01T17:33:46.702Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.530322712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142062024-02-01T17:33:46.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.530438594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142072024-02-01T17:33:46.703Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.531011243Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142082024-02-01T17:33:46.703Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.53108678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142092024-02-01T17:33:46.703Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.531126041Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142102024-02-01T17:33:46.703Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.531311369Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142112024-02-01T17:33:46.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532007222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142122024-02-01T17:33:46.704Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532075841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142132024-02-01T17:33:46.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532104602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142142024-02-01T17:33:46.704Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532256519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142152024-02-01T17:33:46.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.53230901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142162024-02-01T17:33:46.704Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532335071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142172024-02-01T17:33:46.705Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.532357698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142182024-02-01T17:33:46.705Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533039121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142192024-02-01T17:33:46.705Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533103382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142202024-02-01T17:33:46.705Z{"msg":"[0]R flush_numbers[0..12]: [197, 206, 206, 206, 204, 206, 206, 206, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533131844Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142212024-02-01T17:33:46.705Z{"msg":"[0]R generation[0..12]: [125, 130, 130, 130, 128, 130, 130, 130, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533156533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142222024-02-01T17:33:46.705Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533199345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142232024-02-01T17:33:46.705Z{"msg":"[1]R flush_numbers[0..12]: [197, 206, 206, 206, 204, 206, 206, 206, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533225175Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142242024-02-01T17:33:46.705Z{"msg":"[1]R generation[0..12]: [125, 130, 130, 130, 128, 130, 130, 130, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533248537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142252024-02-01T17:33:46.706Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533290035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142262024-02-01T17:33:46.706Z{"msg":"[2]R flush_numbers[0..12]: [197, 206, 206, 206, 204, 206, 206, 206, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533315662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142272024-02-01T17:33:46.706Z{"msg":"[2]R generation[0..12]: [125, 130, 130, 130, 128, 130, 130, 130, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533339069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142282024-02-01T17:33:46.706Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
142292024-02-01T17:33:46.706Z,"time":"2024-02-01T17:33:45.533397839Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142302024-02-01T17:33:46.707Z{"msg":"Max found gen is 131","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533442916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142312024-02-01T17:33:46.707Z{"msg":"Generation requested: 132 >= found:131","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533471203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142322024-02-01T17:33:46.707Z{"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533514724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142332024-02-01T17:33:46.707Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533539266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"":"downstairs","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142342024-02-01T17:33:46.708Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533574931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142352024-02-01T17:33:46.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533638491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"0","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142362024-02-01T17:33:46.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533683148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"1","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142372024-02-01T17:33:46.708Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533750255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"client":"2","":"downstairs","session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142382024-02-01T17:33:46.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533795525Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142392024-02-01T17:33:46.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533855076Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142402024-02-01T17:33:46.709Z{"msg":"3f76f492-0c77-4963-b78c-9b0bc381c92d is now active with session: 082ce4f1-f833-48fa-a49b-9c9e5db38df3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533894098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142412024-02-01T17:33:46.711Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.533963659Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142422024-02-01T17:33:46.711Z:1657,"session_id":"082ce4f1-f833-48fa-a49b-9c9e5db38df3"}
142432024-02-01T17:33:46.712Z{"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:45.53401119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
142442024-02-01T17:33:46.712ZLoading write count information from file "/tmp/test_repair_verify.data"
142452024-02-01T17:33:46.712ZRun Repair workload
142462024-02-01T17:33:46.712Z01/30 Write block 578 len 28672 data: 10 10 10 10 11 10 11
142472024-02-01T17:33:46.712Z02/30 Read block 158 len 8192
142482024-02-01T17:33:46.712Z03/30 Read block 215 len 4096
142492024-02-01T17:33:46.750Z04/30 Read block 283 len 4096
142502024-02-01T17:33:46.750Z05/30 Write block 387 len 12288 data: 7 7 8
142512024-02-01T17:33:46.750Z06/30 Read block 12 len 32768
142522024-02-01T17:33:46.750Z07/30 Read block 220 len 36864
142532024-02-01T17:33:46.750Z08/30 Write block 34 len 12288 data: 8 9 11
142542024-02-01T17:33:46.750Z09/30 Read block 119 len 28672
142552024-02-01T17:33:46.750Z10/30 Write block 426 len 12288 data: 13 9 8
142562024-02-01T17:33:46.750Z11/30 Write block 343 len 36864 data: 8 8 8 9 10 12 10 8 9
142572024-02-01T17:33:46.750Z12/30 Read block 62 len 36864
142582024-02-01T17:33:46.751Z13/30 Read block 499 len 36864
142592024-02-01T17:33:46.751Z14/30 Write block 53 len 32768 data: 11 13 11 10 10 8 10 11
142602024-02-01T17:33:46.751Z15/30 Read block 455 len 28672
142612024-02-01T17:33:46.751Z16/30 Read block 187 len 32768
142622024-02-01T17:33:46.751Z17/30 Read block 466 len 36864
142632024-02-01T17:33:46.751Z18/30 Write block 145 len 32768 data: 9 9 9 13 13 9 9 11
142642024-02-01T17:33:46.751Z19/30 Read block 535 len 32768
142652024-02-01T17:33:46.751Z20/30 Flush
142662024-02-01T17:33:46.751Z21/30 Flush
142672024-02-01T17:33:46.751Z22/30 Write block 310 len 12288 data: 6 6 10
142682024-02-01T17:33:46.751Z23/30 Flush
142692024-02-01T17:33:46.751Z24/30 Write block 316 len 36864 data: 13 14 12 13 14 12 13 10 11
142702024-02-01T17:33:46.751Z25/30 Write block 580 len 12288 data: 11 11 12
142712024-02-01T17:33:46.752Z26/30 Write block 306 len 20480 data: 7 6 9 7 7
142722024-02-01T17:33:46.752Z27/30 Read block 294 len 32768
142732024-02-01T17:33:46.752Z28/30 Write block 99 len 16384 data: 10 10 11 11
142742024-02-01T17:33:46.752Z29/30 Write block 565 len 32768 data: 9 8 7 7 8 10 7 6
142752024-02-01T17:33:46.752Z30/30 Read block 195 len 40960
142762024-02-01T17:33:46.752Z----------------------------------------------------------------
142772024-02-01T17:33:46.752Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
142782024-02-01T17:33:46.752ZStates: Active Active Active
142792024-02-01T17:33:46.752ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
142802024-02-01T17:33:46.752Z 1 Acked 1000 Write 7 Done Done Sent false
142812024-02-01T17:33:46.752Z 2 Acked 1001 Read 2 Done Done Sent false
142822024-02-01T17:33:46.752Z 3 Acked 1002 Read 1 Done Done Sent false
142832024-02-01T17:33:46.752Z 4 Acked 1003 Read 1 Done Done Sent false
142842024-02-01T17:33:46.752Z 5 Acked 1004 Write 3 Done Done Sent false
142852024-02-01T17:33:46.753Z 6 Acked 1005 Read 8 Done Done Sent false
142862024-02-01T17:33:46.753Z 7 Acked 1006 Read 9 Done Done Sent false
142872024-02-01T17:33:46.753Z 8 Acked 1007 Write 3 Done Done Sent false
142882024-02-01T17:33:46.754Z 9 Acked 1008 Read 7 Done Done Sent false
142892024-02-01T17:33:46.754Z 10 Acked 1009 Write 3 Done Done Sent false
142902024-02-01T17:33:46.754Z 11 Acked 1010 Write 9 Done Done Sent false
142912024-02-01T17:33:46.754Z 12 Acked 1011 Read 9 Done Done Sent false
142922024-02-01T17:33:46.754Z 13 Acked 1012 Read 9 Done Done Sent false
142932024-02-01T17:33:46.754Z 14 Acked 1013 Write 8 Done Done Sent false
142942024-02-01T17:33:46.754Z 15 Acked 1014 Read 7 Done Done Sent false
142952024-02-01T17:33:46.755Z 16 Acked 1015 Read 8 Done Done Sent false
142962024-02-01T17:33:46.755Z 17 Acked 1016 Read 9 Done Done Sent false
142972024-02-01T17:33:46.755Z 18 Acked 1017 Write 8 Done Done Sent false
142982024-02-01T17:33:46.755Z 19 Acked 1018 Read 8 Done Done Sent false
142992024-02-01T17:33:46.755Z 20 Acked 1019 Flush 0 Done Done Sent false
143002024-02-01T17:33:46.755Z 21 Acked 1020 Flush 0 Done Done Sent false
143012024-02-01T17:33:46.755Z 22 Acked 1021 Write 3 Done Done Sent false
143022024-02-01T17:33:46.756Z 23 Acked 1022 Flush 0 Done Done Sent false
143032024-02-01T17:33:46.756Z 24 Acked 1023 Write 9 Done Done Sent false
143042024-02-01T17:33:46.756Z 25 Acked 1024 Write 3 Done Done Sent false
143052024-02-01T17:33:46.756Z 26 Acked 1025 Write 5 Done Done Sent false
143062024-02-01T17:33:46.756Z 27 Acked 1026 Read 8 Done Done Sent false
143072024-02-01T17:33:46.756Z 28 Acked 1027 Write 4 Done Done Sent false
143082024-02-01T17:33:46.756Z 29 Acked 1028 Write 8 Done Done Sent false
143092024-02-01T17:33:46.756Z 30 Acked 1029 Read 10 Done Sent Sent false
143102024-02-01T17:33:46.756Z STATES DS:0 DS:1 DS:2 TOTAL
143112024-02-01T17:33:46.758Z New 0 0 0 0
143122024-02-01T17:33:46.758Z Sent 0 1 30 31
143132024-02-01T17:33:46.758Z Done 30 29 0 59
143142024-02-01T17:33:46.758Z Skipped 0 0 0 0
143152024-02-01T17:33:46.758Z Error 0 0 0 0
143162024-02-01T17:33:46.758ZLast Flush: 1022 1022 0
143172024-02-01T17:33:46.758ZDownstairs last five completed:
143182024-02-01T17:33:46.758ZUpstairs last five completed: 30 29 28 27 26
143192024-02-01T17:33:46.759Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:45.582254869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1657}
143202024-02-01T17:33:46.759ZWrote out file "/tmp/test_repair_verify.data"
143212024-02-01T17:33:46.759Z
143222024-02-01T17:33:46.759ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143232024-02-01T17:33:46.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143242024-02-01T17:33:46.759Z 0 000-019 125 125 125 197 197 197 F F F
143252024-02-01T17:33:46.759Z 1 020-039 132 132 130 207 207 206 F F F <---
143262024-02-01T17:33:46.759Z 2 040-059 132 132 130 207 207 206 F F F <---
143272024-02-01T17:33:46.759Z 3 060-079 132 132 130 207 207 206 F F F <---
143282024-02-01T17:33:46.760Z 4 080-099 128 128 128 204 204 204 T T F <---
143292024-02-01T17:33:46.760Z 5 100-119 130 130 130 206 206 206 T T F <---
143302024-02-01T17:33:46.760Z 6 120-139 130 130 130 206 206 206 F F F
143312024-02-01T17:33:46.760Z 7 140-159 132 132 130 207 207 206 F F F <---
143322024-02-01T17:33:46.760Z 8 160-179 129 129 129 205 205 205 F F F
143332024-02-01T17:33:46.760Z 9 180-199 130 130 130 206 206 206 F F F
143342024-02-01T17:33:46.760Z 10 200-219 130 130 130 206 206 206 F F F
143352024-02-01T17:33:46.761Z 11 220-239 122 122 122 190 190 190 F F F
143362024-02-01T17:33:46.761Z 12 240-259 128 128 128 204 204 204 F F F
143372024-02-01T17:33:46.761Z 13 260-279 130 130 130 206 206 206 F F F
143382024-02-01T17:33:46.761Z 14 280-299 130 130 130 206 206 206 F F F
143392024-02-01T17:33:46.761Z 15 300-319 132 132 130 209 209 206 T T F <---
143402024-02-01T17:33:46.761Z 16 320-339 130 130 130 206 206 206 T T F <---
143412024-02-01T17:33:46.762Z 17 340-359 132 132 130 207 207 206 F F F <---
143422024-02-01T17:33:46.762Z 18 360-379 126 126 126 200 200 200 F F F
143432024-02-01T17:33:46.762Z 19 380-399 132 132 130 207 207 206 F F F <---
143442024-02-01T17:33:46.762Z 20 400-419 129 129 129 205 205 205 F F F
143452024-02-01T17:33:46.762Z 21 420-439 132 132 129 207 207 205 F F F <---
143462024-02-01T17:33:46.762Z 22 440-459 129 129 129 205 205 205 F F F
143472024-02-01T17:33:46.762Z 23 460-479 120 120 120 186 186 186 F F F
143482024-02-01T17:33:46.762Z 24 480-499 121 121 121 188 188 188 F F F
143492024-02-01T17:33:46.767Z 25 500-519 119 119 119 185 185 185 F F F
143502024-02-01T17:33:46.768Z 26 520-539 127 127 127 202 202 202 F F F
143512024-02-01T17:33:46.768Z 27 540-559 130 130 130 206 206 206 F F F
143522024-02-01T17:33:46.768Z 28 560-579 132 132 129 207 207 205 T T F <---
143532024-02-01T17:33:46.768Z 29 580-599 132 132 120 207 207 186 T T F <---
143542024-02-01T17:33:46.768ZMax gen: 132, Max flush: 209
143552024-02-01T17:33:46.768ZError: Difference in extent metadata found!
143562024-02-01T17:33:46.768ZA Difference in extent metadata is expected here
143572024-02-01T17:33:46.768ZOn loop 66
143582024-02-01T17:33:46.768Z
143592024-02-01T17:33:46.769ZVerifying data now
143602024-02-01T17:33:51.466ZNo change after verify
143612024-02-01T17:33:51.467ZLoop: 66 Downstairs dump after verify (and repair):
143622024-02-01T17:33:51.483ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143632024-02-01T17:33:51.483Z 0 000-019 125 125 125 197 197 197 F F F
143642024-02-01T17:33:51.483Z 1 020-039 132 132 132 207 207 207 F F F
143652024-02-01T17:33:51.484Z 2 040-059 132 132 132 207 207 207 F F F
143662024-02-01T17:33:51.484Z 3 060-079 132 132 132 207 207 207 F F F
143672024-02-01T17:33:51.550Z 4 080-099 133 133 133 210 210 210 F F F
143682024-02-01T17:33:51.550Z 5 100-119 133 133 133 210 210 210 F F F
143692024-02-01T17:33:51.550Z 6 120-139 130 130 130 206 206 206 F F F
143702024-02-01T17:33:51.550Z 7 140-159 132 132 132 207 207 207 F F F
143712024-02-01T17:33:51.551Z 8 160-179 129 129 129 205 205 205 F F F
143722024-02-01T17:33:51.551Z 9 180-199 130 130 130 206 206 206 F F F
143732024-02-01T17:33:51.553Z 10 200-219 130 130 130 206 206 206 F F F
143742024-02-01T17:33:51.553Z 11 220-239 122 122 122 190 190 190 F F F
143752024-02-01T17:33:51.553Z 12 240-259 128 128 128 204 204 204 F F F
143762024-02-01T17:33:51.553Z 13 260-279 130 130 130 206 206 206 F F F
143772024-02-01T17:33:51.554Z 14 280-299 130 130 130 206 206 206 F F F
143782024-02-01T17:33:51.554Z 15 300-319 133 133 133 210 210 210 F F F
143792024-02-01T17:33:51.554Z 16 320-339 133 133 133 210 210 210 F F F
143802024-02-01T17:33:51.554Z 17 340-359 132 132 132 207 207 207 F F F
143812024-02-01T17:33:51.554Z 18 360-379 126 126 126 200 200 200 F F F
143822024-02-01T17:33:51.554Z 19 380-399 132 132 132 207 207 207 F F F
143832024-02-01T17:33:51.555Z 20 400-419 129 129 129 205 205 205 F F F
143842024-02-01T17:33:51.555Z 21 420-439 132 132 132 207 207 207 F F F
143852024-02-01T17:33:51.555Z 22 440-459 129 129 129 205 205 205 F F F
143862024-02-01T17:33:51.555Z 23 460-479 120 120 120 186 186 186 F F F
143872024-02-01T17:33:51.555Z 24 480-499 121 121 121 188 188 188 F F F
143882024-02-01T17:33:51.556Z 25 500-519 119 119 119 185 185 185 F F F
143892024-02-01T17:33:51.556Z 26 520-539 127 127 127 202 202 202 F F F
143902024-02-01T17:33:51.556Z 27 540-559 130 130 130 206 206 206 F F F
143912024-02-01T17:33:51.556Z 28 560-579 133 133 133 210 210 210 F F F
143922024-02-01T17:33:51.556Z 29 580-599 133 133 133 210 210 210 F F F
143932024-02-01T17:33:51.556ZMax gen: 133, Max flush: 210
143942024-02-01T17:33:51.557Z
143952024-02-01T17:33:51.557ZBegin loop 67
143962024-02-01T17:33:51.557ZDownstairs to restart: 2
143972024-02-01T17:33:51.557Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.79233247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
143982024-02-01T17:33:51.558Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.792849992Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
143992024-02-01T17:33:51.558Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.792887311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
144002024-02-01T17:33:51.558Z{"msg":"Crucible 407cb0f4-9af0-4147-92f8-b00427498e45 has session id: e6adbd9a-e8a4-4b57-87c1-cc661e407d6f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793018386Z","hostname"Crucible runtime is spawned
144012024-02-01T17:33:51.558Z:"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144022024-02-01T17:33:51.558Z{"msg":"Upstairs opts: Upstairs UUID: 407cb0f4-9af0-4147-92f8-b00427498e45, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793145623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144032024-02-01T17:33:51.558Z{"msg":"Crucible stats registered with UUID: 407cb0f4-9af0-4147-92f8-b00427498e45","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793216616Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144042024-02-01T17:33:51.559Z{"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793243569Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
144052024-02-01T17:33:51.559Z{"msg":"407cb0f4-9af0-4147-92f8-b00427498e45 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793370244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144062024-02-01T17:33:51.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793461218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144072024-02-01T17:33:51.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793530898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144082024-02-01T17:33:51.559Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793566683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144092024-02-01T17:33:51.559Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793605053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144102024-02-01T17:33:51.560Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793634339Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144112024-02-01T17:33:51.560Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793681867Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144122024-02-01T17:33:51.560Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793706968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144132024-02-01T17:33:51.560Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793730538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144142024-02-01T17:33:51.560Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793753327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144152024-02-01T17:33:51.560Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.793885653Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144162024-02-01T17:33:51.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.794165376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144172024-02-01T17:33:51.561Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.794295478Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"io task","client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144182024-02-01T17:33:51.561Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.795158098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144192024-02-01T17:33:51.561Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.795284853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144202024-02-01T17:33:51.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.795344342Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144212024-02-01T17:33:51.562Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.79538858Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144222024-02-01T17:33:51.562Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.795912475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144232024-02-01T17:33:51.562Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.795979557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144242024-02-01T17:33:51.562Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.796557526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144252024-02-01T17:33:51.562Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.796655424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144262024-02-01T17:33:51.563Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797045833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144272024-02-01T17:33:51.563Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797492003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144282024-02-01T17:33:51.565Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797559789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144292024-02-01T17:33:51.565Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797587235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144302024-02-01T17:33:51.565Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797655272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144312024-02-01T17:33:51.565Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.79768248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144322024-02-01T17:33:51.566Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797725897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144332024-02-01T17:33:51.566Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.797989484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144342024-02-01T17:33:51.566Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798044813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144352024-02-01T17:33:51.566Z{"msg":"[0]R flush_numbers[0..12]: [197, 207, 207, 207, 210, 210, 206, 207, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798071692Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144362024-02-01T17:33:51.566Z{"msg":"[0]R generation[0..12]: [125, 132, 132, 132, 133, 133, 130, 132, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798114095Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144372024-02-01T17:33:51.566Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798140438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144382024-02-01T17:33:51.566Z{"msg":"[1]R flush_numbers[0..12]: [197, 207, 207, 207, 210, 210, 206, 207, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798183017Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144392024-02-01T17:33:51.566Z{"msg":"[1]R generation[0..12]: [125, 132, 132, 132, 133, 133, 130, 132, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798208429Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144402024-02-01T17:33:51.566Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798250211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144412024-02-01T17:33:51.566Z{"msg":"[2]R flush_numbers[0..12]: [197, 207, 207, 207, 210, 210, 206, 207, 205, 206, 206, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798275896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144422024-02-01T17:33:51.566Z{"msg":"[2]R generation[0..12]: [125, 132, 132, 132, 133, 133, 130, 132, 129, 130, 130, 122]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO
144432024-02-01T17:33:51.567Z2024-02-01T17:33:50.798317337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144442024-02-01T17:33:51.567Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.79834812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144452024-02-01T17:33:51.567Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798396115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144462024-02-01T17:33:51.567Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798420862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144472024-02-01T17:33:51.567Z{"msg":"Next flush: 211","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798466482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144482024-02-01T17:33:51.567Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798490402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"":"downstairs","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144492024-02-01T17:33:51.567Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798534351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144502024-02-01T17:33:51.567Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798556188Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"0","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144512024-02-01T17:33:51.567Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798596265Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"1","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144522024-02-01T17:33:51.567Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798620408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"client":"2","":"downstairs","session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144532024-02-01T17:33:51.567Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798660864Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144542024-02-01T17:33:51.567Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798683224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144552024-02-01T17:33:51.567Z{"msg":"407cb0f4-9af0-4147-92f8-b00427498e45 is now active with session: e6adbd9a-e8a4-4b57-87c1-cc661e407d6f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798728039Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144562024-02-01T17:33:51.567Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.798749874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665,"session_id":"e6adbd9a-e8a4-4b57-87c1-cc661e407d6f"}
144572024-02-01T17:33:51.567Z{"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:50.79879543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
144582024-02-01T17:33:51.567ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
144592024-02-01T17:33:51.568ZLoading write count information from file "/tmp/test_repair_verify.data"
144602024-02-01T17:33:51.568ZRun Repair workload
144612024-02-01T17:33:51.568Z01/30 Write block 202 len 12288 data: 13 15 15
144622024-02-01T17:33:51.568Z02/30 Read block 277 len 28672
144632024-02-01T17:33:51.568Z03/30 Read block 207 len 20480
144642024-02-01T17:33:51.568Z04/30 Flush
144652024-02-01T17:33:51.568Z05/30 Write block 260 len 20480 data: 11 10 11 13 13
144662024-02-01T17:33:51.568Z06/30 Flush
144672024-02-01T17:33:51.580Z07/30 Write block 129 len 36864 data: 10 10 9 6 10 13 14 15 15
144682024-02-01T17:33:51.580Z08/30 Flush
144692024-02-01T17:33:51.580Z09/30 Write block 118 len 24576 data: 9 8 8 7 9 9
144702024-02-01T17:33:51.580Z10/30 Read block 374 len 8192
144712024-02-01T17:33:51.580Z11/30 Flush
144722024-02-01T17:33:51.580Z12/30 Write block 390 len 20480 data: 9 8 9 10 8
144732024-02-01T17:33:51.580Z13/30 Read block 122 len 20480
144742024-02-01T17:33:51.580Z14/30 Write block 432 len 28672 data: 13 16 14 16 14 15 13
144752024-02-01T17:33:51.580Z15/30 Read block 329 len 40960
144762024-02-01T17:33:51.580Z16/30 Write block 299 len 28672 data: 12 12 9 7 9 9 10
144772024-02-01T17:33:51.580Z17/30 Read block 334 len 28672
144782024-02-01T17:33:51.580Z18/30 Write block 440 len 12288 data: 12 12 10
144792024-02-01T17:33:51.580Z19/30 Write block 463 len 24576 data: 10 9 8 10 10 9
144802024-02-01T17:33:51.580Z20/30 Read block 319 len 12288
144812024-02-01T17:33:51.580Z21/30 Write block 3 len 8192 data: 8 10
144822024-02-01T17:33:51.580Z22/30 Read block 548 len 16384
144832024-02-01T17:33:51.580Z23/30 Read block 25 len 32768
144842024-02-01T17:33:51.580Z24/30 Read block 218 len 24576
144852024-02-01T17:33:51.581Z25/30 Read block 185 len 28672
144862024-02-01T17:33:51.581Z26/30 Read block 315 len 24576
144872024-02-01T17:33:51.581Z27/30 Write block 178 len 32768 data: 12 12 10 12 14 16 15 16
144882024-02-01T17:33:51.581Z28/30 Read block 413 len 24576
144892024-02-01T17:33:51.581Z29/30 Read block 536 len 12288
144902024-02-01T17:33:51.581Z30/30 Write block 385 len 32768 data: 9 7 8 8 9 10 9 10
144912024-02-01T17:33:51.581Z----------------------------------------------------------------
144922024-02-01T17:33:51.581Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
144932024-02-01T17:33:51.581ZStates: Active Active Active
144942024-02-01T17:33:51.581ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
144952024-02-01T17:33:51.581Z 1 Acked 1000 Write 3 Done Done Done false
144962024-02-01T17:33:51.581Z 2 Acked 1001 Read 7 Done Done Sent false
144972024-02-01T17:33:51.581Z 3 Acked 1002 Read 5 Done Done Sent false
144982024-02-01T17:33:51.581Z 4 Acked 1003 Flush 0 Done Done Sent false
144992024-02-01T17:33:51.581Z 5 Acked 1004 Write 5 Done Done Sent false
145002024-02-01T17:33:51.581Z 6 Acked 1005 Flush 0 Done Done Sent false
145012024-02-01T17:33:51.581Z 7 Acked 1006 Write 9 Done Done Sent false
145022024-02-01T17:33:51.581Z 8 Acked 1007 Flush 0 Done Done Sent false
145032024-02-01T17:33:51.581Z 9 Acked 1008 Write 6 Done Done Sent false
145042024-02-01T17:33:51.581Z 10 Acked 1009 Read 2 Done Done Sent false
145052024-02-01T17:33:51.581Z 11 Acked 1010 Flush 0 Done Done Sent false
145062024-02-01T17:33:51.582Z 12 Acked 1011 Write 5 Done Done Sent false
145072024-02-01T17:33:51.582Z 13 Acked 1012 Read 5 Done Done Sent false
145082024-02-01T17:33:51.582Z 14 Acked 1013 Write 7 Done Done Sent false
145092024-02-01T17:33:51.584Z 15 Acked 1014 Read 10 Done Done Sent false
145102024-02-01T17:33:51.584Z 16 Acked 1015 Write 7 Done Done Sent false
145112024-02-01T17:33:51.585Z 17 Acked 1016 Read 7 Done Done Sent false
145122024-02-01T17:33:51.585Z 18 Acked 1017 Write 3 Done Done Sent false
145132024-02-01T17:33:51.585Z 19 Acked 1018 Write 6 Done Done Sent false
145142024-02-01T17:33:51.585Z 20 Acked 1019 Read 3 Done Done Sent false
145152024-02-01T17:33:51.585Z 21 Acked 1020 Write 2 Done Done Sent false
145162024-02-01T17:33:51.585Z 22 Acked 1021 Read 4 Done Done Sent false
145172024-02-01T17:33:51.585Z 23 Acked 1022 Read 8 Done Done Sent false
145182024-02-01T17:33:51.585Z 24 Acked 1023 Read 6 Done Done Sent false
145192024-02-01T17:33:51.585Z 25 Acked 1024 Read 7 Done Done Sent false
145202024-02-01T17:33:51.585Z 26 Acked 1025 Read 6 Done Done Sent false
145212024-02-01T17:33:51.585Z 27 Acked 1026 Write 8 Done Done Sent false
145222024-02-01T17:33:51.585Z 28 Acked 1027 Read 6 Done Done Sent false
145232024-02-01T17:33:51.585Z 29 Acked 1028 Read 3 Done Done Sent false
145242024-02-01T17:33:51.585Z 30 Acked 1029 Write 8 Sent Sent Sent false
145252024-02-01T17:33:51.585Z STATES DS:0 DS:1 DS:2 TOTAL
145262024-02-01T17:33:51.585Z New 0 0 0 0
145272024-02-01T17:33:51.585Z Sent 1 1 29 31
145282024-02-01T17:33:51.586Z Done 29 29 1 59
145292024-02-01T17:33:51.586Z Skipped 0 0 0 0
145302024-02-01T17:33:51.586Z Error 0 0 0 0
145312024-02-01T17:33:51.586ZLast Flush: 1010 1010 0
145322024-02-01T17:33:51.586ZDownstairs last five completed:
145332024-02-01T17:33:51.587ZUpstairs last five completed: 30 29 28 27 26
145342024-02-01T17:33:51.587Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:50.849405048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1665}
145352024-02-01T17:33:51.587ZWrote out file "/tmp/test_repair_verify.data"
145362024-02-01T17:33:51.629Z
145372024-02-01T17:33:51.629ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
145382024-02-01T17:33:51.641ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145392024-02-01T17:33:51.641Z 0 000-019 125 125 125 197 197 197 T T F <---
145402024-02-01T17:33:51.641Z 1 020-039 132 132 132 207 207 207 F F F
145412024-02-01T17:33:51.641Z 2 040-059 132 132 132 207 207 207 F F F
145422024-02-01T17:33:51.641Z 3 060-079 132 132 132 207 207 207 F F F
145432024-02-01T17:33:51.641Z 4 080-099 133 133 133 210 210 210 F F F
145442024-02-01T17:33:51.641Z 5 100-119 134 134 133 214 214 210 F F F <---
145452024-02-01T17:33:51.641Z 6 120-139 134 134 130 214 214 206 F F F <---
145462024-02-01T17:33:51.641Z 7 140-159 132 132 132 207 207 207 F F F
145472024-02-01T17:33:51.641Z 8 160-179 129 129 129 205 205 205 T T F <---
145482024-02-01T17:33:51.641Z 9 180-199 130 130 130 206 206 206 T T F <---
145492024-02-01T17:33:51.641Z 10 200-219 134 134 130 211 211 206 F F T <---
145502024-02-01T17:33:51.642Z 11 220-239 122 122 122 190 190 190 F F F
145512024-02-01T17:33:51.642Z 12 240-259 128 128 128 204 204 204 F F F
145522024-02-01T17:33:51.642Z 13 260-279 134 134 130 212 212 206 F F F <---
145532024-02-01T17:33:51.642Z 14 280-299 130 130 130 206 206 206 T T F <---
145542024-02-01T17:33:51.642Z 15 300-319 133 133 133 210 210 210 T T F <---
145552024-02-01T17:33:51.642Z 16 320-339 133 133 133 210 210 210 F F F
145562024-02-01T17:33:51.645Z 17 340-359 132 132 132 207 207 207 F F F
145572024-02-01T17:33:51.645Z 18 360-379 126 126 126 200 200 200 F F F
145582024-02-01T17:33:51.645Z 19 380-399 132 132 132 207 207 207 T T F <---
145592024-02-01T17:33:51.645Z 20 400-419 129 129 129 205 205 205 F F F
145602024-02-01T17:33:51.645Z 21 420-439 132 132 132 207 207 207 T T F <---
145612024-02-01T17:33:51.645Z 22 440-459 129 129 129 205 205 205 T T F <---
145622024-02-01T17:33:51.645Z 23 460-479 120 120 120 186 186 186 T T F <---
145632024-02-01T17:33:51.645Z 24 480-499 121 121 121 188 188 188 F F F
145642024-02-01T17:33:51.645Z 25 500-519 119 119 119 185 185 185 F F F
145652024-02-01T17:33:51.645Z 26 520-539 127 127 127 202 202 202 F F F
145662024-02-01T17:33:51.645Z 27 540-559 130 130 130 206 206 206 F F F
145672024-02-01T17:33:51.645Z 28 560-579 133 133 133 210 210 210 F F F
145682024-02-01T17:33:51.645Z 29 580-599 133 133 133 210 210 210 F F F
145692024-02-01T17:33:51.645ZMax gen: 134, Max flush: 214
145702024-02-01T17:33:51.645ZError: Difference in extent metadata found!
145712024-02-01T17:33:51.645ZA Difference in extent metadata is expected here
145722024-02-01T17:33:51.645ZOn loop 67
145732024-02-01T17:33:51.645Z
145742024-02-01T17:33:51.645ZVerifying data now
145752024-02-01T17:33:56.746ZNo change after verify
145762024-02-01T17:33:56.747ZLoop: 67 Downstairs dump after verify (and repair):
145772024-02-01T17:33:56.769ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
145782024-02-01T17:33:56.769Z 0 000-019 135 135 135 215 215 215 F F F
145792024-02-01T17:33:56.778Z 1 020-039 132 132 132 207 207 207 F F F
145802024-02-01T17:33:56.778Z 2 040-059 132 132 132 207 207 207 F F F
145812024-02-01T17:33:56.778Z 3 060-079 132 132 132 207 207 207 F F F
145822024-02-01T17:33:56.778Z 4 080-099 133 133 133 210 210 210 F F F
145832024-02-01T17:33:56.778Z 5 100-119 134 134 134 214 214 214 F F F
145842024-02-01T17:33:56.778Z 6 120-139 134 134 134 214 214 214 F F F
145852024-02-01T17:33:56.778Z 7 140-159 132 132 132 207 207 207 F F F
145862024-02-01T17:33:56.778Z 8 160-179 135 135 135 215 215 215 F F F
145872024-02-01T17:33:56.778Z 9 180-199 135 135 135 215 215 215 F F F
145882024-02-01T17:33:56.778Z 10 200-219 134 134 134 211 211 211 F F F
145892024-02-01T17:33:56.778Z 11 220-239 122 122 122 190 190 190 F F F
145902024-02-01T17:33:56.778Z 12 240-259 128 128 128 204 204 204 F F F
145912024-02-01T17:33:56.778Z 13 260-279 134 134 134 212 212 212 F F F
145922024-02-01T17:33:56.778Z 14 280-299 135 135 135 215 215 215 F F F
145932024-02-01T17:33:56.778Z 15 300-319 135 135 135 215 215 215 F F F
145942024-02-01T17:33:56.778Z 16 320-339 133 133 133 210 210 210 F F F
145952024-02-01T17:33:56.778Z 17 340-359 132 132 132 207 207 207 F F F
145962024-02-01T17:33:56.778Z 18 360-379 126 126 126 200 200 200 F F F
145972024-02-01T17:33:56.779Z 19 380-399 135 135 135 215 215 215 F F F
145982024-02-01T17:33:56.779Z 20 400-419 129 129 129 205 205 205 F F F
145992024-02-01T17:33:56.779Z 21 420-439 135 135 135 215 215 215 F F F
146002024-02-01T17:33:56.779Z 22 440-459 135 135 135 215 215 215 F F F
146012024-02-01T17:33:56.779Z 23 460-479 135 135 135 215 215 215 F F F
146022024-02-01T17:33:56.779Z 24 480-499 121 121 121 188 188 188 F F F
146032024-02-01T17:33:56.779Z 25 500-519 119 119 119 185 185 185 F F F
146042024-02-01T17:33:56.779Z 26 520-539 127 127 127 202 202 202 F F F
146052024-02-01T17:33:56.779Z 27 540-559 130 130 130 206 206 206 F F F
146062024-02-01T17:33:56.779Z 28 560-579 133 133 133 210 210 210 F F F
146072024-02-01T17:33:56.779Z 29 580-599 133 133 133 210 210 210 F F F
146082024-02-01T17:33:56.779ZMax gen: 135, Max flush: 215
146092024-02-01T17:33:56.779Z
146102024-02-01T17:33:56.779ZBegin loop 68
146112024-02-01T17:33:56.779ZDownstairs to restart: 1
146122024-02-01T17:33:56.806Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.084791998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
146132024-02-01T17:33:56.806Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.085275541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
146142024-02-01T17:33:56.806Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.085332617Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
146152024-02-01T17:33:56.806Z{"msg":"Crucible cb865e62-196a-4542-9858-8af4bcf8d501 has session id: 30da9d8d-60e4-4296-b641-d67fde043259","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.085371097Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146162024-02-01T17:33:56.811Z{"msg":"Upstairs opts: Upstairs UUID: cb865e62-196a-4542-9858-8af4bcf8d501, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.08545792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146172024-02-01T17:33:56.811Z{"msg":"Crucible stats registered with UUID: cb865e62-196a-4542-9858-8af4bcf8d501","v":0,"name":"crucible","level":30Crucible runtime is spawned
146182024-02-01T17:33:56.811Z,"time":"2024-02-01T17:33:56.085529063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146192024-02-01T17:33:56.811Z{"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.085639725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
146202024-02-01T17:33:56.811Z{"msg":"cb865e62-196a-4542-9858-8af4bcf8d501 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.085883838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146212024-02-01T17:33:56.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.08593899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146222024-02-01T17:33:56.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086019034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146232024-02-01T17:33:56.811Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086065675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146242024-02-01T17:33:56.811Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086160554Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146252024-02-01T17:33:56.812Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086243999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146262024-02-01T17:33:56.812Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086345572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146272024-02-01T17:33:56.812Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086424756Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146282024-02-01T17:33:56.812Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086605074Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146292024-02-01T17:33:56.812Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086790016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146302024-02-01T17:33:56.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.086848507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146312024-02-01T17:33:56.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.08693838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146322024-02-01T17:33:56.812Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.087070229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"io task","client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146332024-02-01T17:33:56.812Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.087621325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146342024-02-01T17:33:56.813Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.087715045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146352024-02-01T17:33:56.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.087741066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146362024-02-01T17:33:56.813Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.087776706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146372024-02-01T17:33:56.813Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.088382686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146382024-02-01T17:33:56.813Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.088494654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146392024-02-01T17:33:56.813Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.088612816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146402024-02-01T17:33:56.813Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.088678218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146412024-02-01T17:33:56.813Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089318775Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146422024-02-01T17:33:56.821Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089708878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146432024-02-01T17:33:56.821Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089766292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146442024-02-01T17:33:56.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089792539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146452024-02-01T17:33:56.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.08984947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146462024-02-01T17:33:56.823Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089877086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146472024-02-01T17:33:56.823Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.089907049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146482024-02-01T17:33:56.823Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090092744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146492024-02-01T17:33:56.823Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090151094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146502024-02-01T17:33:56.823Z{"msg":"[0]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 215, 215, 211, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090179162Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146512024-02-01T17:33:56.823Z{"msg":"[0]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 135, 135, 134, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090203676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146522024-02-01T17:33:56.823Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090251384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146532024-02-01T17:33:56.823Z{"msg":"[1]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 215, 215, 211, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090278166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146542024-02-01T17:33:56.825Z{"msg":"[1]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 135, 135, 134, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090322188Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146552024-02-01T17:33:56.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090348596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146562024-02-01T17:33:56.825Z{"msg":"[2]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 215, 215, 211, 190]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090371997Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146572024-02-01T17:33:56.825Z{"msg":"[2]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 135, 135, 134, 122]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090413757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146582024-02-01T17:33:56.825Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090439794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146592024-02-01T17:33:56.825Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090461729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146602024-02-01T17:33:56.825Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090519033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO
146612024-02-01T17:33:56.825Zdownstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146622024-02-01T17:33:56.825Z{"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090547119Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146632024-02-01T17:33:56.825Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090592768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"":"downstairs","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146642024-02-01T17:33:56.826Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090617317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146652024-02-01T17:33:56.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090637766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"0","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146662024-02-01T17:33:56.826Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090684091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"1","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146672024-02-01T17:33:56.827Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090708079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"client":"2","":"downstairs","session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146682024-02-01T17:33:56.827Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090729998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146692024-02-01T17:33:56.827Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090768958Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146702024-02-01T17:33:56.827Z{"msg":"cb865e62-196a-4542-9858-8af4bcf8d501 is now active with session: 30da9d8d-60e4-4296-b641-d67fde043259","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090792635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146712024-02-01T17:33:56.827Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.090812689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673,"session_id":"30da9d8d-60e4-4296-b641-d67fde043259"}
146722024-02-01T17:33:56.827Z{"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:33:56.09085543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
146732024-02-01T17:33:56.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
146742024-02-01T17:33:56.827ZLoading write count information from file "/tmp/test_repair_verify.data"
146752024-02-01T17:33:56.827ZRun Repair workload
146762024-02-01T17:33:56.829Z01/30 Write block 323 len 4096 data: 11
146772024-02-01T17:33:56.829Z02/30 Write block 401 len 32768 data: 7 8 11 11 8 9 11 13
146782024-02-01T17:33:56.829Z03/30 Read block 539 len 24576
146792024-02-01T17:33:56.838Z04/30 Read block 408 len 4096
146802024-02-01T17:33:56.838Z05/30 Write block 243 len 40960 data: 9 9 9 9 9 10 7 8 8 8
146812024-02-01T17:33:56.838Z06/30 Read block 522 len 20480
146822024-02-01T17:33:56.838Z07/30 Read block 243 len 32768
146832024-02-01T17:33:56.838Z08/30 Write block 170 len 16384 data: 18 19 17 15
146842024-02-01T17:33:56.838Z09/30 Read block 139 len 8192
146852024-02-01T17:33:56.838Z10/30 Read block 413 len 36864
146862024-02-01T17:33:56.838Z11/30 Write block 261 len 12288 data: 11 12 14
146872024-02-01T17:33:56.838Z12/30 Flush
146882024-02-01T17:33:56.838Z13/30 Write block 267 len 28672 data: 15 14 12 12 9 8 8
146892024-02-01T17:33:57.262Z14/30 Read block 355 len 8192
146902024-02-01T17:33:57.262Z15/30 Read block 66 len 20480
146912024-02-01T17:33:57.263Z16/30 Write block 200 len 12288 data: 10 11 14
146922024-02-01T17:33:57.263Z17/30 Write block 289 len 24576 data: 13 10 12 10 11 11
146932024-02-01T17:33:57.263Z18/30 Read block 259 len 4096
146942024-02-01T17:33:57.263Z19/30 Read block 130 len 36864
146952024-02-01T17:33:57.263Z20/30 Flush
146962024-02-01T17:33:57.269Z21/30 Flush
146972024-02-01T17:33:57.269Z22/30 Write block 329 len 12288 data: 9 11 14
146982024-02-01T17:33:57.269Z23/30 Write block 410 len 8192 data: 6 5
146992024-02-01T17:33:57.269Z24/30 Flush
147002024-02-01T17:33:57.269Z25/30 Flush
147012024-02-01T17:33:57.269Z26/30 Write block 288 len 4096 data: 12
147022024-02-01T17:33:57.269Z27/30 Read block 229 len 20480
147032024-02-01T17:33:57.269Z28/30 Write block 220 len 28672 data: 15 13 13 13 12 9 8
147042024-02-01T17:33:57.269Z29/30 Read block 435 len 8192
147052024-02-01T17:33:57.269Z30/30 Read block 279 len 40960
147062024-02-01T17:33:57.269Z----------------------------------------------------------------
147072024-02-01T17:33:57.269Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
147082024-02-01T17:33:57.270ZStates: Active Active Active
147092024-02-01T17:33:57.270ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
147102024-02-01T17:33:57.270Z 1 Acked 1000 Write 1 Done Done Done false
147112024-02-01T17:33:57.270Z 2 Acked 1001 Write 8 Done Done Done false
147122024-02-01T17:33:57.270Z 3 Acked 1002 Read 6 Done Done Done false
147132024-02-01T17:33:57.270Z 4 Acked 1003 Read 1 Done Done Done false
147142024-02-01T17:33:57.270Z 5 Acked 1004 Write 10 Done Done Done false
147152024-02-01T17:33:57.270Z 6 Acked 1005 Read 5 Done Done Done false
147162024-02-01T17:33:57.270Z 7 Acked 1006 Read 8 Done Done Done false
147172024-02-01T17:33:57.270Z 8 Acked 1007 Write 4 Done Done Done false
147182024-02-01T17:33:57.270Z 9 Acked 1008 Read 2 Done Sent Done false
147192024-02-01T17:33:57.270Z 10 Acked 1009 Read 9 Done Sent Done false
147202024-02-01T17:33:57.270Z 11 Acked 1010 Write 3 Done Sent Done false
147212024-02-01T17:33:57.271Z 12 Acked 1011 Flush 0 Done Sent Done false
147222024-02-01T17:33:57.271Z 13 Acked 1012 Write 7 Done Sent Done false
147232024-02-01T17:33:57.271Z 14 Acked 1013 Read 2 Done Sent Done false
147242024-02-01T17:33:57.271Z 15 Acked 1014 Read 5 Done Sent Done false
147252024-02-01T17:33:57.271Z 16 Acked 1015 Write 3 Done Sent Done false
147262024-02-01T17:33:57.272Z 17 Acked 1016 Write 6 Done Sent Done false
147272024-02-01T17:33:57.272Z 18 Acked 1017 Read 1 Done Sent Done false
147282024-02-01T17:33:57.272Z 19 Acked 1018 Read 9 Done Sent Done false
147292024-02-01T17:33:57.272Z 20 Acked 1019 Flush 0 Done Sent Done false
147302024-02-01T17:33:57.272Z 21 Acked 1020 Flush 0 Done Sent Done false
147312024-02-01T17:33:57.273Z 22 Acked 1021 Write 3 Done Sent Done false
147322024-02-01T17:33:57.273Z 23 Acked 1022 Write 2 Done Sent Done false
147332024-02-01T17:33:57.273Z 24 Acked 1023 Flush 0 Done Sent Done false
147342024-02-01T17:33:57.273Z 25 Acked 1024 Flush 0 Done Sent Done false
147352024-02-01T17:33:57.273Z 26 Acked 1025 Write 1 Done Sent Done false
147362024-02-01T17:33:57.273Z 27 Acked 1026 Read 5 Done Sent Done false
147372024-02-01T17:33:57.273Z 28 Acked 1027 Write 7 Done Sent Done false
147382024-02-01T17:33:57.274Z 29 Acked 1028 Read 2 Done Sent Done false
147392024-02-01T17:33:57.274Z 30 Acked 1029 Read 10 Done Sent Done false
147402024-02-01T17:33:57.274Z STATES DS:0 DS:1 DS:2 TOTAL
147412024-02-01T17:33:57.274Z New 0 0 0 0
147422024-02-01T17:33:57.274Z Sent 0 22 0 22
147432024-02-01T17:33:57.274Z Done 30 8 30 68
147442024-02-01T17:33:57.274Z Skipped 0 0 0 0
147452024-02-01T17:33:57.274Z Error 0 0 0 0
147462024-02-01T17:33:57.274ZLast Flush: 1024 0 1024
147472024-02-01T17:33:57.274ZDownstairs last five completed:
147482024-02-01T17:33:57.274ZUpstairs last five completed: 30 29 28 27 26
147492024-02-01T17:33:57.275Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:33:56.137435939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1673}
147502024-02-01T17:33:57.275ZWrote out file "/tmp/test_repair_verify.data"
147512024-02-01T17:33:57.275Z
147522024-02-01T17:33:57.275ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
147532024-02-01T17:33:57.275ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147542024-02-01T17:33:57.276Z 0 000-019 135 135 135 215 215 215 F F F
147552024-02-01T17:33:57.276Z 1 020-039 132 132 132 207 207 207 F F F
147562024-02-01T17:33:57.276Z 2 040-059 132 132 132 207 207 207 F F F
147572024-02-01T17:33:57.276Z 3 060-079 132 132 132 207 207 207 F F F
147582024-02-01T17:33:57.276Z 4 080-099 133 133 133 210 210 210 F F F
147592024-02-01T17:33:57.276Z 5 100-119 134 134 134 214 214 214 F F F
147602024-02-01T17:33:57.276Z 6 120-139 134 134 134 214 214 214 F F F
147612024-02-01T17:33:57.276Z 7 140-159 132 132 132 207 207 207 F F F
147622024-02-01T17:33:57.276Z 8 160-179 136 135 136 216 215 216 F T F <---
147632024-02-01T17:33:57.276Z 9 180-199 135 135 135 215 215 215 F F F
147642024-02-01T17:33:57.276Z 10 200-219 136 134 136 217 211 217 F F F <---
147652024-02-01T17:33:57.276Z 11 220-239 122 122 122 190 190 190 T F T <---
147662024-02-01T17:33:57.276Z 12 240-259 136 128 136 216 204 216 F T F <---
147672024-02-01T17:33:57.277Z 13 260-279 136 134 136 217 212 217 F F F <---
147682024-02-01T17:33:57.277Z 14 280-299 136 135 136 217 215 217 T F T <---
147692024-02-01T17:33:57.277Z 15 300-319 135 135 135 215 215 215 F F F
147702024-02-01T17:33:57.277Z 16 320-339 136 133 136 219 210 219 F T F <---
147712024-02-01T17:33:57.277Z 17 340-359 132 132 132 207 207 207 F F F
147722024-02-01T17:33:57.277Z 18 360-379 126 126 126 200 200 200 F F F
147732024-02-01T17:33:57.279Z 19 380-399 135 135 135 215 215 215 F F F
147742024-02-01T17:33:57.279Z 20 400-419 136 129 136 219 205 219 F T F <---
147752024-02-01T17:33:57.279Z 21 420-439 135 135 135 215 215 215 F F F
147762024-02-01T17:33:57.279Z 22 440-459 135 135 135 215 215 215 F F F
147772024-02-01T17:33:57.279Z 23 460-479 135 135 135 215 215 215 F F F
147782024-02-01T17:33:57.279Z 24 480-499 121 121 121 188 188 188 F F F
147792024-02-01T17:33:57.279Z 25 500-519 119 119 119 185 185 185 F F F
147802024-02-01T17:33:57.280Z 26 520-539 127 127 127 202 202 202 F F F
147812024-02-01T17:33:57.280Z 27 540-559 130 130 130 206 206 206 F F F
147822024-02-01T17:33:57.280Z 28 560-579 133 133 133 210 210 210 F F F
147832024-02-01T17:33:57.280Z 29 580-599 133 133 133 210 210 210 F F F
147842024-02-01T17:33:57.280ZMax gen: 136, Max flush: 219
147852024-02-01T17:33:57.280ZError: Difference in extent metadata found!
147862024-02-01T17:33:57.280ZA Difference in extent metadata is expected here
147872024-02-01T17:33:57.280ZOn loop 68
147882024-02-01T17:33:57.280Z
147892024-02-01T17:33:57.280ZVerifying data now
147902024-02-01T17:34:02.653ZNo change after verify
147912024-02-01T17:34:02.653ZLoop: 68 Downstairs dump after verify (and repair):
147922024-02-01T17:34:02.679ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
147932024-02-01T17:34:02.679Z 0 000-019 135 135 135 215 215 215 F F F
147942024-02-01T17:34:02.679Z 1 020-039 132 132 132 207 207 207 F F F
147952024-02-01T17:34:02.679Z 2 040-059 132 132 132 207 207 207 F F F
147962024-02-01T17:34:02.679Z 3 060-079 132 132 132 207 207 207 F F F
147972024-02-01T17:34:02.696Z 4 080-099 133 133 133 210 210 210 F F F
147982024-02-01T17:34:02.697Z 5 100-119 134 134 134 214 214 214 F F F
147992024-02-01T17:34:02.697Z 6 120-139 134 134 134 214 214 214 F F F
148002024-02-01T17:34:02.697Z 7 140-159 132 132 132 207 207 207 F F F
148012024-02-01T17:34:02.697Z 8 160-179 136 136 136 216 216 216 F F F
148022024-02-01T17:34:02.697Z 9 180-199 135 135 135 215 215 215 F F F
148032024-02-01T17:34:02.697Z 10 200-219 136 136 136 217 217 217 F F F
148042024-02-01T17:34:02.697Z 11 220-239 137 137 137 220 220 220 F F F
148052024-02-01T17:34:02.697Z 12 240-259 136 136 136 216 216 216 F F F
148062024-02-01T17:34:02.697Z 13 260-279 136 136 136 217 217 217 F F F
148072024-02-01T17:34:02.697Z 14 280-299 137 137 137 220 220 220 F F F
148082024-02-01T17:34:02.698Z 15 300-319 135 135 135 215 215 215 F F F
148092024-02-01T17:34:02.698Z 16 320-339 136 136 136 219 219 219 F F F
148102024-02-01T17:34:02.698Z 17 340-359 132 132 132 207 207 207 F F F
148112024-02-01T17:34:02.698Z 18 360-379 126 126 126 200 200 200 F F F
148122024-02-01T17:34:02.699Z 19 380-399 135 135 135 215 215 215 F F F
148132024-02-01T17:34:02.699Z 20 400-419 136 136 136 219 219 219 F F F
148142024-02-01T17:34:02.699Z 21 420-439 135 135 135 215 215 215 F F F
148152024-02-01T17:34:02.699Z 22 440-459 135 135 135 215 215 215 F F F
148162024-02-01T17:34:02.700Z 23 460-479 135 135 135 215 215 215 F F F
148172024-02-01T17:34:02.700Z 24 480-499 121 121 121 188 188 188 F F F
148182024-02-01T17:34:02.700Z 25 500-519 119 119 119 185 185 185 F F F
148192024-02-01T17:34:02.700Z 26 520-539 127 127 127 202 202 202 F F F
148202024-02-01T17:34:02.700Z 27 540-559 130 130 130 206 206 206 F F F
148212024-02-01T17:34:02.701Z 28 560-579 133 133 133 210 210 210 F F F
148222024-02-01T17:34:02.701Z 29 580-599 133 133 133 210 210 210 F F F
148232024-02-01T17:34:02.701ZMax gen: 137, Max flush: 220
148242024-02-01T17:34:02.701Z
148252024-02-01T17:34:02.702ZBegin loop 69
148262024-02-01T17:34:02.702ZDownstairs to restart: 2
148272024-02-01T17:34:02.725Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.98591225Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
148282024-02-01T17:34:02.725Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.986557592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
148292024-02-01T17:34:02.725Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.98660828Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
148302024-02-01T17:34:02.725Z{"msg":"Crucible a8364dbf-ef95-48ee-95bc-1c906e56ae31 has session id: 52b25753-f778-41f3-9411-c0de79d15d88","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.986659802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"Crucible runtime is spawned
148312024-02-01T17:34:02.731Z52b25753-f778-41f3-9411-c0de79d15d88"}
148322024-02-01T17:34:02.731Z{"msg":"Upstairs opts: Upstairs UUID: a8364dbf-ef95-48ee-95bc-1c906e56ae31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.986784195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148332024-02-01T17:34:02.731Z{"msg":"Crucible stats registered with UUID: a8364dbf-ef95-48ee-95bc-1c906e56ae31","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.986861973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148342024-02-01T17:34:02.731Z{"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.98692542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
148352024-02-01T17:34:02.731Z{"msg":"a8364dbf-ef95-48ee-95bc-1c906e56ae31 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987155449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148362024-02-01T17:34:02.731Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987204933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148372024-02-01T17:34:02.731Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987237905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148382024-02-01T17:34:02.732Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987280795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148392024-02-01T17:34:02.732Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987305835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148402024-02-01T17:34:02.732Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987327861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148412024-02-01T17:34:02.732Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987349814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148422024-02-01T17:34:02.732Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987373634Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148432024-02-01T17:34:02.732Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987456712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148442024-02-01T17:34:02.732Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987546821Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148452024-02-01T17:34:02.733Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987619628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148462024-02-01T17:34:02.733Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.987664242Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148472024-02-01T17:34:02.733Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.988027251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"io task","client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148482024-02-01T17:34:02.733Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.988785583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148492024-02-01T17:34:02.733Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.988845734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148502024-02-01T17:34:02.733Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.988892615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148512024-02-01T17:34:02.735Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.989003541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148522024-02-01T17:34:02.735Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.989691745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148532024-02-01T17:34:02.735Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990011032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148542024-02-01T17:34:02.736Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990082002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148552024-02-01T17:34:02.736Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990131882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148562024-02-01T17:34:02.736Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990557474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148572024-02-01T17:34:02.736Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990623288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148582024-02-01T17:34:02.736Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990650808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148592024-02-01T17:34:02.736Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.990979117Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148602024-02-01T17:34:02.737Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991042251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148612024-02-01T17:34:02.737Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991069759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148622024-02-01T17:34:02.737Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991187735Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148632024-02-01T17:34:02.750Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991818047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148642024-02-01T17:34:02.750Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991879743Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148652024-02-01T17:34:02.751Z{"msg":"[0]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 216, 215, 217, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.99190717Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148662024-02-01T17:34:02.751Z{"msg":"[0]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 136, 135, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991939466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148672024-02-01T17:34:02.751Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.991977354Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148682024-02-01T17:34:02.751Z{"msg":"[1]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 216, 215, 217, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992016352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148692024-02-01T17:34:02.752Z{"msg":"[1]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 136, 135, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992046566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148702024-02-01T17:34:02.753Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.99209833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148712024-02-01T17:34:02.753Z{"msg":"[2]R flush_numbers[0..12]: [215, 207, 207, 207, 210, 214, 214, 207, 216, 215, 217, 220]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992124189Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"Wait for a query_work_queue command to finish before sending IO
148722024-02-01T17:34:02.753Zdownstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148732024-02-01T17:34:02.753Z{"msg":"[2]R generation[0..12]: [135, 132, 132, 132, 133, 134, 134, 132, 136, 135, 136, 137]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992152812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148742024-02-01T17:34:02.754Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992200288Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148752024-02-01T17:34:02.754Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.99222427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148762024-02-01T17:34:02.754Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992247344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148772024-02-01T17:34:02.754Z{"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992286512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148782024-02-01T17:34:02.754Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992310202Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"":"downstairs","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148792024-02-01T17:34:02.754Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992352794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148802024-02-01T17:34:02.755Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992374876Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"0","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148812024-02-01T17:34:02.755Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992396956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"1","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148822024-02-01T17:34:02.756Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992436564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"client":"2","":"downstairs","session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148832024-02-01T17:34:02.756Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992460333Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148842024-02-01T17:34:02.756Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992481182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148852024-02-01T17:34:02.756Z{"msg":"a8364dbf-ef95-48ee-95bc-1c906e56ae31 is now active with session: 52b25753-f778-41f3-9411-c0de79d15d88","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.99253106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148862024-02-01T17:34:02.756Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992552573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681,"session_id":"52b25753-f778-41f3-9411-c0de79d15d88"}
148872024-02-01T17:34:02.756Z{"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:01.992573322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
148882024-02-01T17:34:02.757ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148892024-02-01T17:34:02.757ZLoading write count information from file "/tmp/test_repair_verify.data"
148902024-02-01T17:34:02.757ZRun Repair workload
148912024-02-01T17:34:02.757Z01/30 Write block 153 len 32768 data: 11 9 12 9 9 10 9 10
148922024-02-01T17:34:02.757Z02/30 Read block 560 len 28672
148932024-02-01T17:34:02.758Z03/30 Write block 1 len 24576 data: 5 6 9 11 10 11
148942024-02-01T17:34:02.758Z04/30 Read block 185 len 28672
148952024-02-01T17:34:02.758Z05/30 Read block 359 len 40960
148962024-02-01T17:34:02.758Z06/30 Flush
148972024-02-01T17:34:02.758Z07/30 Write block 100 len 36864 data: 11 12 12 10 11 14 15 16 14
148982024-02-01T17:34:02.758Z08/30 Write block 554 len 4096 data: 8
148992024-02-01T17:34:02.758Z09/30 Write block 16 len 16384 data: 7 10 12 10
149002024-02-01T17:34:02.758Z10/30 Read block 497 len 24576
149012024-02-01T17:34:02.758Z11/30 Read block 211 len 20480
149022024-02-01T17:34:02.758Z12/30 Write block 321 len 36864 data: 13 14 12 12 6 7 9 8 10
149032024-02-01T17:34:02.758Z13/30 Write block 401 len 12288 data: 8 9 12
149042024-02-01T17:34:02.758Z14/30 Read block 234 len 8192
149052024-02-01T17:34:02.758Z15/30 Read block 486 len 40960
149062024-02-01T17:34:02.759Z16/30 Write block 497 len 16384 data: 8 10 10 9
149072024-02-01T17:34:02.759Z17/30 Flush
149082024-02-01T17:34:02.763Z18/30 Write block 562 len 20480 data: 10 9 8 10 9
149092024-02-01T17:34:02.763Z19/30 Write block 544 len 4096 data: 15
149102024-02-01T17:34:02.763Z20/30 Write block 229 len 16384 data: 10 8 9 8
149112024-02-01T17:34:02.763Z21/30 Write block 127 len 28672 data: 10 11 11 11 10 7 11
149122024-02-01T17:34:02.763Z22/30 Read block 372 len 28672
149132024-02-01T17:34:02.763Z23/30 Write block 547 len 4096 data: 14
149142024-02-01T17:34:02.764Z24/30 Flush
149152024-02-01T17:34:02.764Z25/30 Write block 35 len 24576 data: 10 12 13 14 13 9
149162024-02-01T17:34:02.764Z26/30 Read block 573 len 36864
149172024-02-01T17:34:02.764Z27/30 Read block 557 len 16384
149182024-02-01T17:34:02.764Z28/30 Read block 258 len 32768
149192024-02-01T17:34:02.765Z29/30 Read block 89 len 32768
149202024-02-01T17:34:02.765Z30/30 Read block 453 len 28672
149212024-02-01T17:34:02.775Z----------------------------------------------------------------
149222024-02-01T17:34:02.775Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:24
149232024-02-01T17:34:02.775ZStates: Active Active Active
149242024-02-01T17:34:02.777ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
149252024-02-01T17:34:02.777Z 7 Acked 1006 Write 9 Done Done Done false
149262024-02-01T17:34:02.777Z 8 Acked 1007 Write 1 Done Done Sent false
149272024-02-01T17:34:02.777Z 9 Acked 1008 Write 4 Done Done Sent false
149282024-02-01T17:34:02.777Z 10 Acked 1009 Read 6 Done Done Sent false
149292024-02-01T17:34:02.777Z 11 Acked 1010 Read 5 Done Done Sent false
149302024-02-01T17:34:02.778Z 12 Acked 1011 Write 9 Done Done Sent false
149312024-02-01T17:34:02.778Z 13 Acked 1012 Write 3 Done Done Sent false
149322024-02-01T17:34:02.778Z 14 Acked 1013 Read 2 Done Done Sent false
149332024-02-01T17:34:02.778Z 15 Acked 1014 Read 10 Done Done Sent false
149342024-02-01T17:34:02.778Z 16 Acked 1015 Write 4 Done Done Sent false
149352024-02-01T17:34:02.778Z 17 Acked 1016 Flush 0 Done Done Sent false
149362024-02-01T17:34:02.778Z 18 Acked 1017 Write 5 Done Done Sent false
149372024-02-01T17:34:02.778Z 19 Acked 1018 Write 1 Done Done Sent false
149382024-02-01T17:34:02.778Z 20 Acked 1019 Write 4 Done Done Sent false
149392024-02-01T17:34:02.778Z 21 Acked 1020 Write 7 Done Done Sent false
149402024-02-01T17:34:02.779Z 22 Acked 1021 Read 7 Done Done Sent false
149412024-02-01T17:34:02.779Z 23 Acked 1022 Write 1 Done Done Sent false
149422024-02-01T17:34:02.779Z 24 Acked 1023 Flush 0 Done Done Sent false
149432024-02-01T17:34:02.779Z 25 Acked 1024 Write 6 Done Done Sent false
149442024-02-01T17:34:02.779Z 26 Acked 1025 Read 9 Done Done Sent false
149452024-02-01T17:34:02.779Z 27 Acked 1026 Read 4 Done Done Sent false
149462024-02-01T17:34:02.779Z 28 Acked 1027 Read 8 Done Done Sent false
149472024-02-01T17:34:02.780Z 29 Acked 1028 Read 8 Done Done Sent false
149482024-02-01T17:34:02.780Z 30 Acked 1029 Read 7 Done Done Sent false
149492024-02-01T17:34:02.780Z STATES DS:0 DS:1 DS:2 TOTAL
149502024-02-01T17:34:02.780Z New 0 0 0 0
149512024-02-01T17:34:02.780Z Sent 0 0 23 23
149522024-02-01T17:34:02.780Z Done 24 24 1 49
149532024-02-01T17:34:02.780Z Skipped 0 0 0 0
149542024-02-01T17:34:02.780Z Error 0 0 0 0
149552024-02-01T17:34:02.780ZLast Flush: 1023 1023 1005
149562024-02-01T17:34:02.781ZDownstairs last five completed: 1005 1004 1003 1002 1001
149572024-02-01T17:34:02.781ZUpstairs last five completed: 30 29 28 27 26
149582024-02-01T17:34:02.781Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:02.042847586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1681}
149592024-02-01T17:34:02.781ZWrote out file "/tmp/test_repair_verify.data"
149602024-02-01T17:34:02.781Z
149612024-02-01T17:34:02.789ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
149622024-02-01T17:34:02.803ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149632024-02-01T17:34:02.803Z 0 000-019 138 138 138 222 222 221 F F F <---
149642024-02-01T17:34:02.803Z 1 020-039 132 132 132 207 207 207 T T F <---
149652024-02-01T17:34:02.803Z 2 040-059 132 132 132 207 207 207 T T F <---
149662024-02-01T17:34:02.804Z 3 060-079 132 132 132 207 207 207 F F F
149672024-02-01T17:34:02.804Z 4 080-099 133 133 133 210 210 210 F F F
149682024-02-01T17:34:02.804Z 5 100-119 138 138 134 222 222 214 F F T <---
149692024-02-01T17:34:02.805Z 6 120-139 138 138 134 223 223 214 F F F <---
149702024-02-01T17:34:02.805Z 7 140-159 138 138 138 221 221 221 F F F
149712024-02-01T17:34:02.812Z 8 160-179 138 138 138 221 221 221 F F F
149722024-02-01T17:34:02.812Z 9 180-199 135 135 135 215 215 215 F F F
149732024-02-01T17:34:02.812Z 10 200-219 136 136 136 217 217 217 F F F
149742024-02-01T17:34:02.812Z 11 220-239 138 138 137 223 223 220 F F F <---
149752024-02-01T17:34:02.813Z 12 240-259 136 136 136 216 216 216 F F F
149762024-02-01T17:34:02.813Z 13 260-279 136 136 136 217 217 217 F F F
149772024-02-01T17:34:02.813Z 14 280-299 137 137 137 220 220 220 F F F
149782024-02-01T17:34:02.813Z 15 300-319 135 135 135 215 215 215 F F F
149792024-02-01T17:34:02.813Z 16 320-339 138 138 136 222 222 219 F F F <---
149802024-02-01T17:34:02.813Z 17 340-359 132 132 132 207 207 207 F F F
149812024-02-01T17:34:02.813Z 18 360-379 126 126 126 200 200 200 F F F
149822024-02-01T17:34:02.814Z 19 380-399 135 135 135 215 215 215 F F F
149832024-02-01T17:34:02.814Z 20 400-419 138 138 136 222 222 219 F F F <---
149842024-02-01T17:34:02.814Z 21 420-439 135 135 135 215 215 215 F F F
149852024-02-01T17:34:02.814Z 22 440-459 135 135 135 215 215 215 F F F
149862024-02-01T17:34:02.814Z 23 460-479 135 135 135 215 215 215 F F F
149872024-02-01T17:34:02.814Z 24 480-499 138 138 121 222 222 188 F F F <---
149882024-02-01T17:34:02.814Z 25 500-519 138 138 119 222 222 185 F F F <---
149892024-02-01T17:34:02.814Z 26 520-539 127 127 127 202 202 202 F F F
149902024-02-01T17:34:02.815Z 27 540-559 138 138 130 223 223 206 F F F <---
149912024-02-01T17:34:02.815Z 28 560-579 138 138 133 223 223 210 F F F <---
149922024-02-01T17:34:02.816Z 29 580-599 133 133 133 210 210 210 F F F
149932024-02-01T17:34:02.816ZMax gen: 138, Max flush: 223
149942024-02-01T17:34:02.816ZError: Difference in extent metadata found!
149952024-02-01T17:34:02.816ZA Difference in extent metadata is expected here
149962024-02-01T17:34:02.816ZOn loop 69
149972024-02-01T17:34:02.816Z
149982024-02-01T17:34:02.816ZVerifying data now
149992024-02-01T17:34:03.856ZNo change after verify
150002024-02-01T17:34:03.856ZLoop: 69 Downstairs dump after verify (and repair):
150012024-02-01T17:34:03.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150022024-02-01T17:34:03.876Z 0 000-019 138 138 138 222 222 222 F F F
150032024-02-01T17:34:03.876Z 1 020-039 139 139 139 224 224 224 F F F
150042024-02-01T17:34:03.876Z 2 040-059 139 139 139 224 224 224 F F F
150052024-02-01T17:34:03.877Z 3 060-079 132 132 132 207 207 207 F F F
150062024-02-01T17:34:03.889Z 4 080-099 133 133 133 210 210 210 F F F
150072024-02-01T17:34:03.889Z 5 100-119 138 138 138 222 222 222 F F F
150082024-02-01T17:34:03.889Z 6 120-139 138 138 138 223 223 223 F F F
150092024-02-01T17:34:03.890Z 7 140-159 138 138 138 221 221 221 F F F
150102024-02-01T17:34:03.890Z 8 160-179 138 138 138 221 221 221 F F F
150112024-02-01T17:34:03.890Z 9 180-199 135 135 135 215 215 215 F F F
150122024-02-01T17:34:03.890Z 10 200-219 136 136 136 217 217 217 F F F
150132024-02-01T17:34:03.891Z 11 220-239 138 138 138 223 223 223 F F F
150142024-02-01T17:34:03.891Z 12 240-259 136 136 136 216 216 216 F F F
150152024-02-01T17:34:03.891Z 13 260-279 136 136 136 217 217 217 F F F
150162024-02-01T17:34:03.891Z 14 280-299 137 137 137 220 220 220 F F F
150172024-02-01T17:34:03.891Z 15 300-319 135 135 135 215 215 215 F F F
150182024-02-01T17:34:03.892Z 16 320-339 138 138 138 222 222 222 F F F
150192024-02-01T17:34:03.892Z 17 340-359 132 132 132 207 207 207 F F F
150202024-02-01T17:34:03.892Z 18 360-379 126 126 126 200 200 200 F F F
150212024-02-01T17:34:03.892Z 19 380-399 135 135 135 215 215 215 F F F
150222024-02-01T17:34:03.892Z 20 400-419 138 138 138 222 222 222 F F F
150232024-02-01T17:34:03.892Z 21 420-439 135 135 135 215 215 215 F F F
150242024-02-01T17:34:03.893Z 22 440-459 135 135 135 215 215 215 F F F
150252024-02-01T17:34:03.893Z 23 460-479 135 135 135 215 215 215 F F F
150262024-02-01T17:34:03.893Z 24 480-499 138 138 138 222 222 222 F F F
150272024-02-01T17:34:03.894Z 25 500-519 138 138 138 222 222 222 F F F
150282024-02-01T17:34:03.894Z 26 520-539 127 127 127 202 202 202 F F F
150292024-02-01T17:34:03.894Z 27 540-559 138 138 138 223 223 223 F F F
150302024-02-01T17:34:03.894Z 28 560-579 138 138 138 223 223 223 F F F
150312024-02-01T17:34:03.894Z 29 580-599 133 133 133 210 210 210 F F F
150322024-02-01T17:34:03.894ZMax gen: 139, Max flush: 224
150332024-02-01T17:34:03.894Z
150342024-02-01T17:34:03.895ZBegin loop 70
150352024-02-01T17:34:03.895ZDownstairs to restart: 2
150362024-02-01T17:34:03.911Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.184870444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
150372024-02-01T17:34:03.911Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
150382024-02-01T17:34:03.912Z,"time":"2024-02-01T17:34:03.185402542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
150392024-02-01T17:34:03.934Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185457959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
150402024-02-01T17:34:03.934Z{"msg":"Crucible c85bb376-bbb9-4b7e-be66-bd1bf90fb7a2 has session id: 050852dc-2b25-42d0-9de0-61cd4b03c84a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185531306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150412024-02-01T17:34:03.934Z{"msg":"Upstairs opts: Upstairs UUID: c85bb376-bbb9-4b7e-be66-bd1bf90fb7a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185583732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150422024-02-01T17:34:03.934Z{"msg":"Crucible stats registered with UUID: c85bb376-bbb9-4b7e-be66-bd1bf90fb7a2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185649796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150432024-02-01T17:34:03.934Z{"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185731272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
150442024-02-01T17:34:03.934Z{"msg":"c85bb376-bbb9-4b7e-be66-bd1bf90fb7a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.185854439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150452024-02-01T17:34:03.934Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.1859039Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150462024-02-01T17:34:03.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186006832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150472024-02-01T17:34:03.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186054596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150482024-02-01T17:34:03.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186150935Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150492024-02-01T17:34:03.935Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186255169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150502024-02-01T17:34:03.935Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186339775Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150512024-02-01T17:34:03.935Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186387947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150522024-02-01T17:34:03.935Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186514606Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150532024-02-01T17:34:03.935Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186565626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150542024-02-01T17:34:03.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.1866407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150552024-02-01T17:34:03.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186688201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150562024-02-01T17:34:03.935Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.186732469Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"io task","client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150572024-02-01T17:34:03.935Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.187719886Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150582024-02-01T17:34:03.935Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.187802221Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150592024-02-01T17:34:03.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.18790546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150602024-02-01T17:34:03.936Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.187963071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150612024-02-01T17:34:03.936Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.188705507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150622024-02-01T17:34:03.936Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.188812326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150632024-02-01T17:34:03.936Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.188852938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150642024-02-01T17:34:03.936Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.189037886Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150652024-02-01T17:34:03.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.18979058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150662024-02-01T17:34:03.936Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.189896545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150672024-02-01T17:34:03.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.189927654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150682024-02-01T17:34:03.936Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.189991605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150692024-02-01T17:34:03.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.190018257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150702024-02-01T17:34:03.936Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.190041148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150712024-02-01T17:34:03.936Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.190063678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150722024-02-01T17:34:03.936Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.190884668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150732024-02-01T17:34:03.936Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.190945235Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150742024-02-01T17:34:03.936Z{"msg":"[0]R flush_numbers[0..12]: [222, 224, 224, 207, 210, 222, 223, 221, 221, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191023587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150752024-02-01T17:34:03.936Z{"msg":"[0]R generation[0..12]: [138, 139, 139, 132, 133, 138, 138, 138, 138, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191071179Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150762024-02-01T17:34:03.936Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191103077Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150772024-02-01T17:34:03.936Z{"msg":"[1]R flush_numbers[0..12]: [222, 224, 224, 207, 210, 222, 223, 221, 221, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191159029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150782024-02-01T17:34:03.936Z{"msg":"[1]R generation[0..12]: [138, 139, 139, 132, 133, 138, 138, 138, 138, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191184843Z","hostname":"Wait for a query_work_queue command to finish before sending IO
150792024-02-01T17:34:03.936Zip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150802024-02-01T17:34:03.936Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191219917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150812024-02-01T17:34:03.936Z{"msg":"[2]R flush_numbers[0..12]: [222, 224, 224, 207, 210, 222, 223, 221, 221, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191279724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150822024-02-01T17:34:03.936Z{"msg":"[2]R generation[0..12]: [138, 139, 139, 132, 133, 138, 138, 138, 138, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.1913164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150832024-02-01T17:34:03.937Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191375367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150842024-02-01T17:34:03.937Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191418444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150852024-02-01T17:34:03.937Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191442809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150862024-02-01T17:34:03.937Z{"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191488338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150872024-02-01T17:34:03.937Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191514492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"":"downstairs","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150882024-02-01T17:34:03.937Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191565411Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150892024-02-01T17:34:03.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191587721Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"0","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150902024-02-01T17:34:03.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191609883Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"1","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150912024-02-01T17:34:03.937Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191650151Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"client":"2","":"downstairs","session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150922024-02-01T17:34:03.937Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.19169441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150932024-02-01T17:34:03.937Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191757502Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150942024-02-01T17:34:03.937Z{"msg":"c85bb376-bbb9-4b7e-be66-bd1bf90fb7a2 is now active with session: 050852dc-2b25-42d0-9de0-61cd4b03c84a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191784744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150952024-02-01T17:34:03.937Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191839143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689,"session_id":"050852dc-2b25-42d0-9de0-61cd4b03c84a"}
150962024-02-01T17:34:03.937Z{"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:03.191881445Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
150972024-02-01T17:34:03.937ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150982024-02-01T17:34:03.937ZLoading write count information from file "/tmp/test_repair_verify.data"
150992024-02-01T17:34:03.937ZRun Repair workload
151002024-02-01T17:34:03.937Z01/30 Write block 159 len 28672 data: 10 11 13 13 13 13 10
151012024-02-01T17:34:03.937Z02/30 Write block 125 len 4096 data: 11
151022024-02-01T17:34:03.937Z03/30 Read block 358 len 36864
151032024-02-01T17:34:03.947Z04/30 Read block 454 len 28672
151042024-02-01T17:34:03.947Z05/30 Write block 408 len 40960 data: 14 10 7 6 7 7 10 12 15 9
151052024-02-01T17:34:03.947Z06/30 Read block 348 len 32768
151062024-02-01T17:34:03.948Z07/30 Read block 488 len 28672
151072024-02-01T17:34:03.948Z08/30 Flush
151082024-02-01T17:34:03.948Z09/30 Write block 487 len 24576 data: 11 11 11 12 12 14
151092024-02-01T17:34:03.948Z10/30 Read block 289 len 16384
151102024-02-01T17:34:03.949Z11/30 Flush
151112024-02-01T17:34:03.949Z12/30 Write block 368 len 36864 data: 8 8 11 14 13 12 11 12 14
151122024-02-01T17:34:03.949Z13/30 Read block 343 len 8192
151132024-02-01T17:34:03.949Z14/30 Write block 349 len 12288 data: 11 9 10
151142024-02-01T17:34:03.949Z15/30 Read block 294 len 24576
151152024-02-01T17:34:03.949Z16/30 Read block 368 len 24576
151162024-02-01T17:34:03.949Z17/30 Read block 124 len 24576
151172024-02-01T17:34:03.949Z18/30 Read block 228 len 24576
151182024-02-01T17:34:03.960Z19/30 Write block 330 len 28672 data: 12 15 13 13 14 13 13
151192024-02-01T17:34:03.960Z20/30 Read block 10 len 8192
151202024-02-01T17:34:03.960Z21/30 Write block 290 len 4096 data: 11
151212024-02-01T17:34:03.960Z22/30 Read block 288 len 8192
151222024-02-01T17:34:03.960Z23/30 Write block 497 len 12288 data: 9 11 11
151232024-02-01T17:34:03.960Z24/30 Write block 497 len 20480 data: 10 12 12 10 9
151242024-02-01T17:34:03.960Z25/30 Read block 155 len 8192
151252024-02-01T17:34:03.960Z26/30 Write block 0 len 32768 data: 4 6 7 10 12 11 12 10
151262024-02-01T17:34:03.960Z27/30 Read block 314 len 16384
151272024-02-01T17:34:03.960Z28/30 Write block 293 len 40960 data: 12 12 11 12 11 14 13 13 10 8
151282024-02-01T17:34:03.983Z29/30 Write block 287 len 32768 data: 11 13 14 12 13 11 13 13
151292024-02-01T17:34:03.984Z30/30 Write block 46 len 4096 data: 13
151302024-02-01T17:34:03.984Z----------------------------------------------------------------
151312024-02-01T17:34:03.984Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:19
151322024-02-01T17:34:03.984ZStates: Active Active Active
151332024-02-01T17:34:03.984ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151342024-02-01T17:34:03.984Z 12 Acked 1011 Write 9 Done Done Sent false
151352024-02-01T17:34:03.984Z 13 Acked 1012 Read 2 Done Done Sent false
151362024-02-01T17:34:03.984Z 14 Acked 1013 Write 3 Done Done Sent false
151372024-02-01T17:34:03.984Z 15 Acked 1014 Read 6 Done Done Sent false
151382024-02-01T17:34:03.985Z 16 Acked 1015 Read 6 Done Done Sent false
151392024-02-01T17:34:03.985Z 17 Acked 1016 Read 6 Done Done Sent false
151402024-02-01T17:34:03.985Z 18 Acked 1017 Read 6 Done Done Sent false
151412024-02-01T17:34:03.985Z 19 Acked 1018 Write 7 Done Done Sent false
151422024-02-01T17:34:03.985Z 20 Acked 1019 Read 2 Done Done Sent false
151432024-02-01T17:34:03.985Z 21 Acked 1020 Write 1 Done Done Sent false
151442024-02-01T17:34:03.985Z 22 Acked 1021 Read 2 Done Done Sent false
151452024-02-01T17:34:03.985Z 23 Acked 1022 Write 3 Done Done Sent false
151462024-02-01T17:34:03.985Z 24 Acked 1023 Write 5 Done Done Sent false
151472024-02-01T17:34:03.985Z 25 Acked 1024 Read 2 Done Done Sent false
151482024-02-01T17:34:03.985Z 26 Acked 1025 Write 8 Done Done Sent false
151492024-02-01T17:34:03.985Z 27 Acked 1026 Read 4 Done Done Sent false
151502024-02-01T17:34:03.985Z 28 Acked 1027 Write 10 Done Done Sent false
151512024-02-01T17:34:03.986Z 29 Acked 1028 Write 8 Sent Sent Sent false
151522024-02-01T17:34:03.986Z 30 Acked 1029 Write 1 Sent Sent Sent false
151532024-02-01T17:34:03.986Z STATES DS:0 DS:1 DS:2 TOTAL
151542024-02-01T17:34:03.986Z New 0 0 0 0
151552024-02-01T17:34:03.986Z Sent 2 2 19 23
151562024-02-01T17:34:03.986Z Done 17 17 0 34
151572024-02-01T17:34:03.986Z Skipped 0 0 0 0
151582024-02-01T17:34:03.986Z Error 0 0 0 0
151592024-02-01T17:34:03.986ZLast Flush: 1010 1010 1010
151602024-02-01T17:34:03.986ZDownstairs last five completed: 1010 1009 1008 1007 1006
151612024-02-01T17:34:03.987ZUpstairs last five completed: 30 29 28 27 26
151622024-02-01T17:34:03.987Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:03.238834087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1689}
151632024-02-01T17:34:03.987ZWrote out file "/tmp/test_repair_verify.data"
151642024-02-01T17:34:03.987Z
151652024-02-01T17:34:04.010ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151662024-02-01T17:34:04.010ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151672024-02-01T17:34:04.010Z 0 000-019 138 138 138 222 222 222 T T F <---
151682024-02-01T17:34:04.010Z 1 020-039 139 139 139 224 224 224 F F F
151692024-02-01T17:34:04.011Z 2 040-059 139 139 139 224 224 224 T T F <---
151702024-02-01T17:34:04.011Z 3 060-079 132 132 132 207 207 207 F F F
151712024-02-01T17:34:04.011Z 4 080-099 133 133 133 210 210 210 F F F
151722024-02-01T17:34:04.011Z 5 100-119 138 138 138 222 222 222 F F F
151732024-02-01T17:34:04.011Z 6 120-139 140 140 140 225 225 225 F F F
151742024-02-01T17:34:04.011Z 7 140-159 140 140 140 225 225 225 F F F
151752024-02-01T17:34:04.012Z 8 160-179 140 140 140 225 225 225 F F F
151762024-02-01T17:34:04.012Z 9 180-199 135 135 135 215 215 215 F F F
151772024-02-01T17:34:04.012Z 10 200-219 136 136 136 217 217 217 F F F
151782024-02-01T17:34:04.012Z 11 220-239 138 138 138 223 223 223 F F F
151792024-02-01T17:34:04.012Z 12 240-259 136 136 136 216 216 216 F F F
151802024-02-01T17:34:04.012Z 13 260-279 136 136 136 217 217 217 F F F
151812024-02-01T17:34:04.012Z 14 280-299 137 137 137 220 220 220 T T F <---
151822024-02-01T17:34:04.012Z 15 300-319 135 135 135 215 215 215 T T F <---
151832024-02-01T17:34:04.012Z 16 320-339 138 138 138 222 222 222 T T F <---
151842024-02-01T17:34:04.012Z 17 340-359 132 132 132 207 207 207 T T F <---
151852024-02-01T17:34:04.012Z 18 360-379 126 126 126 200 200 200 T T F <---
151862024-02-01T17:34:04.012Z 19 380-399 135 135 135 215 215 215 F F F
151872024-02-01T17:34:04.012Z 20 400-419 140 140 140 225 225 225 F F F
151882024-02-01T17:34:04.014Z 21 420-439 135 135 135 215 215 215 F F F
151892024-02-01T17:34:04.015Z 22 440-459 135 135 135 215 215 215 F F F
151902024-02-01T17:34:04.015Z 23 460-479 135 135 135 215 215 215 F F F
151912024-02-01T17:34:04.015Z 24 480-499 140 140 140 226 226 226 T T F <---
151922024-02-01T17:34:04.015Z 25 500-519 138 138 138 222 222 222 T T F <---
151932024-02-01T17:34:04.015Z 26 520-539 127 127 127 202 202 202 F F F
151942024-02-01T17:34:04.015Z 27 540-559 138 138 138 223 223 223 F F F
151952024-02-01T17:34:04.015Z 28 560-579 138 138 138 223 223 223 F F F
151962024-02-01T17:34:04.015Z 29 580-599 133 133 133 210 210 210 F F F
151972024-02-01T17:34:04.015ZMax gen: 140, Max flush: 226
151982024-02-01T17:34:04.016ZError: Difference in extent metadata found!
151992024-02-01T17:34:04.016ZA Difference in extent metadata is expected here
152002024-02-01T17:34:04.016ZOn loop 70
152012024-02-01T17:34:04.016Z
152022024-02-01T17:34:04.030ZVerifying data now
152032024-02-01T17:34:09.996ZNo change after verify
152042024-02-01T17:34:09.997ZLoop: 70 Downstairs dump after verify (and repair):
152052024-02-01T17:34:10.068ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152062024-02-01T17:34:10.068Z 0 000-019 141 141 141 227 227 227 F F F
152072024-02-01T17:34:10.068Z 1 020-039 139 139 139 224 224 224 F F F
152082024-02-01T17:34:10.068Z 2 040-059 141 141 141 227 227 227 F F F
152092024-02-01T17:34:10.068Z 3 060-079 132 132 132 207 207 207 F F F
152102024-02-01T17:34:10.068Z 4 080-099 133 133 133 210 210 210 F F F
152112024-02-01T17:34:10.068Z 5 100-119 138 138 138 222 222 222 F F F
152122024-02-01T17:34:10.069Z 6 120-139 140 140 140 225 225 225 F F F
152132024-02-01T17:34:10.069Z 7 140-159 140 140 140 225 225 225 F F F
152142024-02-01T17:34:10.069Z 8 160-179 140 140 140 225 225 225 F F F
152152024-02-01T17:34:10.069Z 9 180-199 135 135 135 215 215 215 F F F
152162024-02-01T17:34:10.069Z 10 200-219 136 136 136 217 217 217 F F F
152172024-02-01T17:34:10.069Z 11 220-239 138 138 138 223 223 223 F F F
152182024-02-01T17:34:10.069Z 12 240-259 136 136 136 216 216 216 F F F
152192024-02-01T17:34:10.069Z 13 260-279 136 136 136 217 217 217 F F F
152202024-02-01T17:34:10.069Z 14 280-299 141 141 141 227 227 227 F F F
152212024-02-01T17:34:10.070Z 15 300-319 141 141 141 227 227 227 F F F
152222024-02-01T17:34:10.070Z 16 320-339 141 141 141 227 227 227 F F F
152232024-02-01T17:34:10.070Z 17 340-359 141 141 141 227 227 227 F F F
152242024-02-01T17:34:10.070Z 18 360-379 141 141 141 227 227 227 F F F
152252024-02-01T17:34:10.070Z 19 380-399 135 135 135 215 215 215 F F F
152262024-02-01T17:34:10.071Z 20 400-419 140 140 140 225 225 225 F F F
152272024-02-01T17:34:10.071Z 21 420-439 135 135 135 215 215 215 F F F
152282024-02-01T17:34:10.071Z 22 440-459 135 135 135 215 215 215 F F F
152292024-02-01T17:34:10.071Z 23 460-479 135 135 135 215 215 215 F F F
152302024-02-01T17:34:10.071Z 24 480-499 141 141 141 227 227 227 F F F
152312024-02-01T17:34:10.071Z 25 500-519 141 141 141 227 227 227 F F F
152322024-02-01T17:34:10.071Z 26 520-539 127 127 127 202 202 202 F F F
152332024-02-01T17:34:10.072Z 27 540-559 138 138 138 223 223 223 F F F
152342024-02-01T17:34:10.072Z 28 560-579 138 138 138 223 223 223 F F F
152352024-02-01T17:34:10.072Z 29 580-599 133 133 133 210 210 210 F F F
152362024-02-01T17:34:10.072ZMax gen: 141, Max flush: 227
152372024-02-01T17:34:10.072Z
152382024-02-01T17:34:10.072ZBegin loop 71
152392024-02-01T17:34:10.072ZDownstairs to restart: 1
152402024-02-01T17:34:10.072Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.189107241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
152412024-02-01T17:34:10.072Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.189646745Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
152422024-02-01T17:34:10.073Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
152432024-02-01T17:34:10.073Z,"time":"2024-02-01T17:34:08.189788146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
152442024-02-01T17:34:10.074Z{"msg":"Crucible 3d9ba75f-55bb-417f-80be-c66a1028af21 has session id: 65f3667c-7893-445c-94f3-6fdf8ce6b516","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.18989925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152452024-02-01T17:34:10.074Z{"msg":"Upstairs opts: Upstairs UUID: 3d9ba75f-55bb-417f-80be-c66a1028af21, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190004931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152462024-02-01T17:34:10.074Z{"msg":"Crucible stats registered with UUID: 3d9ba75f-55bb-417f-80be-c66a1028af21","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190106114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152472024-02-01T17:34:10.074Z{"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190148604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
152482024-02-01T17:34:10.074Z{"msg":"3d9ba75f-55bb-417f-80be-c66a1028af21 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190183672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152492024-02-01T17:34:10.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190223566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152502024-02-01T17:34:10.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190308373Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152512024-02-01T17:34:10.075Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190352283Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152522024-02-01T17:34:10.075Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190427252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152532024-02-01T17:34:10.076Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190471519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152542024-02-01T17:34:10.076Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190532598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152552024-02-01T17:34:10.076Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190571567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152562024-02-01T17:34:10.076Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190670556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152572024-02-01T17:34:10.076Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190718637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152582024-02-01T17:34:10.076Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190762979Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152592024-02-01T17:34:10.076Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190804839Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152602024-02-01T17:34:10.077Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.190881977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"io task","client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152612024-02-01T17:34:10.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.192009027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152622024-02-01T17:34:10.077Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.192074047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152632024-02-01T17:34:10.077Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.192183555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152642024-02-01T17:34:10.077Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.19226688Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152652024-02-01T17:34:10.078Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.192313177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152662024-02-01T17:34:10.078Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.192387104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152672024-02-01T17:34:10.078Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193110365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152682024-02-01T17:34:10.078Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193252233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152692024-02-01T17:34:10.078Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193311594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152702024-02-01T17:34:10.078Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193837147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152712024-02-01T17:34:10.079Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193877263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152722024-02-01T17:34:10.079Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.193901888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152732024-02-01T17:34:10.079Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194178489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152742024-02-01T17:34:10.079Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194264422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152752024-02-01T17:34:10.079Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194311549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152762024-02-01T17:34:10.079Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194359673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152772024-02-01T17:34:10.080Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194384706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152782024-02-01T17:34:10.080Z{"msg":"[0]R flush_numbers[0..12]: [227, 224, 227, 207, 210, 222, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194427035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152792024-02-01T17:34:10.080Z{"msg":"[0]R generation[0..12]: [141, 139, 141, 132, 133, 138, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194452671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152802024-02-01T17:34:10.080Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194476588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152812024-02-01T17:34:10.080Z{"msg":"[1]R flush_numbers[0..12]: [227, 224, 227, 207, 210, 222, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194518639Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152822024-02-01T17:34:10.082Z{"msg":"[1]R generation[0..12]: [141, 139, 141, 132, 133, 138, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194543758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152832024-02-01T17:34:10.082Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194599221Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152842024-02-01T17:34:10.082Z{"msg":"[2]R flush_numbers[0..12]: [227, 224, 227, 207, 210, 222, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194624831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152852024-02-01T17:34:10.082Z{"msg":"[2]R generation[0..12]: [141, 139, 141, 132, 133, 138, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
152862024-02-01T17:34:10.083Z,"time":"2024-02-01T17:34:08.194647937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152872024-02-01T17:34:10.083Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194672223Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152882024-02-01T17:34:10.083Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.19471259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152892024-02-01T17:34:10.083Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194737446Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152902024-02-01T17:34:10.083Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194777349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152912024-02-01T17:34:10.083Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194800875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"":"downstairs","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152922024-02-01T17:34:10.084Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194845128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152932024-02-01T17:34:10.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194867482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"0","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152942024-02-01T17:34:10.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194889452Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"1","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152952024-02-01T17:34:10.084Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194929558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"client":"2","":"downstairs","session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152962024-02-01T17:34:10.086Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.194953488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152972024-02-01T17:34:10.087Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.195000763Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152982024-02-01T17:34:10.087Z{"msg":"3d9ba75f-55bb-417f-80be-c66a1028af21 is now active with session: 65f3667c-7893-445c-94f3-6fdf8ce6b516","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.195024665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
152992024-02-01T17:34:10.087Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.195075224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697,"session_id":"65f3667c-7893-445c-94f3-6fdf8ce6b516"}
153002024-02-01T17:34:10.087Z{"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:08.195097862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
153012024-02-01T17:34:10.087ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153022024-02-01T17:34:10.087ZLoading write count information from file "/tmp/test_repair_verify.data"
153032024-02-01T17:34:10.087ZRun Repair workload
153042024-02-01T17:34:10.088Z01/30 Write block 51 len 24576 data: 12 10 12 14 12 11
153052024-02-01T17:34:10.096Z02/30 Write block 9 len 40960 data: 10 12 9 7 6 6 7 8 11 13
153062024-02-01T17:34:10.096Z03/30 Read block 286 len 4096
153072024-02-01T17:34:10.097Z04/30 Write block 486 len 40960 data: 10 12 12 12 13 13 15 11 9 10
153082024-02-01T17:34:10.097Z05/30 Read block 456 len 16384
153092024-02-01T17:34:10.097Z06/30 Write block 351 len 24576 data: 11 12 10 16 13 12
153102024-02-01T17:34:10.097Z07/30 Read block 132 len 20480
153112024-02-01T17:34:10.097Z08/30 Write block 109 len 4096 data: 12
153122024-02-01T17:34:10.097Z09/30 Write block 419 len 8192 data: 10 8
153132024-02-01T17:34:10.097Z10/30 Read block 2 len 32768
153142024-02-01T17:34:10.097Z11/30 Write block 487 len 28672 data: 13 13 13 14 14 16 12
153152024-02-01T17:34:10.097Z12/30 Read block 144 len 12288
153162024-02-01T17:34:10.097Z13/30 Write block 506 len 24576 data: 6 6 9 10 9 10
153172024-02-01T17:34:10.098Z14/30 Read block 46 len 32768
153182024-02-01T17:34:10.098Z15/30 Write block 457 len 8192 data: 14 14
153192024-02-01T17:34:10.098Z16/30 Read block 405 len 4096
153202024-02-01T17:34:10.098Z17/30 Read block 248 len 4096
153212024-02-01T17:34:10.098Z18/30 Flush
153222024-02-01T17:34:10.098Z19/30 Write block 283 len 36864 data: 8 9 13 13 12 14 15 13 14
153232024-02-01T17:34:10.098Z20/30 Write block 352 len 12288 data: 13 11 17
153242024-02-01T17:34:10.098Z21/30 Read block 288 len 16384
153252024-02-01T17:34:10.098Z22/30 Read block 355 len 24576
153262024-02-01T17:34:10.098Z23/30 Read block 469 len 28672
153272024-02-01T17:34:10.098Z24/30 Read block 136 len 8192
153282024-02-01T17:34:10.098Z25/30 Flush
153292024-02-01T17:34:10.098Z26/30 Write block 559 len 12288 data: 13 11 10
153302024-02-01T17:34:10.099Z27/30 Read block 6 len 24576
153312024-02-01T17:34:10.099Z28/30 Write block 504 len 40960 data: 7 6 7 7 10 11 10 11 9 9
153322024-02-01T17:34:10.099Z29/30 Read block 540 len 12288
153332024-02-01T17:34:10.099Z30/30 Read block 382 len 12288
153342024-02-01T17:34:10.099Z----------------------------------------------------------------
153352024-02-01T17:34:10.099Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
153362024-02-01T17:34:10.099ZStates: Active Active Active
153372024-02-01T17:34:10.099ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153382024-02-01T17:34:10.099Z 1 Acked 1000 Write 6 Done Done Done false
153392024-02-01T17:34:10.099Z 2 Acked 1001 Write 10 Done Done Done false
153402024-02-01T17:34:10.099Z 3 Acked 1002 Read 1 Done Sent Done false
153412024-02-01T17:34:10.099Z 4 Acked 1003 Write 10 Done Sent Done false
153422024-02-01T17:34:10.100Z 5 Acked 1004 Read 4 Done Sent Done false
153432024-02-01T17:34:10.100Z 6 Acked 1005 Write 6 Done Sent Done false
153442024-02-01T17:34:10.100Z 7 Acked 1006 Read 5 Done Sent Done false
153452024-02-01T17:34:10.100Z 8 Acked 1007 Write 1 Done Sent Done false
153462024-02-01T17:34:10.100Z 9 Acked 1008 Write 2 Done Sent Done false
153472024-02-01T17:34:10.100Z 10 Acked 1009 Read 8 Done Sent Done false
153482024-02-01T17:34:10.100Z 11 Acked 1010 Write 7 Done Sent Done false
153492024-02-01T17:34:10.100Z 12 Acked 1011 Read 3 Done Sent Done false
153502024-02-01T17:34:10.100Z 13 Acked 1012 Write 6 Done Sent Done false
153512024-02-01T17:34:10.100Z 14 Acked 1013 Read 8 Done Sent Done false
153522024-02-01T17:34:10.100Z 15 Acked 1014 Write 2 Done Sent Done false
153532024-02-01T17:34:10.100Z 16 Acked 1015 Read 1 Done Sent Done false
153542024-02-01T17:34:10.101Z 17 Acked 1016 Read 1 Done Sent Done false
153552024-02-01T17:34:10.101Z 18 Acked 1017 Flush 0 Done Sent Done false
153562024-02-01T17:34:10.101Z 19 Acked 1018 Write 9 Done Sent Done false
153572024-02-01T17:34:10.101Z 20 Acked 1019 Write 3 Done Sent Done false
153582024-02-01T17:34:10.101Z 21 Acked 1020 Read 4 Done Sent Done false
153592024-02-01T17:34:10.101Z 22 Acked 1021 Read 6 Done Sent Done false
153602024-02-01T17:34:10.101Z 23 Acked 1022 Read 7 Done Sent Done false
153612024-02-01T17:34:10.101Z 24 Acked 1023 Read 2 Done Sent Done false
153622024-02-01T17:34:10.101Z 25 Acked 1024 Flush 0 Done Sent Done false
153632024-02-01T17:34:10.101Z 26 Acked 1025 Write 3 Done Sent Done false
153642024-02-01T17:34:10.101Z 27 Acked 1026 Read 6 Done Sent Done false
153652024-02-01T17:34:10.101Z 28 Acked 1027 Write 10 Done Sent Done false
153662024-02-01T17:34:10.102Z 29 Acked 1028 Read 3 Done Sent Done false
153672024-02-01T17:34:10.102Z 30 Acked 1029 Read 3 Done Sent Done false
153682024-02-01T17:34:10.102Z STATES DS:0 DS:1 DS:2 TOTAL
153692024-02-01T17:34:10.102Z New 0 0 0 0
153702024-02-01T17:34:10.102Z Sent 0 28 0 28
153712024-02-01T17:34:10.102Z Done 30 2 30 62
153722024-02-01T17:34:10.102Z Skipped 0 0 0 0
153732024-02-01T17:34:10.102Z Error 0 0 0 0
153742024-02-01T17:34:10.102ZLast Flush: 1024 0 1024
153752024-02-01T17:34:10.102ZDownstairs last five completed:
153762024-02-01T17:34:10.103ZUpstairs last five completed: 30 29 28 27 26
153772024-02-01T17:34:10.103Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:08.242645357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1697}
153782024-02-01T17:34:10.104ZWrote out file "/tmp/test_repair_verify.data"
153792024-02-01T17:34:10.105Z
153802024-02-01T17:34:10.105ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153812024-02-01T17:34:10.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153822024-02-01T17:34:10.105Z 0 000-019 142 141 142 228 227 228 F T F <---
153832024-02-01T17:34:10.105Z 1 020-039 139 139 139 224 224 224 F F F
153842024-02-01T17:34:10.105Z 2 040-059 142 141 142 228 227 228 F T F <---
153852024-02-01T17:34:10.106Z 3 060-079 132 132 132 207 207 207 F F F
153862024-02-01T17:34:10.106Z 4 080-099 133 133 133 210 210 210 F F F
153872024-02-01T17:34:10.106Z 5 100-119 142 138 142 228 222 228 F F F <---
153882024-02-01T17:34:10.106Z 6 120-139 140 140 140 225 225 225 F F F
153892024-02-01T17:34:10.106Z 7 140-159 140 140 140 225 225 225 F F F
153902024-02-01T17:34:10.107Z 8 160-179 140 140 140 225 225 225 F F F
153912024-02-01T17:34:10.107Z 9 180-199 135 135 135 215 215 215 F F F
153922024-02-01T17:34:10.107Z 10 200-219 136 136 136 217 217 217 F F F
153932024-02-01T17:34:10.107Z 11 220-239 138 138 138 223 223 223 F F F
153942024-02-01T17:34:10.107Z 12 240-259 136 136 136 216 216 216 F F F
153952024-02-01T17:34:10.107Z 13 260-279 136 136 136 217 217 217 F F F
153962024-02-01T17:34:10.107Z 14 280-299 142 141 142 229 227 229 F F F <---
153972024-02-01T17:34:10.107Z 15 300-319 141 141 141 227 227 227 F F F
153982024-02-01T17:34:10.107Z 16 320-339 141 141 141 227 227 227 F F F
153992024-02-01T17:34:10.108Z 17 340-359 142 141 142 229 227 229 F F F <---
154002024-02-01T17:34:10.108Z 18 360-379 141 141 141 227 227 227 F F F
154012024-02-01T17:34:10.108Z 19 380-399 135 135 135 215 215 215 F F F
154022024-02-01T17:34:10.108Z 20 400-419 142 140 142 228 225 228 F F F <---
154032024-02-01T17:34:10.108Z 21 420-439 142 135 142 228 215 228 F F F <---
154042024-02-01T17:34:10.108Z 22 440-459 142 135 142 228 215 228 F F F <---
154052024-02-01T17:34:10.114Z 23 460-479 135 135 135 215 215 215 F F F
154062024-02-01T17:34:10.114Z 24 480-499 142 141 142 228 227 228 F F F <---
154072024-02-01T17:34:10.114Z 25 500-519 142 141 142 228 227 228 T F T <---
154082024-02-01T17:34:10.114Z 26 520-539 127 127 127 202 202 202 F F F
154092024-02-01T17:34:10.115Z 27 540-559 138 138 138 223 223 223 T F T <---
154102024-02-01T17:34:10.115Z 28 560-579 138 138 138 223 223 223 T F T <---
154112024-02-01T17:34:10.115Z 29 580-599 133 133 133 210 210 210 F F F
154122024-02-01T17:34:10.115ZMax gen: 142, Max flush: 229
154132024-02-01T17:34:10.115ZError: Difference in extent metadata found!
154142024-02-01T17:34:10.115ZA Difference in extent metadata is expected here
154152024-02-01T17:34:10.115ZOn loop 71
154162024-02-01T17:34:10.115Z
154172024-02-01T17:34:10.116ZVerifying data now
154182024-02-01T17:34:14.047ZNo change after verify
154192024-02-01T17:34:14.048ZLoop: 71 Downstairs dump after verify (and repair):
154202024-02-01T17:34:14.079ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154212024-02-01T17:34:14.079Z 0 000-019 142 142 142 228 228 228 F F F
154222024-02-01T17:34:14.089Z 1 020-039 139 139 139 224 224 224 F F F
154232024-02-01T17:34:14.090Z 2 040-059 142 142 142 228 228 228 F F F
154242024-02-01T17:34:14.090Z 3 060-079 132 132 132 207 207 207 F F F
154252024-02-01T17:34:14.090Z 4 080-099 133 133 133 210 210 210 F F F
154262024-02-01T17:34:14.090Z 5 100-119 142 142 142 228 228 228 F F F
154272024-02-01T17:34:14.090Z 6 120-139 140 140 140 225 225 225 F F F
154282024-02-01T17:34:14.091Z 7 140-159 140 140 140 225 225 225 F F F
154292024-02-01T17:34:14.091Z 8 160-179 140 140 140 225 225 225 F F F
154302024-02-01T17:34:14.091Z 9 180-199 135 135 135 215 215 215 F F F
154312024-02-01T17:34:14.092Z 10 200-219 136 136 136 217 217 217 F F F
154322024-02-01T17:34:14.092Z 11 220-239 138 138 138 223 223 223 F F F
154332024-02-01T17:34:14.092Z 12 240-259 136 136 136 216 216 216 F F F
154342024-02-01T17:34:14.092Z 13 260-279 136 136 136 217 217 217 F F F
154352024-02-01T17:34:14.093Z 14 280-299 142 142 142 229 229 229 F F F
154362024-02-01T17:34:14.093Z 15 300-319 141 141 141 227 227 227 F F F
154372024-02-01T17:34:14.093Z 16 320-339 141 141 141 227 227 227 F F F
154382024-02-01T17:34:14.093Z 17 340-359 142 142 142 229 229 229 F F F
154392024-02-01T17:34:14.094Z 18 360-379 141 141 141 227 227 227 F F F
154402024-02-01T17:34:14.094Z 19 380-399 135 135 135 215 215 215 F F F
154412024-02-01T17:34:14.094Z 20 400-419 142 142 142 228 228 228 F F F
154422024-02-01T17:34:14.094Z 21 420-439 142 142 142 228 228 228 F F F
154432024-02-01T17:34:14.095Z 22 440-459 142 142 142 228 228 228 F F F
154442024-02-01T17:34:14.095Z 23 460-479 135 135 135 215 215 215 F F F
154452024-02-01T17:34:14.095Z 24 480-499 142 142 142 228 228 228 F F F
154462024-02-01T17:34:14.095Z 25 500-519 143 143 143 230 230 230 F F F
154472024-02-01T17:34:14.095Z 26 520-539 127 127 127 202 202 202 F F F
154482024-02-01T17:34:14.095Z 27 540-559 143 143 143 230 230 230 F F F
154492024-02-01T17:34:14.095Z 28 560-579 143 143 143 230 230 230 F F F
154502024-02-01T17:34:14.096Z 29 580-599 133 133 133 210 210 210 F F F
154512024-02-01T17:34:14.096ZMax gen: 143, Max flush: 230
154522024-02-01T17:34:14.096Z
154532024-02-01T17:34:14.096ZBegin loop 72
154542024-02-01T17:34:14.096ZDownstairs to restart: 2
154552024-02-01T17:34:14.102Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.37838576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
154562024-02-01T17:34:14.102Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.378853977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
154572024-02-01T17:34:14.102Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.378951011Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
154582024-02-01T17:34:14.102Z{"msg":"Crucible dc55ae4f-df94-4eb5-a567-c98d72b2dfc9 has session id: caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379015179Z","hostname":"Crucible runtime is spawned
154592024-02-01T17:34:14.102Zip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154602024-02-01T17:34:14.102Z{"msg":"Upstairs opts: Upstairs UUID: dc55ae4f-df94-4eb5-a567-c98d72b2dfc9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379087846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154612024-02-01T17:34:14.102Z{"msg":"Crucible stats registered with UUID: dc55ae4f-df94-4eb5-a567-c98d72b2dfc9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379139183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154622024-02-01T17:34:14.102Z{"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379163488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
154632024-02-01T17:34:14.102Z{"msg":"dc55ae4f-df94-4eb5-a567-c98d72b2dfc9 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379202414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154642024-02-01T17:34:14.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379243456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154652024-02-01T17:34:14.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379276253Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154662024-02-01T17:34:14.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379345005Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154672024-02-01T17:34:14.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379391692Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154682024-02-01T17:34:14.103Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379433153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154692024-02-01T17:34:14.103Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379505679Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154702024-02-01T17:34:14.103Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379578051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154712024-02-01T17:34:14.103Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379620742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154722024-02-01T17:34:14.103Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.379721053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154732024-02-01T17:34:14.111Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.37988631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154742024-02-01T17:34:14.111Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.380142239Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154752024-02-01T17:34:14.111Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.380170761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"io task","client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154762024-02-01T17:34:14.111Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.380805754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154772024-02-01T17:34:14.111Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.380885555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154782024-02-01T17:34:14.111Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.380931087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154792024-02-01T17:34:14.111Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.381088416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154802024-02-01T17:34:14.111Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.38178198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154812024-02-01T17:34:14.114Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.381991367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154822024-02-01T17:34:14.114Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.382258708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154832024-02-01T17:34:14.114Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.38231532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154842024-02-01T17:34:14.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.382531355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154852024-02-01T17:34:14.114Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.382643923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154862024-02-01T17:34:14.114Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.382689231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154872024-02-01T17:34:14.114Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.382908085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154882024-02-01T17:34:14.114Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.38296969Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154892024-02-01T17:34:14.114Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.383002652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154902024-02-01T17:34:14.114Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.383292814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154912024-02-01T17:34:14.116Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.383965102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154922024-02-01T17:34:14.116Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384044474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154932024-02-01T17:34:14.116Z{"msg":"[0]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384083418Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154942024-02-01T17:34:14.116Z{"msg":"[0]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.38413999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154952024-02-01T17:34:14.116Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384174954Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154962024-02-01T17:34:14.116Z{"msg":"[1]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384233184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154972024-02-01T17:34:14.116Z{"msg":"[1]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
154982024-02-01T17:34:14.116Z,"time":"2024-02-01T17:34:13.384266475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
154992024-02-01T17:34:14.116Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384376918Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155002024-02-01T17:34:14.116Z{"msg":"[2]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 215, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384411453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155012024-02-01T17:34:14.116Z{"msg":"[2]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 135, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384468245Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155022024-02-01T17:34:14.116Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384502391Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155032024-02-01T17:34:14.116Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384557679Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155042024-02-01T17:34:14.116Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384586364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155052024-02-01T17:34:14.116Z{"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384644944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155062024-02-01T17:34:14.116Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384670734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"":"downstairs","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155072024-02-01T17:34:14.116Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.38473949Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155082024-02-01T17:34:14.116Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384764439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"0","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155092024-02-01T17:34:14.116Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155102024-02-01T17:34:14.116Z"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384825368Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"1","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155112024-02-01T17:34:14.117Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384884093Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"client":"2","":"downstairs","session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155122024-02-01T17:34:14.117Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384919512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155132024-02-01T17:34:14.117Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384968686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155142024-02-01T17:34:14.117Z{"msg":"dc55ae4f-df94-4eb5-a567-c98d72b2dfc9 is now active with session: caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.384998301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155152024-02-01T17:34:14.117Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.385058135Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705,"session_id":"caf85df7-bd6e-4cbf-9ea3-a6dce2a03a27"}
155162024-02-01T17:34:14.117Z{"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:13.385086857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
155172024-02-01T17:34:14.118ZLoading write count information from file "/tmp/test_repair_verify.data"
155182024-02-01T17:34:14.118ZRun Repair workload
155192024-02-01T17:34:14.118Z01/30 Write block 181 len 8192 data: 13 15
155202024-02-01T17:34:14.124Z02/30 Read block 578 len 12288
155212024-02-01T17:34:14.124Z03/30 Read block 1 len 12288
155222024-02-01T17:34:14.124Z04/30 Read block 533 len 32768
155232024-02-01T17:34:14.124Z05/30 Read block 491 len 8192
155242024-02-01T17:34:14.124Z06/30 Read block 145 len 12288
155252024-02-01T17:34:14.124Z07/30 Read block 370 len 32768
155262024-02-01T17:34:14.124Z08/30 Write block 258 len 20480 data: 13 13 12 12 13
155272024-02-01T17:34:14.126Z09/30 Write block 421 len 20480 data: 8 10 9 9 12
155282024-02-01T17:34:14.126Z10/30 Read block 205 len 8192
155292024-02-01T17:34:14.126Z11/30 Read block 354 len 36864
155302024-02-01T17:34:14.126Z12/30 Write block 270 len 40960 data: 13 10 9 9 8 10 14 14 14 10
155312024-02-01T17:34:14.133Z13/30 Write block 494 len 8192 data: 10 11
155322024-02-01T17:34:14.133Z14/30 Write block 391 len 36864 data: 10 11 11 9 6 6 8 7 5
155332024-02-01T17:34:14.133Z15/30 Read block 70 len 4096
155342024-02-01T17:34:14.133Z16/30 Read block 87 len 16384
155352024-02-01T17:34:14.133Z17/30 Read block 426 len 8192
155362024-02-01T17:34:14.133Z18/30 Read block 179 len 8192
155372024-02-01T17:34:14.133Z19/30 Write block 459 len 8192 data: 15 13
155382024-02-01T17:34:14.133Z20/30 Read block 405 len 40960
155392024-02-01T17:34:14.137Z21/30 Read block 405 len 8192
155402024-02-01T17:34:14.137Z22/30 Read block 383 len 20480
155412024-02-01T17:34:14.137Z23/30 Write block 395 len 4096 data: 7
155422024-02-01T17:34:14.137Z24/30 Read block 48 len 24576
155432024-02-01T17:34:14.142Z25/30 Read block 489 len 12288
155442024-02-01T17:34:14.143Z26/30 Flush
155452024-02-01T17:34:14.157Z27/30 Write block 419 len 32768 data: 11 9 9 11 10 10 13 14
155462024-02-01T17:34:14.157Z28/30 Write block 341 len 12288 data: 8 9 9
155472024-02-01T17:34:14.157Z29/30 Read block 411 len 8192
155482024-02-01T17:34:14.158Z30/30 Read block 348 len 12288
155492024-02-01T17:34:14.158Z----------------------------------------------------------------
155502024-02-01T17:34:14.158Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
155512024-02-01T17:34:14.158ZStates: Active Active Active
155522024-02-01T17:34:14.158ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155532024-02-01T17:34:14.159Z 1 Acked 1000 Write 2 Done Done Sent false
155542024-02-01T17:34:14.159Z 2 Acked 1001 Read 3 Done Done Sent false
155552024-02-01T17:34:14.159Z 3 Acked 1002 Read 3 Done Done Sent false
155562024-02-01T17:34:14.159Z 4 Acked 1003 Read 8 Done Done Sent false
155572024-02-01T17:34:14.159Z 5 Acked 1004 Read 2 Done Done Sent false
155582024-02-01T17:34:14.159Z 6 Acked 1005 Read 3 Done Done Sent false
155592024-02-01T17:34:14.159Z 7 Acked 1006 Read 8 Done Done Sent false
155602024-02-01T17:34:14.159Z 8 Acked 1007 Write 5 Done Done Sent false
155612024-02-01T17:34:14.159Z 9 Acked 1008 Write 5 Done Done Sent false
155622024-02-01T17:34:14.159Z 10 Acked 1009 Read 2 Done Done Sent false
155632024-02-01T17:34:14.159Z 11 Acked 1010 Read 9 Done Done Sent false
155642024-02-01T17:34:14.159Z 12 Acked 1011 Write 10 Done Done Sent false
155652024-02-01T17:34:14.159Z 13 Acked 1012 Write 2 Done Done Sent false
155662024-02-01T17:34:14.159Z 14 Acked 1013 Write 9 Done Done Sent false
155672024-02-01T17:34:14.159Z 15 Acked 1014 Read 1 Done Done Sent false
155682024-02-01T17:34:14.159Z 16 Acked 1015 Read 4 Done Done Sent false
155692024-02-01T17:34:14.159Z 17 Acked 1016 Read 2 Done Done Sent false
155702024-02-01T17:34:14.159Z 18 Acked 1017 Read 2 Done Done Sent false
155712024-02-01T17:34:14.160Z 19 Acked 1018 Write 2 Done Done Sent false
155722024-02-01T17:34:14.160Z 20 Acked 1019 Read 10 Done Done Sent false
155732024-02-01T17:34:14.160Z 21 Acked 1020 Read 2 Done Done Sent false
155742024-02-01T17:34:14.160Z 22 Acked 1021 Read 5 Done Done Sent false
155752024-02-01T17:34:14.163Z 23 Acked 1022 Write 1 Done Done Sent false
155762024-02-01T17:34:14.164Z 24 Acked 1023 Read 6 Done Done Sent false
155772024-02-01T17:34:14.164Z 25 Acked 1024 Read 3 Done Done Sent false
155782024-02-01T17:34:14.164Z 26 Acked 1025 Flush 0 Done Done Sent false
155792024-02-01T17:34:14.164Z 27 Acked 1026 Write 8 Done Done Sent false
155802024-02-01T17:34:14.164Z 28 Acked 1027 Write 3 Done Done Sent false
155812024-02-01T17:34:14.164Z 29 Acked 1028 Read 2 Done Done Sent false
155822024-02-01T17:34:14.164Z 30 Acked 1029 Read 3 Done Done Sent false
155832024-02-01T17:34:14.164Z STATES DS:0 DS:1 DS:2 TOTAL
155842024-02-01T17:34:14.164Z New 0 0 0 0
155852024-02-01T17:34:14.164Z Sent 0 0 30 30
155862024-02-01T17:34:14.164Z Done 30 30 0 60
155872024-02-01T17:34:14.164Z Skipped 0 0 0 0
155882024-02-01T17:34:14.165Z Error 0 0 0 0
155892024-02-01T17:34:14.165ZLast Flush: 1025 1025 0
155902024-02-01T17:34:14.165ZDownstairs last five completed:
155912024-02-01T17:34:14.165ZUpstairs last five completed: 30 29 28 27 26
155922024-02-01T17:34:14.165Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:13.426611741Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1705}
155932024-02-01T17:34:14.165ZWrote out file "/tmp/test_repair_verify.data"
155942024-02-01T17:34:14.165Z
155952024-02-01T17:34:14.171ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155962024-02-01T17:34:14.183ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155972024-02-01T17:34:14.183Z 0 000-019 142 142 142 228 228 228 F F F
155982024-02-01T17:34:14.184Z 1 020-039 139 139 139 224 224 224 F F F
155992024-02-01T17:34:14.184Z 2 040-059 142 142 142 228 228 228 F F F
156002024-02-01T17:34:14.184Z 3 060-079 132 132 132 207 207 207 F F F
156012024-02-01T17:34:14.187Z 4 080-099 133 133 133 210 210 210 F F F
156022024-02-01T17:34:14.188Z 5 100-119 142 142 142 228 228 228 F F F
156032024-02-01T17:34:14.188Z 6 120-139 140 140 140 225 225 225 F F F
156042024-02-01T17:34:14.188Z 7 140-159 140 140 140 225 225 225 F F F
156052024-02-01T17:34:14.188Z 8 160-179 140 140 140 225 225 225 F F F
156062024-02-01T17:34:14.188Z 9 180-199 144 144 135 231 231 215 F F F <---
156072024-02-01T17:34:14.188Z 10 200-219 136 136 136 217 217 217 F F F
156082024-02-01T17:34:14.188Z 11 220-239 138 138 138 223 223 223 F F F
156092024-02-01T17:34:14.188Z 12 240-259 144 144 136 231 231 216 F F F <---
156102024-02-01T17:34:14.188Z 13 260-279 144 144 136 231 231 217 F F F <---
156112024-02-01T17:34:14.188Z 14 280-299 142 142 142 229 229 229 F F F
156122024-02-01T17:34:14.188Z 15 300-319 141 141 141 227 227 227 F F F
156132024-02-01T17:34:14.188Z 16 320-339 141 141 141 227 227 227 F F F
156142024-02-01T17:34:14.188Z 17 340-359 142 142 142 229 229 229 T T F <---
156152024-02-01T17:34:14.188Z 18 360-379 141 141 141 227 227 227 F F F
156162024-02-01T17:34:14.188Z 19 380-399 144 144 135 231 231 215 F F F <---
156172024-02-01T17:34:14.188Z 20 400-419 142 142 142 228 228 228 T T F <---
156182024-02-01T17:34:14.188Z 21 420-439 144 144 142 231 231 228 T T F <---
156192024-02-01T17:34:14.188Z 22 440-459 144 144 142 231 231 228 F F F <---
156202024-02-01T17:34:14.188Z 23 460-479 144 144 135 231 231 215 F F F <---
156212024-02-01T17:34:14.188Z 24 480-499 144 144 142 231 231 228 F F F <---
156222024-02-01T17:34:14.188Z 25 500-519 143 143 143 230 230 230 F F F
156232024-02-01T17:34:14.188Z 26 520-539 127 127 127 202 202 202 F F F
156242024-02-01T17:34:14.188Z 27 540-559 143 143 143 230 230 230 F F F
156252024-02-01T17:34:14.189Z 28 560-579 143 143 143 230 230 230 F F F
156262024-02-01T17:34:14.189Z 29 580-599 133 133 133 210 210 210 F F F
156272024-02-01T17:34:14.189ZMax gen: 144, Max flush: 231
156282024-02-01T17:34:14.189ZError: Difference in extent metadata found!
156292024-02-01T17:34:14.192ZA Difference in extent metadata is expected here
156302024-02-01T17:34:14.192ZOn loop 72
156312024-02-01T17:34:14.192Z
156322024-02-01T17:34:14.192ZVerifying data now
156332024-02-01T17:34:21.581ZNo change after verify
156342024-02-01T17:34:21.581ZLoop: 72 Downstairs dump after verify (and repair):
156352024-02-01T17:34:21.796ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156362024-02-01T17:34:21.796Z 0 000-019 142 142 142 228 228 228 F F F
156372024-02-01T17:34:21.797Z 1 020-039 139 139 139 224 224 224 F F F
156382024-02-01T17:34:21.797Z 2 040-059 142 142 142 228 228 228 F F F
156392024-02-01T17:34:21.797Z 3 060-079 132 132 132 207 207 207 F F F
156402024-02-01T17:34:21.797Z 4 080-099 133 133 133 210 210 210 F F F
156412024-02-01T17:34:21.797Z 5 100-119 142 142 142 228 228 228 F F F
156422024-02-01T17:34:21.797Z 6 120-139 140 140 140 225 225 225 F F F
156432024-02-01T17:34:21.798Z 7 140-159 140 140 140 225 225 225 F F F
156442024-02-01T17:34:21.798Z 8 160-179 140 140 140 225 225 225 F F F
156452024-02-01T17:34:21.798Z 9 180-199 144 144 144 231 231 231 F F F
156462024-02-01T17:34:21.798Z 10 200-219 136 136 136 217 217 217 F F F
156472024-02-01T17:34:21.798Z 11 220-239 138 138 138 223 223 223 F F F
156482024-02-01T17:34:21.798Z 12 240-259 144 144 144 231 231 231 F F F
156492024-02-01T17:34:21.798Z 13 260-279 144 144 144 231 231 231 F F F
156502024-02-01T17:34:21.800Z 14 280-299 142 142 142 229 229 229 F F F
156512024-02-01T17:34:21.800Z 15 300-319 141 141 141 227 227 227 F F F
156522024-02-01T17:34:21.800Z 16 320-339 141 141 141 227 227 227 F F F
156532024-02-01T17:34:21.800Z 17 340-359 145 145 145 232 232 232 F F F
156542024-02-01T17:34:21.801Z 18 360-379 141 141 141 227 227 227 F F F
156552024-02-01T17:34:21.801Z 19 380-399 144 144 144 231 231 231 F F F
156562024-02-01T17:34:21.801Z 20 400-419 145 145 145 232 232 232 F F F
156572024-02-01T17:34:21.801Z 21 420-439 145 145 145 232 232 232 F F F
156582024-02-01T17:34:21.801Z 22 440-459 144 144 144 231 231 231 F F F
156592024-02-01T17:34:21.801Z 23 460-479 144 144 144 231 231 231 F F F
156602024-02-01T17:34:21.801Z 24 480-499 144 144 144 231 231 231 F F F
156612024-02-01T17:34:21.801Z 25 500-519 143 143 143 230 230 230 F F F
156622024-02-01T17:34:21.801Z 26 520-539 127 127 127 202 202 202 F F F
156632024-02-01T17:34:21.801Z 27 540-559 143 143 143 230 230 230 F F F
156642024-02-01T17:34:21.801Z 28 560-579 143 143 143 230 230 230 F F F
156652024-02-01T17:34:21.801Z 29 580-599 133 133 133 210 210 210 F F F
156662024-02-01T17:34:21.801ZMax gen: 145, Max flush: 232
156672024-02-01T17:34:21.801Z
156682024-02-01T17:34:21.802ZBegin loop 73
156692024-02-01T17:34:21.802ZDownstairs to restart: 0
156702024-02-01T17:34:21.802Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.415811595Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
156712024-02-01T17:34:21.802Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416300956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
156722024-02-01T17:34:21.802Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416331174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
156732024-02-01T17:34:21.802Z{"msg":"Crucible 8e3061a0-9d1e-4d77-a9d2-2ebd880c28f4 has session id: 2d6833de-e535-4fd4-8ff3-b9597df9afce","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41640582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156742024-02-01T17:34:21.802Z{"msg":"Upstairs opts: Upstairs UUID: 8e3061a0-9d1e-4d77-a9d2-2ebd880c28f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
156752024-02-01T17:34:21.802Z,"time":"2024-02-01T17:34:18.416455575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156762024-02-01T17:34:21.802Z{"msg":"Crucible stats registered with UUID: 8e3061a0-9d1e-4d77-a9d2-2ebd880c28f4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416590257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156772024-02-01T17:34:21.802Z{"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416647263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
156782024-02-01T17:34:21.802Z{"msg":"8e3061a0-9d1e-4d77-a9d2-2ebd880c28f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416691105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156792024-02-01T17:34:21.802Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416735947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156802024-02-01T17:34:21.803Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41677355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156812024-02-01T17:34:21.803Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416830425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156822024-02-01T17:34:21.803Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416855911Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156832024-02-01T17:34:21.803Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416892484Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156842024-02-01T17:34:21.803Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416947946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156852024-02-01T17:34:21.804Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.416974176Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156862024-02-01T17:34:21.804Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41702899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156872024-02-01T17:34:21.804Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.417071596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156882024-02-01T17:34:21.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.417203461Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156892024-02-01T17:34:21.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41748251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156902024-02-01T17:34:21.804Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.417568919Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"io task","client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156912024-02-01T17:34:21.804Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41850297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156922024-02-01T17:34:21.806Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.418631672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156932024-02-01T17:34:21.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.418725598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156942024-02-01T17:34:21.806Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.418797968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156952024-02-01T17:34:21.806Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.419375667Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156962024-02-01T17:34:21.806Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.41950895Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156972024-02-01T17:34:21.806Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.419611803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156982024-02-01T17:34:21.806Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.419752422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
156992024-02-01T17:34:21.806Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420270581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157002024-02-01T17:34:21.806Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420332638Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157012024-02-01T17:34:21.806Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420359415Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157022024-02-01T17:34:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420706413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157032024-02-01T17:34:21.807Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42076444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157042024-02-01T17:34:21.807Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420790114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157052024-02-01T17:34:21.807Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.420831717Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157062024-02-01T17:34:21.807Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42148873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157072024-02-01T17:34:21.807Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42156953Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157082024-02-01T17:34:21.807Z{"msg":"[0]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 231, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421600701Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157092024-02-01T17:34:21.807Z{"msg":"[0]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 144, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42164472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157102024-02-01T17:34:21.807Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421671144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157112024-02-01T17:34:21.807Z{"msg":"[1]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 231, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421712645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157122024-02-01T17:34:21.807Z{"msg":"[1]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 144, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421737408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157132024-02-01T17:34:21.807Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v"Wait for a query_work_queue command to finish before sending IO
157142024-02-01T17:34:21.807Z:0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42180511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157152024-02-01T17:34:21.807Z{"msg":"[2]R flush_numbers[0..12]: [228, 224, 228, 207, 210, 228, 225, 225, 225, 231, 217, 223]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.42187818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157162024-02-01T17:34:21.808Z{"msg":"[2]R generation[0..12]: [142, 139, 142, 132, 133, 142, 140, 140, 140, 144, 136, 138]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421911146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157172024-02-01T17:34:21.808Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421935268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157182024-02-01T17:34:21.808Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.421977408Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157192024-02-01T17:34:21.808Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422014816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157202024-02-01T17:34:21.808Z{"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422083426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157212024-02-01T17:34:21.808Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422113618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"":"downstairs","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157222024-02-01T17:34:21.809Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422183814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157232024-02-01T17:34:21.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422247143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"0","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157242024-02-01T17:34:21.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422274956Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"1","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157252024-02-01T17:34:21.809Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422331615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"client":"2","":"downstairs","session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157262024-02-01T17:34:21.809Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422402647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157272024-02-01T17:34:21.809Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422476787Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157282024-02-01T17:34:21.809Z{"msg":"8e3061a0-9d1e-4d77-a9d2-2ebd880c28f4 is now active with session: 2d6833de-e535-4fd4-8ff3-b9597df9afce","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422536343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157292024-02-01T17:34:21.810Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422559401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713,"session_id":"2d6833de-e535-4fd4-8ff3-b9597df9afce"}
157302024-02-01T17:34:21.810ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157312024-02-01T17:34:21.810Z{"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:18.422627324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
157322024-02-01T17:34:21.810ZLoading write count information from file "/tmp/test_repair_verify.data"
157332024-02-01T17:34:21.810ZRun Repair workload
157342024-02-01T17:34:21.811Z01/30 Write block 500 len 24576 data: 11 10 10 9 8 7
157352024-02-01T17:34:21.872Z02/30 Flush
157362024-02-01T17:34:21.872Z03/30 Write block 84 len 20480 data: 8 8 8 7 8
157372024-02-01T17:34:21.873Z04/30 Read block 168 len 28672
157382024-02-01T17:34:21.873Z05/30 Read block 516 len 32768
157392024-02-01T17:34:21.873Z06/30 Read block 437 len 40960
157402024-02-01T17:34:21.873Z07/30 Read block 400 len 28672
157412024-02-01T17:34:21.873Z08/30 Write block 216 len 40960 data: 6 6 12 14 16 14 14 14 13 10
157422024-02-01T17:34:21.873Z09/30 Write block 330 len 36864 data: 13 16 14 14 15 14 14 10 8
157432024-02-01T17:34:21.873Z10/30 Read block 144 len 40960
157442024-02-01T17:34:21.873Z11/30 Write block 67 len 32768 data: 11 12 15 12 14 16 15 16
157452024-02-01T17:34:21.873Z12/30 Read block 13 len 16384
157462024-02-01T17:34:21.874Z13/30 Read block 366 len 12288
157472024-02-01T17:34:21.874Z14/30 Read block 108 len 16384
157482024-02-01T17:34:21.874Z15/30 Write block 584 len 40960 data: 12 11 11 10 11 10 10 9 6 6
157492024-02-01T17:34:21.874Z16/30 Write block 81 len 4096 data: 5
157502024-02-01T17:34:21.874Z17/30 Write block 572 len 40960 data: 7 7 6 10 9 9 11 11 12 12
157512024-02-01T17:34:21.874Z18/30 Write block 165 len 20480 data: 11 9 9 12 16
157522024-02-01T17:34:21.874Z19/30 Read block 359 len 40960
157532024-02-01T17:34:21.874Z20/30 Write block 102 len 28672 data: 13 11 12 15 16 17 15
157542024-02-01T17:34:21.874Z21/30 Read block 51 len 8192
157552024-02-01T17:34:21.874Z22/30 Read block 444 len 32768
157562024-02-01T17:34:21.874Z23/30 Write block 16 len 40960 data: 9 12 14 11 12 11 10 9 9 10
157572024-02-01T17:34:21.874Z24/30 Write block 80 len 20480 data: 7 6 6 9 9
157582024-02-01T17:34:21.874Z25/30 Write block 537 len 24576 data: 13 10 8 12 16 17
157592024-02-01T17:34:21.874Z26/30 Write block 457 len 28672 data: 15 15 16 14 14 12 11
157602024-02-01T17:34:21.874Z27/30 Read block 306 len 28672
157612024-02-01T17:34:21.880Z28/30 Read block 327 len 24576
157622024-02-01T17:34:21.880Z29/30 Write block 178 len 20480 data: 13 13 11 14 16
157632024-02-01T17:34:21.880Z30/30 Write block 495 len 8192 data: 12 10
157642024-02-01T17:34:21.880Z----------------------------------------------------------------
157652024-02-01T17:34:21.882Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:28
157662024-02-01T17:34:21.882ZStates: Active Active Active
157672024-02-01T17:34:21.882ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157682024-02-01T17:34:21.882Z 3 Acked 1002 Write 5 Done Done Done false
157692024-02-01T17:34:21.882Z 4 Acked 1003 Read 7 Done Done Done false
157702024-02-01T17:34:21.882Z 5 Acked 1004 Read 8 Sent Done Done false
157712024-02-01T17:34:21.882Z 6 Acked 1005 Read 10 Sent Done Done false
157722024-02-01T17:34:21.882Z 7 Acked 1006 Read 7 Sent Done Done false
157732024-02-01T17:34:21.882Z 8 Acked 1007 Write 10 Sent Done Done false
157742024-02-01T17:34:21.882Z 9 Acked 1008 Write 9 Sent Done Done false
157752024-02-01T17:34:21.882Z 10 Acked 1009 Read 10 Sent Done Done false
157762024-02-01T17:34:21.882Z 11 Acked 1010 Write 8 Sent Done Done false
157772024-02-01T17:34:21.882Z 12 Acked 1011 Read 4 Sent Done Done false
157782024-02-01T17:34:21.883Z 13 Acked 1012 Read 3 Sent Done Done false
157792024-02-01T17:34:21.883Z 14 Acked 1013 Read 4 Sent Done Done false
157802024-02-01T17:34:21.883Z 15 Acked 1014 Write 10 Sent Done Done false
157812024-02-01T17:34:21.883Z 16 Acked 1015 Write 1 Sent Done Done false
157822024-02-01T17:34:21.883Z 17 Acked 1016 Write 10 Sent Done Done false
157832024-02-01T17:34:21.883Z 18 Acked 1017 Write 5 Sent Done Done false
157842024-02-01T17:34:21.883Z 19 Acked 1018 Read 10 Sent Done Done false
157852024-02-01T17:34:21.883Z 20 Acked 1019 Write 7 Sent Done Done false
157862024-02-01T17:34:21.883Z 21 Acked 1020 Read 2 Sent Done Done false
157872024-02-01T17:34:21.883Z 22 Acked 1021 Read 8 Sent Done Done false
157882024-02-01T17:34:21.884Z 23 Acked 1022 Write 10 Sent Done Done false
157892024-02-01T17:34:21.884Z 24 Acked 1023 Write 5 Sent Done Done false
157902024-02-01T17:34:21.884Z 25 Acked 1024 Write 6 Sent Done Done false
157912024-02-01T17:34:21.884Z 26 Acked 1025 Write 7 Sent Done Done false
157922024-02-01T17:34:21.884Z 27 Acked 1026 Read 7 Sent Done Done false
157932024-02-01T17:34:21.884Z 28 Acked 1027 Read 6 Sent Done Done false
157942024-02-01T17:34:21.884Z 29 Acked 1028 Write 5 Sent Sent Sent false
157952024-02-01T17:34:21.884Z 30 Acked 1029 Write 2 Sent Sent Sent false
157962024-02-01T17:34:21.884Z STATES DS:0 DS:1 DS:2 TOTAL
157972024-02-01T17:34:21.884Z New 0 0 0 0
157982024-02-01T17:34:21.884Z Sent 26 2 2 30
157992024-02-01T17:34:21.884Z Done 2 26 26 54
158002024-02-01T17:34:21.884Z Skipped 0 0 0 0
158012024-02-01T17:34:21.884Z Error 0 0 0 0
158022024-02-01T17:34:21.885ZLast Flush: 1001 1001 1001
158032024-02-01T17:34:21.885ZDownstairs last five completed: 1001 1000
158042024-02-01T17:34:21.885ZUpstairs last five completed: 30 29 28 27 26
158052024-02-01T17:34:21.885Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:18.472103453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1713}
158062024-02-01T17:34:21.885ZWrote out file "/tmp/test_repair_verify.data"
158072024-02-01T17:34:21.885Z
158082024-02-01T17:34:21.885ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158092024-02-01T17:34:21.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158102024-02-01T17:34:21.885Z 0 000-019 142 142 142 228 228 228 F T T <---
158112024-02-01T17:34:21.886Z 1 020-039 139 139 139 224 224 224 F T T <---
158122024-02-01T17:34:21.886Z 2 040-059 142 142 142 228 228 228 F F F
158132024-02-01T17:34:21.886Z 3 060-079 132 132 132 207 207 207 F T T <---
158142024-02-01T17:34:21.887Z 4 080-099 133 133 133 210 210 210 T T T
158152024-02-01T17:34:21.887Z 5 100-119 142 142 142 228 228 228 F T T <---
158162024-02-01T17:34:21.887Z 6 120-139 140 140 140 225 225 225 F F F
158172024-02-01T17:34:21.887Z 7 140-159 140 140 140 225 225 225 F F F
158182024-02-01T17:34:21.887Z 8 160-179 140 140 140 225 225 225 F T T <---
158192024-02-01T17:34:21.887Z 9 180-199 144 144 144 231 231 231 F T T <---
158202024-02-01T17:34:21.887Z 10 200-219 136 136 136 217 217 217 F T T <---
158212024-02-01T17:34:21.887Z 11 220-239 138 138 138 223 223 223 F T T <---
158222024-02-01T17:34:21.887Z 12 240-259 144 144 144 231 231 231 F F F
158232024-02-01T17:34:21.887Z 13 260-279 144 144 144 231 231 231 F F F
158242024-02-01T17:34:21.887Z 14 280-299 142 142 142 229 229 229 F F F
158252024-02-01T17:34:21.887Z 15 300-319 141 141 141 227 227 227 F F F
158262024-02-01T17:34:21.887Z 16 320-339 141 141 141 227 227 227 F T T <---
158272024-02-01T17:34:21.887Z 17 340-359 145 145 145 232 232 232 F F F
158282024-02-01T17:34:21.888Z 18 360-379 141 141 141 227 227 227 F F F
158292024-02-01T17:34:21.888Z 19 380-399 144 144 144 231 231 231 F F F
158302024-02-01T17:34:21.888Z 20 400-419 145 145 145 232 232 232 F F F
158312024-02-01T17:34:21.888Z 21 420-439 145 145 145 232 232 232 F F F
158322024-02-01T17:34:21.888Z 22 440-459 144 144 144 231 231 231 F T T <---
158332024-02-01T17:34:21.888Z 23 460-479 144 144 144 231 231 231 F T T <---
158342024-02-01T17:34:21.888Z 24 480-499 144 144 144 231 231 231 F T T <---
158352024-02-01T17:34:21.917Z 25 500-519 146 146 146 233 233 233 F F F
158362024-02-01T17:34:21.917Z 26 520-539 127 127 127 202 202 202 F T T <---
158372024-02-01T17:34:21.917Z 27 540-559 143 143 143 230 230 230 F T T <---
158382024-02-01T17:34:21.917Z 28 560-579 143 143 143 230 230 230 F T T <---
158392024-02-01T17:34:21.917Z 29 580-599 133 133 133 210 210 210 F T T <---
158402024-02-01T17:34:21.917ZMax gen: 146, Max flush: 233
158412024-02-01T17:34:21.917ZError: Difference in extent metadata found!
158422024-02-01T17:34:21.917ZA Difference in extent metadata is expected here
158432024-02-01T17:34:21.917ZOn loop 73
158442024-02-01T17:34:21.917Z
158452024-02-01T17:34:21.917ZVerifying data now
158462024-02-01T17:34:24.745ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
158472024-02-01T17:34:24.749Z82a83
158482024-02-01T17:34:24.749Z> 7,
158492024-02-01T17:34:24.749Z84,85c85,87
158502024-02-01T17:34:24.749Z< 4,
158512024-02-01T17:34:24.749Z< 5,
158522024-02-01T17:34:24.749Z---
158532024-02-01T17:34:24.749Z> 6,
158542024-02-01T17:34:24.750Z> 9,
158552024-02-01T17:34:24.750Z> 9,
158562024-02-01T17:34:24.750Z88,89d89
158572024-02-01T17:34:24.751Z< 8,
158582024-02-01T17:34:24.751Z< 8,
158592024-02-01T17:34:24.751Zdiff found after verify
158602024-02-01T17:34:24.755ZLoop: 73 Downstairs dump after verify (and repair):
158612024-02-01T17:34:24.778ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158622024-02-01T17:34:24.778Z 0 000-019 147 147 147 234 234 234 F F F
158632024-02-01T17:34:24.778Z 1 020-039 147 147 147 234 234 234 F F F
158642024-02-01T17:34:24.779Z 2 040-059 142 142 142 228 228 228 F F F
158652024-02-01T17:34:24.779Z 3 060-079 147 147 147 234 234 234 F F F
158662024-02-01T17:34:24.779Z 4 080-099 147 147 147 234 234 234 F F F
158672024-02-01T17:34:24.779Z 5 100-119 147 147 147 234 234 234 F F F
158682024-02-01T17:34:24.779Z 6 120-139 140 140 140 225 225 225 F F F
158692024-02-01T17:34:24.779Z 7 140-159 140 140 140 225 225 225 F F F
158702024-02-01T17:34:24.788Z 8 160-179 147 147 147 234 234 234 F F F
158712024-02-01T17:34:24.788Z 9 180-199 147 147 147 234 234 234 F F F
158722024-02-01T17:34:24.788Z 10 200-219 147 147 147 234 234 234 F F F
158732024-02-01T17:34:24.788Z 11 220-239 147 147 147 234 234 234 F F F
158742024-02-01T17:34:24.788Z 12 240-259 144 144 144 231 231 231 F F F
158752024-02-01T17:34:24.789Z 13 260-279 144 144 144 231 231 231 F F F
158762024-02-01T17:34:24.789Z 14 280-299 142 142 142 229 229 229 F F F
158772024-02-01T17:34:24.789Z 15 300-319 141 141 141 227 227 227 F F F
158782024-02-01T17:34:24.789Z 16 320-339 147 147 147 234 234 234 F F F
158792024-02-01T17:34:24.789Z 17 340-359 145 145 145 232 232 232 F F F
158802024-02-01T17:34:24.789Z 18 360-379 141 141 141 227 227 227 F F F
158812024-02-01T17:34:24.789Z 19 380-399 144 144 144 231 231 231 F F F
158822024-02-01T17:34:24.789Z 20 400-419 145 145 145 232 232 232 F F F
158832024-02-01T17:34:24.789Z 21 420-439 145 145 145 232 232 232 F F F
158842024-02-01T17:34:24.792Z 22 440-459 147 147 147 234 234 234 F F F
158852024-02-01T17:34:24.792Z 23 460-479 147 147 147 234 234 234 F F F
158862024-02-01T17:34:24.792Z 24 480-499 147 147 147 234 234 234 F F F
158872024-02-01T17:34:24.792Z 25 500-519 146 146 146 233 233 233 F F F
158882024-02-01T17:34:24.792Z 26 520-539 147 147 147 234 234 234 F F F
158892024-02-01T17:34:24.792Z 27 540-559 147 147 147 234 234 234 F F F
158902024-02-01T17:34:24.792Z 28 560-579 147 147 147 234 234 234 F F F
158912024-02-01T17:34:24.792Z 29 580-599 147 147 147 234 234 234 F F F
158922024-02-01T17:34:24.792ZMax gen: 147, Max flush: 234
158932024-02-01T17:34:24.792Z
158942024-02-01T17:34:24.793ZBegin loop 74
158952024-02-01T17:34:24.793ZDownstairs to restart: 2
158962024-02-01T17:34:24.813Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.091598658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
158972024-02-01T17:34:24.813Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092112973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
158982024-02-01T17:34:24.820Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092189541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
158992024-02-01T17:34:24.820Z{"msg":"Crucible 0e8bec53-e289-4e0c-b5f0-218e51e8ced6 has session id: a7e5cfb0-b876-44bc-9b3c-9f1f73eef647","v":0,"name":"crucible","level":30Crucible runtime is spawned
159002024-02-01T17:34:24.820Z,"time":"2024-02-01T17:34:24.092264033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159012024-02-01T17:34:24.820Z{"msg":"Upstairs opts: Upstairs UUID: 0e8bec53-e289-4e0c-b5f0-218e51e8ced6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092435677Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159022024-02-01T17:34:24.820Z{"msg":"Crucible stats registered with UUID: 0e8bec53-e289-4e0c-b5f0-218e51e8ced6","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092503864Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159032024-02-01T17:34:24.820Z{"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092530795Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
159042024-02-01T17:34:24.820Z{"msg":"0e8bec53-e289-4e0c-b5f0-218e51e8ced6 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092550207Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159052024-02-01T17:34:24.820Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092596836Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159062024-02-01T17:34:24.820Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092635824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159072024-02-01T17:34:24.821Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092661285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159082024-02-01T17:34:24.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092684665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159092024-02-01T17:34:24.821Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092707594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159102024-02-01T17:34:24.821Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092796841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159112024-02-01T17:34:24.821Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092868516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159122024-02-01T17:34:24.821Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.092897272Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159132024-02-01T17:34:24.821Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.093017888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159142024-02-01T17:34:24.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.093331418Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159152024-02-01T17:34:24.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.093436023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159162024-02-01T17:34:24.821Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.093480132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"io task","client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159172024-02-01T17:34:24.821Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09489835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159182024-02-01T17:34:24.821Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.094970019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159192024-02-01T17:34:24.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.095116378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159202024-02-01T17:34:24.822Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.095164656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159212024-02-01T17:34:24.822Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.095240524Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159222024-02-01T17:34:24.822Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.095285942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159232024-02-01T17:34:24.822Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.096103527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159242024-02-01T17:34:24.822Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.096284191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159252024-02-01T17:34:24.822Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.096397494Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159262024-02-01T17:34:24.822Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097363075Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159272024-02-01T17:34:24.822Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097432475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159282024-02-01T17:34:24.829Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097465297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159292024-02-01T17:34:24.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09751828Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159302024-02-01T17:34:24.829Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097545604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159312024-02-01T17:34:24.829Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097587121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159322024-02-01T17:34:24.829Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097609968Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159332024-02-01T17:34:24.829Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097650589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159342024-02-01T17:34:24.829Z{"msg":"[0]R flush_numbers[0..12]: [234, 234, 228, 234, 234, 234, 225, 225, 234, 234, 234, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09767607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159352024-02-01T17:34:24.829Z{"msg":"[0]R generation[0..12]: [147, 147, 142, 147, 147, 147, 140, 140, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097717908Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159362024-02-01T17:34:24.830Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097744108Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159372024-02-01T17:34:24.830Z{"msg":"[1]R flush_numbers[0..12]: [234, 234, 228, 234, 234, 234, 225, 225, 234, 234, 234, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097786344Z","hostname":"Wait for a query_work_queue command to finish before sending IO
159382024-02-01T17:34:24.830Zip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159392024-02-01T17:34:24.830Z{"msg":"[1]R generation[0..12]: [147, 147, 142, 147, 147, 147, 140, 140, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097815341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159402024-02-01T17:34:24.830Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097847167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159412024-02-01T17:34:24.830Z{"msg":"[2]R flush_numbers[0..12]: [234, 234, 228, 234, 234, 234, 225, 225, 234, 234, 234, 234]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097878905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159422024-02-01T17:34:24.830Z{"msg":"[2]R generation[0..12]: [147, 147, 142, 147, 147, 147, 140, 140, 147, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097903598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159432024-02-01T17:34:24.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.097953057Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159442024-02-01T17:34:24.831Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09797751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159452024-02-01T17:34:24.831Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098000573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159462024-02-01T17:34:24.831Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098021806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159472024-02-01T17:34:24.832Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098043506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"":"downstairs","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159482024-02-01T17:34:24.833Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09806543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159492024-02-01T17:34:24.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09808543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"0","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159502024-02-01T17:34:24.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09812752Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"1","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159512024-02-01T17:34:24.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098158309Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"client":"2","":"downstairs","session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159522024-02-01T17:34:24.839Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098201114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159532024-02-01T17:34:24.839Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098224201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159542024-02-01T17:34:24.839Z{"msg":"0e8bec53-e289-4e0c-b5f0-218e51e8ced6 is now active with session: a7e5cfb0-b876-44bc-9b3c-9f1f73eef647","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098246628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159552024-02-01T17:34:24.839Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.098285021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724,"session_id":"a7e5cfb0-b876-44bc-9b3c-9f1f73eef647"}
159562024-02-01T17:34:24.839Z{"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:24.09830776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
159572024-02-01T17:34:24.839ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159582024-02-01T17:34:24.840ZLoading write count information from file "/tmp/test_repair_verify.data"
159592024-02-01T17:34:24.840ZRun Repair workload
159602024-02-01T17:34:24.840Z01/30 Write block 493 len 20480 data: 13 11 13 11 11
159612024-02-01T17:34:24.840Z02/30 Write block 464 len 40960 data: 10 9 11 11 10 8 9 10 10 8
159622024-02-01T17:34:24.840Z03/30 Write block 470 len 24576 data: 10 11 11 9 7 8
159632024-02-01T17:34:24.840Z04/30 Write block 560 len 20480 data: 12 11 11 10 9
159642024-02-01T17:34:24.840Z05/30 Read block 67 len 12288
159652024-02-01T17:34:24.841Z06/30 Write block 219 len 24576 data: 15 17 15 15 15 14
159662024-02-01T17:34:24.841Z07/30 Write block 341 len 40960 data: 9 10 10 9 9 10 11 13 12 10
159672024-02-01T17:34:24.841Z08/30 Read block 498 len 4096
159682024-02-01T17:34:24.841Z09/30 Write block 154 len 28672 data: 10 13 10 10 11 11 12
159692024-02-01T17:34:24.841Z10/30 Read block 233 len 4096
159702024-02-01T17:34:24.841Z11/30 Write block 360 len 28672 data: 14 16 17 14 11 10 11
159712024-02-01T17:34:24.841Z12/30 Read block 218 len 4096
159722024-02-01T17:34:24.841Z13/30 Read block 84 len 12288
159732024-02-01T17:34:24.846Z14/30 Read block 243 len 28672
159742024-02-01T17:34:24.846Z15/30 Read block 466 len 12288
159752024-02-01T17:34:24.846Z16/30 Write block 392 len 40960 data: 12 12 10 8 7 9 8 6 6 9
159762024-02-01T17:34:24.847Z17/30 Read block 222 len 24576
159772024-02-01T17:34:24.847Z18/30 Write block 360 len 32768 data: 15 17 18 15 12 11 12 10
159782024-02-01T17:34:24.851Z19/30 Read block 250 len 12288
159792024-02-01T17:34:24.851Z20/30 Read block 537 len 32768
159802024-02-01T17:34:24.851Z21/30 Flush
159812024-02-01T17:34:24.859Z22/30 Write block 542 len 4096 data: 18
159822024-02-01T17:34:24.859Z23/30 Read block 562 len 16384
159832024-02-01T17:34:24.862Z24/30 Write block 441 len 4096 data: 13
159842024-02-01T17:34:24.862Z25/30 Read block 358 len 4096
159852024-02-01T17:34:24.862Z26/30 Write block 237 len 40960 data: 10 10 10 9 7 10 10 10 10 10
159862024-02-01T17:34:24.917Z27/30 Write block 41 len 36864 data: 12 11 13 14 13 14 10 13 11
159872024-02-01T17:34:24.917Z28/30 Read block 486 len 4096
159882024-02-01T17:34:24.917Z29/30 Read block 205 len 32768
159892024-02-01T17:34:24.917Z30/30 Read block 227 len 24576
159902024-02-01T17:34:24.918Z----------------------------------------------------------------
159912024-02-01T17:34:24.918Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
159922024-02-01T17:34:24.918ZStates: Active Active Active
159932024-02-01T17:34:24.918ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
159942024-02-01T17:34:24.918Z 1 Acked 1000 Write 5 Done Done Done false
159952024-02-01T17:34:24.920Z 2 Acked 1001 Write 10 Done Done Sent false
159962024-02-01T17:34:24.920Z 3 Acked 1002 Write 6 Done Done Sent false
159972024-02-01T17:34:24.920Z 4 Acked 1003 Write 5 Done Done Sent false
159982024-02-01T17:34:24.921Z 5 Acked 1004 Read 3 Done Done Sent false
159992024-02-01T17:34:24.921Z 6 Acked 1005 Write 6 Done Done Sent false
160002024-02-01T17:34:24.921Z 7 Acked 1006 Write 10 Done Done Sent false
160012024-02-01T17:34:24.921Z 8 Acked 1007 Read 1 Done Done Sent false
160022024-02-01T17:34:24.921Z 9 Acked 1008 Write 7 Done Done Sent false
160032024-02-01T17:34:24.921Z 10 Acked 1009 Read 1 Done Done Sent false
160042024-02-01T17:34:24.921Z 11 Acked 1010 Write 7 Done Done Sent false
160052024-02-01T17:34:24.921Z 12 Acked 1011 Read 1 Done Done Sent false
160062024-02-01T17:34:24.922Z 13 Acked 1012 Read 3 Done Done Sent false
160072024-02-01T17:34:24.922Z 14 Acked 1013 Read 7 Done Done Sent false
160082024-02-01T17:34:24.922Z 15 Acked 1014 Read 3 Done Done Sent false
160092024-02-01T17:34:24.922Z 16 Acked 1015 Write 10 Done Done Sent false
160102024-02-01T17:34:24.922Z 17 Acked 1016 Read 6 Done Done Sent false
160112024-02-01T17:34:24.922Z 18 Acked 1017 Write 8 Done Done Sent false
160122024-02-01T17:34:24.922Z 19 Acked 1018 Read 3 Done Done Sent false
160132024-02-01T17:34:24.923Z 20 Acked 1019 Read 8 Done Done Sent false
160142024-02-01T17:34:24.923Z 21 Acked 1020 Flush 0 Done Done Sent false
160152024-02-01T17:34:24.923Z 22 Acked 1021 Write 1 Done Done Sent false
160162024-02-01T17:34:24.923Z 23 Acked 1022 Read 4 Done Done Sent false
160172024-02-01T17:34:24.923Z 24 Acked 1023 Write 1 Done Done Sent false
160182024-02-01T17:34:24.924Z 25 Acked 1024 Read 1 Done Done Sent false
160192024-02-01T17:34:24.924Z 26 Acked 1025 Write 10 Done Done Sent false
160202024-02-01T17:34:24.924Z 27 Acked 1026 Write 9 Done Done Sent false
160212024-02-01T17:34:24.924Z 28 Acked 1027 Read 1 Done Done Sent false
160222024-02-01T17:34:24.924Z 29 Acked 1028 Read 8 Done Done Sent false
160232024-02-01T17:34:24.924Z 30 Acked 1029 Read 6 Done Done Sent false
160242024-02-01T17:34:24.924Z STATES DS:0 DS:1 DS:2 TOTAL
160252024-02-01T17:34:24.924Z New 0 0 0 0
160262024-02-01T17:34:24.925Z Sent 0 0 29 29
160272024-02-01T17:34:24.925Z Done 30 30 1 61
160282024-02-01T17:34:24.925Z Skipped 0 0 0 0
160292024-02-01T17:34:24.925Z Error 0 0 0 0
160302024-02-01T17:34:24.925ZLast Flush: 1020 1020 0
160312024-02-01T17:34:24.925ZDownstairs last five completed:
160322024-02-01T17:34:24.925ZUpstairs last five completed: 30 29 28 27 26
160332024-02-01T17:34:24.925Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:24.145083014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1724}
160342024-02-01T17:34:24.925ZWrote out file "/tmp/test_repair_verify.data"
160352024-02-01T17:34:24.925Z
160362024-02-01T17:34:24.925ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160372024-02-01T17:34:24.941ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160382024-02-01T17:34:24.942Z 0 000-019 147 147 147 234 234 234 F F F
160392024-02-01T17:34:24.942Z 1 020-039 147 147 147 234 234 234 F F F
160402024-02-01T17:34:24.942Z 2 040-059 142 142 142 228 228 228 T T F <---
160412024-02-01T17:34:24.942Z 3 060-079 147 147 147 234 234 234 F F F
160422024-02-01T17:34:24.942Z 4 080-099 147 147 147 234 234 234 F F F
160432024-02-01T17:34:24.942Z 5 100-119 147 147 147 234 234 234 F F F
160442024-02-01T17:34:24.943Z 6 120-139 140 140 140 225 225 225 F F F
160452024-02-01T17:34:24.943Z 7 140-159 148 148 140 235 235 225 F F F <---
160462024-02-01T17:34:24.943Z 8 160-179 148 148 147 235 235 234 F F F <---
160472024-02-01T17:34:24.943Z 9 180-199 147 147 147 234 234 234 F F F
160482024-02-01T17:34:24.944Z 10 200-219 148 148 147 235 235 234 F F F <---
160492024-02-01T17:34:24.944Z 11 220-239 148 148 147 235 235 234 T T F <---
160502024-02-01T17:34:24.944Z 12 240-259 144 144 144 231 231 231 T T F <---
160512024-02-01T17:34:24.946Z 13 260-279 144 144 144 231 231 231 F F F
160522024-02-01T17:34:24.947Z 14 280-299 142 142 142 229 229 229 F F F
160532024-02-01T17:34:24.947Z 15 300-319 141 141 141 227 227 227 F F F
160542024-02-01T17:34:24.947Z 16 320-339 147 147 147 234 234 234 F F F
160552024-02-01T17:34:24.947Z 17 340-359 148 148 145 235 235 232 F F F <---
160562024-02-01T17:34:24.947Z 18 360-379 148 148 141 235 235 227 F F F <---
160572024-02-01T17:34:24.947Z 19 380-399 148 148 144 235 235 231 F F F <---
160582024-02-01T17:34:24.947Z 20 400-419 148 148 145 235 235 232 F F F <---
160592024-02-01T17:34:24.947Z 21 420-439 145 145 145 232 232 232 F F F
160602024-02-01T17:34:24.947Z 22 440-459 147 147 147 234 234 234 T T F <---
160612024-02-01T17:34:24.947Z 23 460-479 148 148 147 235 235 234 F F F <---
160622024-02-01T17:34:24.947Z 24 480-499 148 148 147 235 235 234 F F T <---
160632024-02-01T17:34:24.948Z 25 500-519 146 146 146 233 233 233 F F F
160642024-02-01T17:34:24.948Z 26 520-539 147 147 147 234 234 234 F F F
160652024-02-01T17:34:24.948Z 27 540-559 147 147 147 234 234 234 T T F <---
160662024-02-01T17:34:24.948Z 28 560-579 148 148 147 235 235 234 F F F <---
160672024-02-01T17:34:24.948Z 29 580-599 147 147 147 234 234 234 F F F
160682024-02-01T17:34:24.949ZMax gen: 148, Max flush: 235
160692024-02-01T17:34:24.949ZError: Difference in extent metadata found!
160702024-02-01T17:34:24.949ZA Difference in extent metadata is expected here
160712024-02-01T17:34:24.949ZOn loop 74
160722024-02-01T17:34:24.949Z
160732024-02-01T17:34:24.949ZVerifying data now
160742024-02-01T17:34:30.179ZNo change after verify
160752024-02-01T17:34:30.179ZLoop: 74 Downstairs dump after verify (and repair):
160762024-02-01T17:34:30.203ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160772024-02-01T17:34:30.203Z 0 000-019 147 147 147 234 234 234 F F F
160782024-02-01T17:34:30.203Z 1 020-039 147 147 147 234 234 234 F F F
160792024-02-01T17:34:30.203Z 2 040-059 149 149 149 236 236 236 F F F
160802024-02-01T17:34:30.204Z 3 060-079 147 147 147 234 234 234 F F F
160812024-02-01T17:34:30.204Z 4 080-099 147 147 147 234 234 234 F F F
160822024-02-01T17:34:30.204Z 5 100-119 147 147 147 234 234 234 F F F
160832024-02-01T17:34:30.217Z 6 120-139 140 140 140 225 225 225 F F F
160842024-02-01T17:34:30.217Z 7 140-159 148 148 148 235 235 235 F F F
160852024-02-01T17:34:30.217Z 8 160-179 148 148 148 235 235 235 F F F
160862024-02-01T17:34:30.218Z 9 180-199 147 147 147 234 234 234 F F F
160872024-02-01T17:34:30.218Z 10 200-219 148 148 148 235 235 235 F F F
160882024-02-01T17:34:30.218Z 11 220-239 149 149 149 236 236 236 F F F
160892024-02-01T17:34:30.218Z 12 240-259 149 149 149 236 236 236 F F F
160902024-02-01T17:34:30.218Z 13 260-279 144 144 144 231 231 231 F F F
160912024-02-01T17:34:30.218Z 14 280-299 142 142 142 229 229 229 F F F
160922024-02-01T17:34:30.218Z 15 300-319 141 141 141 227 227 227 F F F
160932024-02-01T17:34:30.218Z 16 320-339 147 147 147 234 234 234 F F F
160942024-02-01T17:34:30.219Z 17 340-359 148 148 148 235 235 235 F F F
160952024-02-01T17:34:30.219Z 18 360-379 148 148 148 235 235 235 F F F
160962024-02-01T17:34:30.219Z 19 380-399 148 148 148 235 235 235 F F F
160972024-02-01T17:34:30.221Z 20 400-419 148 148 148 235 235 235 F F F
160982024-02-01T17:34:30.221Z 21 420-439 145 145 145 232 232 232 F F F
160992024-02-01T17:34:30.221Z 22 440-459 149 149 149 236 236 236 F F F
161002024-02-01T17:34:30.221Z 23 460-479 148 148 148 235 235 235 F F F
161012024-02-01T17:34:30.221Z 24 480-499 148 148 148 235 235 235 F F F
161022024-02-01T17:34:30.221Z 25 500-519 146 146 146 233 233 233 F F F
161032024-02-01T17:34:30.222Z 26 520-539 147 147 147 234 234 234 F F F
161042024-02-01T17:34:30.222Z 27 540-559 149 149 149 236 236 236 F F F
161052024-02-01T17:34:30.222Z 28 560-579 148 148 148 235 235 235 F F F
161062024-02-01T17:34:30.222Z 29 580-599 147 147 147 234 234 234 F F F
161072024-02-01T17:34:30.223ZMax gen: 149, Max flush: 236
161082024-02-01T17:34:30.223Z
161092024-02-01T17:34:30.223ZBegin loop 75
161102024-02-01T17:34:30.223ZDownstairs to restart: 2
161112024-02-01T17:34:30.646Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.516476114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
161122024-02-01T17:34:30.646Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.516981055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
161132024-02-01T17:34:30.646Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517065115Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
161142024-02-01T17:34:30.647Z{"msg":"Crucible 33febf2d-0f27-4f92-b1a3-7713b8b8a658 has session id: 73187b2a-ef8a-47a0-a194-cc61af42ebe0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517104824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161152024-02-01T17:34:30.647Z{"msg":"Upstairs opts: Upstairs UUID: 33febf2d-0f27-4f92-b1a3-7713b8b8a658, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"Crucible runtime is spawned
161162024-02-01T17:34:30.647Zname":"crucible","level":30,"time":"2024-02-01T17:34:29.517225681Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161172024-02-01T17:34:30.647Z{"msg":"Crucible stats registered with UUID: 33febf2d-0f27-4f92-b1a3-7713b8b8a658","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517309141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161182024-02-01T17:34:30.647Z{"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517354435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
161192024-02-01T17:34:30.647Z{"msg":"33febf2d-0f27-4f92-b1a3-7713b8b8a658 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517509834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161202024-02-01T17:34:30.647Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.51757412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161212024-02-01T17:34:30.647Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517662744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161222024-02-01T17:34:30.648Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517798223Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161232024-02-01T17:34:30.648Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517843553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161242024-02-01T17:34:30.649Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517928831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161252024-02-01T17:34:30.649Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.517973329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161262024-02-01T17:34:30.649Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518042291Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161272024-02-01T17:34:30.649Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518141332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161282024-02-01T17:34:30.649Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518213945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161292024-02-01T17:34:30.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518287782Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161302024-02-01T17:34:30.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518335562Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161312024-02-01T17:34:30.649Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.518448105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"io task","client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161322024-02-01T17:34:30.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.519746807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161332024-02-01T17:34:30.650Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.519823014Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161342024-02-01T17:34:30.650Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.519867747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161352024-02-01T17:34:30.651Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.519946945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161362024-02-01T17:34:30.651Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.520038331Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161372024-02-01T17:34:30.651Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.520089276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161382024-02-01T17:34:30.651Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.520818552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161392024-02-01T17:34:30.651Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.520880051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161402024-02-01T17:34:30.651Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.521056761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161412024-02-01T17:34:30.651Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.521768862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161422024-02-01T17:34:30.652Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.521847905Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161432024-02-01T17:34:30.652Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.521875196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161442024-02-01T17:34:30.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522068471Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161452024-02-01T17:34:30.652Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522123605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161462024-02-01T17:34:30.652Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522149947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161472024-02-01T17:34:30.652Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522171898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161482024-02-01T17:34:30.653Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522194711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161492024-02-01T17:34:30.653Z{"msg":"[0]R flush_numbers[0..12]: [234, 234, 236, 234, 234, 234, 225, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522236615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161502024-02-01T17:34:30.653Z{"msg":"[0]R generation[0..12]: [147, 147, 149, 147, 147, 147, 140, 148, 148, 147, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522262457Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161512024-02-01T17:34:30.653Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522304637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161522024-02-01T17:34:30.653Z{"msg":"[1]R flush_numbers[0..12]: [234, 234, 236, 234, 234, 234, 225, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522330519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161532024-02-01T17:34:30.653Z{"msg":"[1]R generation[0..12]: [147, 147, 149, 147, 147, 147, 140, 148, 148, 147, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522371714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161542024-02-01T17:34:30.653ZWait for a query_work_queue command to finish before sending IO
161552024-02-01T17:34:30.653Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522403863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161562024-02-01T17:34:30.654Z{"msg":"[2]R flush_numbers[0..12]: [234, 234, 236, 234, 234, 234, 225, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.52245435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161572024-02-01T17:34:30.654Z{"msg":"[2]R generation[0..12]: [147, 147, 149, 147, 147, 147, 140, 148, 148, 147, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522479704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161582024-02-01T17:34:30.654Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522528302Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161592024-02-01T17:34:30.654Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522552106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161602024-02-01T17:34:30.654Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522574641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161612024-02-01T17:34:30.654Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522617751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161622024-02-01T17:34:30.654Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522641618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"":"downstairs","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161632024-02-01T17:34:30.654Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522681552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161642024-02-01T17:34:30.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.52270326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"0","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161652024-02-01T17:34:30.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522743109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"1","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161662024-02-01T17:34:30.657Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522766754Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"client":"2","":"downstairs","session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161672024-02-01T17:34:30.657Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522806929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161682024-02-01T17:34:30.657Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522829321Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161692024-02-01T17:34:30.657Z{"msg":"33febf2d-0f27-4f92-b1a3-7713b8b8a658 is now active with session: 73187b2a-ef8a-47a0-a194-cc61af42ebe0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522870661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161702024-02-01T17:34:30.657Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522892396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732,"session_id":"73187b2a-ef8a-47a0-a194-cc61af42ebe0"}
161712024-02-01T17:34:30.657Z{"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:29.522931297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
161722024-02-01T17:34:30.657ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161732024-02-01T17:34:30.658ZLoading write count information from file "/tmp/test_repair_verify.data"
161742024-02-01T17:34:30.658ZRun Repair workload
161752024-02-01T17:34:30.658Z01/30 Flush
161762024-02-01T17:34:30.658Z02/30 Write block 343 len 12288 data: 11 10 10
161772024-02-01T17:34:30.659Z03/30 Write block 518 len 4096 data: 11
161782024-02-01T17:34:30.659Z04/30 Flush
161792024-02-01T17:34:30.659Z05/30 Write block 520 len 12288 data: 10 10 9
161802024-02-01T17:34:30.659Z06/30 Write block 122 len 16384 data: 10 10 10 12
161812024-02-01T17:34:30.659Z07/30 Read block 339 len 12288
161822024-02-01T17:34:30.659Z08/30 Read block 113 len 20480
161832024-02-01T17:34:30.659Z09/30 Read block 250 len 20480
161842024-02-01T17:34:30.659Z10/30 Write block 578 len 36864 data: 12 12 13 13 13 11 13 12 12
161852024-02-01T17:34:30.659Z11/30 Read block 51 len 32768
161862024-02-01T17:34:30.660Z12/30 Read block 146 len 4096
161872024-02-01T17:34:30.660Z13/30 Write block 482 len 36864 data: 10 10 11 9 11 14 14 14 15
161882024-02-01T17:34:30.660Z14/30 Read block 576 len 28672
161892024-02-01T17:34:30.660Z15/30 Read block 72 len 4096
161902024-02-01T17:34:30.660Z16/30 Flush
161912024-02-01T17:34:30.660Z17/30 Write block 307 len 16384 data: 7 10 8 8
161922024-02-01T17:34:30.660Z18/30 Read block 547 len 28672
161932024-02-01T17:34:30.660Z19/30 Write block 389 len 16384 data: 10 11 11 13
161942024-02-01T17:34:30.660Z20/30 Read block 123 len 12288
161952024-02-01T17:34:30.660Z21/30 Write block 252 len 40960 data: 9 7 9 10 10 11 14 14 13 13
161962024-02-01T17:34:30.660Z22/30 Read block 177 len 4096
161972024-02-01T17:34:30.660Z23/30 Read block 481 len 40960
161982024-02-01T17:34:30.660Z24/30 Write block 282 len 4096 data: 7
161992024-02-01T17:34:30.660Z25/30 Read block 17 len 20480
162002024-02-01T17:34:30.660Z26/30 Write block 444 len 20480 data: 11 12 10 8 8
162012024-02-01T17:34:30.660Z27/30 Write block 65 len 36864 data: 9 11 12 13 16 13 15 17 16
162022024-02-01T17:34:30.660Z28/30 Write block 283 len 16384 data: 9 10 14 14
162032024-02-01T17:34:30.661Z29/30 Read block 499 len 16384
162042024-02-01T17:34:30.661Z30/30 Read block 563 len 4096
162052024-02-01T17:34:30.661Z----------------------------------------------------------------
162062024-02-01T17:34:30.661Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
162072024-02-01T17:34:30.661ZStates: Active Active Active
162082024-02-01T17:34:30.661ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162092024-02-01T17:34:30.661Z 1 Acked 1000 Flush 0 Done Done Sent false
162102024-02-01T17:34:30.661Z 2 Acked 1001 Write 3 Done Done Sent false
162112024-02-01T17:34:30.667Z 3 Acked 1002 Write 1 Done Done Sent false
162122024-02-01T17:34:30.667Z 4 Acked 1003 Flush 0 Done Done Sent false
162132024-02-01T17:34:30.668Z 5 Acked 1004 Write 3 Done Done Sent false
162142024-02-01T17:34:30.668Z 6 Acked 1005 Write 4 Done Done Sent false
162152024-02-01T17:34:30.668Z 7 Acked 1006 Read 3 Done Done Sent false
162162024-02-01T17:34:30.668Z 8 Acked 1007 Read 5 Done Done Sent false
162172024-02-01T17:34:30.668Z 9 Acked 1008 Read 5 Done Done Sent false
162182024-02-01T17:34:30.668Z 10 Acked 1009 Write 9 Done Done Sent false
162192024-02-01T17:34:30.668Z 11 Acked 1010 Read 8 Done Done Sent false
162202024-02-01T17:34:30.668Z 12 Acked 1011 Read 1 Done Done Sent false
162212024-02-01T17:34:30.668Z 13 Acked 1012 Write 9 Done Done Sent false
162222024-02-01T17:34:30.668Z 14 Acked 1013 Read 7 Done Done Sent false
162232024-02-01T17:34:30.668Z 15 Acked 1014 Read 1 Done Done Sent false
162242024-02-01T17:34:30.668Z 16 Acked 1015 Flush 0 Done Done Sent false
162252024-02-01T17:34:30.668Z 17 Acked 1016 Write 4 Done Done Sent false
162262024-02-01T17:34:30.668Z 18 Acked 1017 Read 7 Done Done Sent false
162272024-02-01T17:34:30.670Z 19 Acked 1018 Write 4 Done Done Sent false
162282024-02-01T17:34:30.670Z 20 Acked 1019 Read 3 Done Done Sent false
162292024-02-01T17:34:30.670Z 21 Acked 1020 Write 10 Done Done Sent false
162302024-02-01T17:34:30.670Z 22 Acked 1021 Read 1 Done Done Sent false
162312024-02-01T17:34:30.670Z 23 Acked 1022 Read 10 Done Done Sent false
162322024-02-01T17:34:30.670Z 24 Acked 1023 Write 1 Done Done Sent false
162332024-02-01T17:34:30.670Z 25 Acked 1024 Read 5 Done Done Sent false
162342024-02-01T17:34:30.670Z 26 Acked 1025 Write 5 Done Done Sent false
162352024-02-01T17:34:30.670Z 27 Acked 1026 Write 9 Done Done Sent false
162362024-02-01T17:34:30.670Z 28 Acked 1027 Write 4 Done Done Sent false
162372024-02-01T17:34:30.670Z 29 Acked 1028 Read 4 Done Done Sent false
162382024-02-01T17:34:30.670Z 30 Acked 1029 Read 1 Done Sent Sent false
162392024-02-01T17:34:30.670Z STATES DS:0 DS:1 DS:2 TOTAL
162402024-02-01T17:34:30.670Z New 0 0 0 0
162412024-02-01T17:34:30.670Z Sent 0 1 30 31
162422024-02-01T17:34:30.671Z Done 30 29 0 59
162432024-02-01T17:34:30.671Z Skipped 0 0 0 0
162442024-02-01T17:34:30.671Z Error 0 0 0 0
162452024-02-01T17:34:30.671ZLast Flush: 1015 1015 0
162462024-02-01T17:34:30.671ZDownstairs last five completed:
162472024-02-01T17:34:30.671ZUpstairs last five completed: 30 29 28 27 26
162482024-02-01T17:34:30.671Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:29.567032755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1732}
162492024-02-01T17:34:30.671ZWrote out file "/tmp/test_repair_verify.data"
162502024-02-01T17:34:30.671Z
162512024-02-01T17:34:30.674ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
162522024-02-01T17:34:30.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162532024-02-01T17:34:30.674Z 0 000-019 147 147 147 234 234 234 F F F
162542024-02-01T17:34:30.674Z 1 020-039 147 147 147 234 234 234 F F F
162552024-02-01T17:34:30.674Z 2 040-059 149 149 149 236 236 236 F F F
162562024-02-01T17:34:30.674Z 3 060-079 147 147 147 234 234 234 T T F <---
162572024-02-01T17:34:30.674Z 4 080-099 147 147 147 234 234 234 F F F
162582024-02-01T17:34:30.674Z 5 100-119 147 147 147 234 234 234 F F F
162592024-02-01T17:34:30.674Z 6 120-139 150 150 140 239 239 225 F F F <---
162602024-02-01T17:34:30.674Z 7 140-159 148 148 148 235 235 235 F F F
162612024-02-01T17:34:30.674Z 8 160-179 148 148 148 235 235 235 F F F
162622024-02-01T17:34:30.674Z 9 180-199 147 147 147 234 234 234 F F F
162632024-02-01T17:34:30.674Z 10 200-219 148 148 148 235 235 235 F F F
162642024-02-01T17:34:30.674Z 11 220-239 149 149 149 236 236 236 F F F
162652024-02-01T17:34:30.674Z 12 240-259 149 149 149 236 236 236 T T F <---
162662024-02-01T17:34:30.674Z 13 260-279 144 144 144 231 231 231 T T F <---
162672024-02-01T17:34:30.674Z 14 280-299 142 142 142 229 229 229 T T F <---
162682024-02-01T17:34:30.674Z 15 300-319 141 141 141 227 227 227 T T F <---
162692024-02-01T17:34:30.674Z 16 320-339 147 147 147 234 234 234 F F F
162702024-02-01T17:34:30.674Z 17 340-359 150 150 148 238 238 235 F F F <---
162712024-02-01T17:34:30.674Z 18 360-379 148 148 148 235 235 235 F F F
162722024-02-01T17:34:30.674Z 19 380-399 148 148 148 235 235 235 T T F <---
162732024-02-01T17:34:30.674Z 20 400-419 148 148 148 235 235 235 F F F
162742024-02-01T17:34:30.674Z 21 420-439 145 145 145 232 232 232 F F F
162752024-02-01T17:34:30.675Z 22 440-459 149 149 149 236 236 236 T T F <---
162762024-02-01T17:34:30.675Z 23 460-479 148 148 148 235 235 235 F F F
162772024-02-01T17:34:30.675Z 24 480-499 150 150 148 239 239 235 F F F <---
162782024-02-01T17:34:30.675Z 25 500-519 150 150 146 238 238 233 F F F <---
162792024-02-01T17:34:30.675Z 26 520-539 150 150 147 239 239 234 F F F <---
162802024-02-01T17:34:30.675Z 27 540-559 149 149 149 236 236 236 F F F
162812024-02-01T17:34:30.675Z 28 560-579 150 150 148 239 239 235 F F F <---
162822024-02-01T17:34:30.675Z 29 580-599 150 150 147 239 239 234 F F F <---
162832024-02-01T17:34:30.675ZMax gen: 150, Max flush: 239
162842024-02-01T17:34:30.675ZError: Difference in extent metadata found!
162852024-02-01T17:34:30.675ZA Difference in extent metadata is expected here
162862024-02-01T17:34:30.675ZOn loop 75
162872024-02-01T17:34:30.675Z
162882024-02-01T17:34:30.675ZVerifying data now
162892024-02-01T17:34:35.534ZNo change after verify
162902024-02-01T17:34:35.534ZLoop: 75 Downstairs dump after verify (and repair):
162912024-02-01T17:34:35.571ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162922024-02-01T17:34:35.571Z 0 000-019 147 147 147 234 234 234 F F F
162932024-02-01T17:34:35.577Z 1 020-039 147 147 147 234 234 234 F F F
162942024-02-01T17:34:35.577Z 2 040-059 149 149 149 236 236 236 F F F
162952024-02-01T17:34:35.577Z 3 060-079 151 151 151 240 240 240 F F F
162962024-02-01T17:34:35.577Z 4 080-099 147 147 147 234 234 234 F F F
162972024-02-01T17:34:35.577Z 5 100-119 147 147 147 234 234 234 F F F
162982024-02-01T17:34:35.577Z 6 120-139 150 150 150 239 239 239 F F F
162992024-02-01T17:34:35.577Z 7 140-159 148 148 148 235 235 235 F F F
163002024-02-01T17:34:35.577Z 8 160-179 148 148 148 235 235 235 F F F
163012024-02-01T17:34:35.577Z 9 180-199 147 147 147 234 234 234 F F F
163022024-02-01T17:34:35.577Z 10 200-219 148 148 148 235 235 235 F F F
163032024-02-01T17:34:35.577Z 11 220-239 149 149 149 236 236 236 F F F
163042024-02-01T17:34:35.577Z 12 240-259 151 151 151 240 240 240 F F F
163052024-02-01T17:34:35.577Z 13 260-279 151 151 151 240 240 240 F F F
163062024-02-01T17:34:35.578Z 14 280-299 151 151 151 240 240 240 F F F
163072024-02-01T17:34:35.578Z 15 300-319 151 151 151 240 240 240 F F F
163082024-02-01T17:34:35.578Z 16 320-339 147 147 147 234 234 234 F F F
163092024-02-01T17:34:35.578Z 17 340-359 150 150 150 238 238 238 F F F
163102024-02-01T17:34:35.578Z 18 360-379 148 148 148 235 235 235 F F F
163112024-02-01T17:34:35.578Z 19 380-399 151 151 151 240 240 240 F F F
163122024-02-01T17:34:35.578Z 20 400-419 148 148 148 235 235 235 F F F
163132024-02-01T17:34:35.578Z 21 420-439 145 145 145 232 232 232 F F F
163142024-02-01T17:34:35.578Z 22 440-459 151 151 151 240 240 240 F F F
163152024-02-01T17:34:35.578Z 23 460-479 148 148 148 235 235 235 F F F
163162024-02-01T17:34:35.578Z 24 480-499 150 150 150 239 239 239 F F F
163172024-02-01T17:34:35.578Z 25 500-519 150 150 150 238 238 238 F F F
163182024-02-01T17:34:35.578Z 26 520-539 150 150 150 239 239 239 F F F
163192024-02-01T17:34:35.586Z 27 540-559 149 149 149 236 236 236 F F F
163202024-02-01T17:34:35.586Z 28 560-579 150 150 150 239 239 239 F F F
163212024-02-01T17:34:35.586Z 29 580-599 150 150 150 239 239 239 F F F
163222024-02-01T17:34:35.586ZMax gen: 151, Max flush: 240
163232024-02-01T17:34:35.586Z
163242024-02-01T17:34:35.586ZBegin loop 76
163252024-02-01T17:34:35.586ZDownstairs to restart: 2
163262024-02-01T17:34:35.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.871074126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
163272024-02-01T17:34:35.598Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.8715881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
163282024-02-01T17:34:35.598Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.871731297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
163292024-02-01T17:34:35.598Z{"msg":"Crucible f84942e2-28ed-48fe-9acc-bdbf448db47c has session id: 277cf6cc-c4f2-4f96-bc37-13173b621448","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.871839657Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163302024-02-01T17:34:35.598Z{"msg":"Upstairs opts: Upstairs UUID: f84942e2-28ed-48fe-9acc-bdbf448db47c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.871893151Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163312024-02-01T17:34:35.608Z{"msg":"Crucible stats registered with UUID: f84942e2-28ed-48fe-9acc-bdbf448db47c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.871943605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163322024-02-01T17:34:35.608ZCrucible runtime is spawned
163332024-02-01T17:34:35.608Z{"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872179555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
163342024-02-01T17:34:35.608Z{"msg":"f84942e2-28ed-48fe-9acc-bdbf448db47c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872369197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163352024-02-01T17:34:35.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872492086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163362024-02-01T17:34:35.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872628069Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163372024-02-01T17:34:35.608Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872716047Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163382024-02-01T17:34:35.608Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872750374Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163392024-02-01T17:34:35.609Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872815314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163402024-02-01T17:34:35.609Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.872860314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163412024-02-01T17:34:35.609Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87299509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163422024-02-01T17:34:35.609Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.873139258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163432024-02-01T17:34:35.609Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.873200198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163442024-02-01T17:34:35.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.873276551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163452024-02-01T17:34:35.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.873305163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163462024-02-01T17:34:35.609Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.873334933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"io task","client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163472024-02-01T17:34:35.609Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87434006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163482024-02-01T17:34:35.609Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.874470686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163492024-02-01T17:34:35.610Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.874519888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163502024-02-01T17:34:35.610Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.874693332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163512024-02-01T17:34:35.610Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.875385964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163522024-02-01T17:34:35.610Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.875425714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163532024-02-01T17:34:35.610Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87545145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163542024-02-01T17:34:35.610Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87555536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163552024-02-01T17:34:35.610Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.876403689Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163562024-02-01T17:34:35.610Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.876455626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163572024-02-01T17:34:35.610Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.876480536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163582024-02-01T17:34:35.610Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87659038Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163592024-02-01T17:34:35.610Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.876723965Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163602024-02-01T17:34:35.610Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.87678727Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163612024-02-01T17:34:35.611Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.876814346Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163622024-02-01T17:34:35.611Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877247639Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163632024-02-01T17:34:35.611Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877330942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163642024-02-01T17:34:35.641Z{"msg":"[0]R flush_numbers[0..12]: [234, 234, 236, 240, 234, 234, 239, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877377573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163652024-02-01T17:34:35.642Z{"msg":"[0]R generation[0..12]: [147, 147, 149, 151, 147, 147, 150, 148, 148, 147, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877449932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163662024-02-01T17:34:35.642Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877522483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163672024-02-01T17:34:35.642Z{"msg":"[1]R flush_numbers[0..12]: [234, 234, 236, 240, 234, 234, 239, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877598801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163682024-02-01T17:34:35.642Z{"msg":"[1]R generation[0..12]: [147, 147, 149, 151, 147, 147, 150, 148, 148, 147, 148, 149]","v":Wait for a query_work_queue command to finish before sending IO
163692024-02-01T17:34:35.642Z0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877759493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163702024-02-01T17:34:35.642Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877837664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163712024-02-01T17:34:35.642Z{"msg":"[2]R flush_numbers[0..12]: [234, 234, 236, 240, 234, 234, 239, 235, 235, 234, 235, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877914066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163722024-02-01T17:34:35.642Z{"msg":"[2]R generation[0..12]: [147, 147, 149, 151, 147, 147, 150, 148, 148, 147, 148, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.877961337Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163732024-02-01T17:34:35.642Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878033036Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163742024-02-01T17:34:35.642Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878112348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163752024-02-01T17:34:35.642Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878164676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163762024-02-01T17:34:35.642Z{"msg":"Next flush: 241","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878189048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163772024-02-01T17:34:35.642Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878210667Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"":"downstairs","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163782024-02-01T17:34:35.642Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878266786Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163792024-02-01T17:34:35.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878303064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"0","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163802024-02-01T17:34:35.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878349845Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"1","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163812024-02-01T17:34:35.643Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878374414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"client":"2","":"downstairs","session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163822024-02-01T17:34:35.643Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878397977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163832024-02-01T17:34:35.644Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878418514Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163842024-02-01T17:34:35.644Z{"msg":"f84942e2-28ed-48fe-9acc-bdbf448db47c is now active with session: 277cf6cc-c4f2-4f96-bc37-13173b621448","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878451129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163852024-02-01T17:34:35.644Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878482851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740,"session_id":"277cf6cc-c4f2-4f96-bc37-13173b621448"}
163862024-02-01T17:34:35.644Z{"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:34.878529413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
163872024-02-01T17:34:35.644ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163882024-02-01T17:34:35.644ZLoading write count information from file "/tmp/test_repair_verify.data"
163892024-02-01T17:34:35.644ZRun Repair workload
163902024-02-01T17:34:35.644Z01/30 Write block 204 len 8192 data: 16 13
163912024-02-01T17:34:35.644Z02/30 Flush
163922024-02-01T17:34:35.644Z03/30 Write block 255 len 20480 data: 11 11 12 15 15
163932024-02-01T17:34:35.645Z04/30 Write block 242 len 32768 data: 11 11 11 11 11 10 11 8
163942024-02-01T17:34:35.645Z05/30 Flush
163952024-02-01T17:34:35.645Z06/30 Write block 47 len 24576 data: 11 14 12 13 13 11
163962024-02-01T17:34:35.645Z07/30 Read block 379 len 32768
163972024-02-01T17:34:35.646Z08/30 Flush
163982024-02-01T17:34:35.646Z09/30 Write block 485 len 32768 data: 10 12 15 15 15 16 15 17
163992024-02-01T17:34:35.646Z10/30 Read block 435 len 12288
164002024-02-01T17:34:35.646Z11/30 Read block 100 len 4096
164012024-02-01T17:34:35.646Z12/30 Write block 550 len 4096 data: 8
164022024-02-01T17:34:35.647Z13/30 Read block 261 len 24576
164032024-02-01T17:34:35.647Z14/30 Write block 513 len 12288 data: 10 8 8
164042024-02-01T17:34:35.647Z15/30 Read block 1 len 20480
164052024-02-01T17:34:35.647Z16/30 Write block 252 len 20480 data: 10 8 10 12 12
164062024-02-01T17:34:35.647Z17/30 Read block 429 len 20480
164072024-02-01T17:34:35.647Z18/30 Write block 358 len 36864 data: 14 13 16 18 19 16 13 12 13
164082024-02-01T17:34:35.647Z19/30 Read block 395 len 16384
164092024-02-01T17:34:35.647Z20/30 Write block 318 len 8192 data: 13 14
164102024-02-01T17:34:35.647Z21/30 Write block 579 len 8192 data: 13 14
164112024-02-01T17:34:35.647Z22/30 Flush
164122024-02-01T17:34:35.647Z23/30 Write block 400 len 8192 data: 7 10
164132024-02-01T17:34:35.647Z24/30 Read block 124 len 24576
164142024-02-01T17:34:35.654Z25/30 Write block 572 len 24576 data: 8 8 7 11 10 10
164152024-02-01T17:34:35.654Z26/30 Flush
164162024-02-01T17:34:35.654Z27/30 Write block 22 len 8192 data: 11 10
164172024-02-01T17:34:35.654Z28/30 Read block 553 len 28672
164182024-02-01T17:34:35.654Z29/30 Read block 188 len 32768
164192024-02-01T17:34:35.655Z30/30 Read block 137 len 12288
164202024-02-01T17:34:35.655Z----------------------------------------------------------------
164212024-02-01T17:34:35.655Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:22
164222024-02-01T17:34:35.656ZStates: Active Active Active
164232024-02-01T17:34:35.656ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164242024-02-01T17:34:35.656Z 9 Acked 1008 Write 8 Done Done Sent false
164252024-02-01T17:34:35.656Z 10 Acked 1009 Read 3 Done Done Sent false
164262024-02-01T17:34:35.656Z 11 Acked 1010 Read 1 Done Done Sent false
164272024-02-01T17:34:35.656Z 12 Acked 1011 Write 1 Done Done Sent false
164282024-02-01T17:34:35.656Z 13 Acked 1012 Read 6 Done Done Sent false
164292024-02-01T17:34:35.656Z 14 Acked 1013 Write 3 Done Done Sent false
164302024-02-01T17:34:35.657Z 15 Acked 1014 Read 5 Done Done Sent false
164312024-02-01T17:34:35.657Z 16 Acked 1015 Write 5 Done Done Sent false
164322024-02-01T17:34:35.657Z 17 Acked 1016 Read 5 Done Done Sent false
164332024-02-01T17:34:35.657Z 18 Acked 1017 Write 9 Done Done Sent false
164342024-02-01T17:34:35.657Z 19 Acked 1018 Read 4 Done Done Sent false
164352024-02-01T17:34:35.657Z 20 Acked 1019 Write 2 Done Done Sent false
164362024-02-01T17:34:35.657Z 21 Acked 1020 Write 2 Done Done Sent false
164372024-02-01T17:34:35.657Z 22 Acked 1021 Flush 0 Done Done Sent false
164382024-02-01T17:34:35.657Z 23 Acked 1022 Write 2 Done Done Sent false
164392024-02-01T17:34:35.657Z 24 Acked 1023 Read 6 Done Done Sent false
164402024-02-01T17:34:35.658Z 25 Acked 1024 Write 6 Done Done Sent false
164412024-02-01T17:34:35.658Z 26 Acked 1025 Flush 0 Done Done Sent false
164422024-02-01T17:34:35.658Z 27 Acked 1026 Write 2 Done Done Sent false
164432024-02-01T17:34:35.658Z 28 Acked 1027 Read 7 Done Done Sent false
164442024-02-01T17:34:35.658Z 29 Acked 1028 Read 8 Done Done Sent false
164452024-02-01T17:34:35.658Z 30 Acked 1029 Read 3 Done Sent Sent false
164462024-02-01T17:34:35.658Z STATES DS:0 DS:1 DS:2 TOTAL
164472024-02-01T17:34:35.658Z New 0 0 0 0
164482024-02-01T17:34:35.658Z Sent 0 1 22 23
164492024-02-01T17:34:35.659Z Done 22 21 0 43
164502024-02-01T17:34:35.659Z Skipped 0 0 0 0
164512024-02-01T17:34:35.659Z Error 0 0 0 0
164522024-02-01T17:34:35.659ZLast Flush: 1025 1025 1007
164532024-02-01T17:34:35.659ZDownstairs last five completed: 1007 1006 1005 1004 1003
164542024-02-01T17:34:35.659ZUpstairs last five completed: 30 29 28 27 26
164552024-02-01T17:34:35.659Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:34.928883658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1740}
164562024-02-01T17:34:35.659ZWrote out file "/tmp/test_repair_verify.data"
164572024-02-01T17:34:35.664Z
164582024-02-01T17:34:35.664ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164592024-02-01T17:34:35.683ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164602024-02-01T17:34:35.684Z 0 000-019 147 147 147 234 234 234 F F F
164612024-02-01T17:34:35.684Z 1 020-039 147 147 147 234 234 234 T T F <---
164622024-02-01T17:34:35.684Z 2 040-059 152 152 152 243 243 243 F F F
164632024-02-01T17:34:35.691Z 3 060-079 151 151 151 240 240 240 F F F
164642024-02-01T17:34:35.691Z 4 080-099 147 147 147 234 234 234 F F F
164652024-02-01T17:34:35.691Z 5 100-119 147 147 147 234 234 234 F F F
164662024-02-01T17:34:35.692Z 6 120-139 150 150 150 239 239 239 F F F
164672024-02-01T17:34:35.693Z 7 140-159 148 148 148 235 235 235 F F F
164682024-02-01T17:34:35.693Z 8 160-179 148 148 148 235 235 235 F F F
164692024-02-01T17:34:35.693Z 9 180-199 147 147 147 234 234 234 F F F
164702024-02-01T17:34:35.693Z 10 200-219 152 152 152 241 241 241 F F F
164712024-02-01T17:34:35.693Z 11 220-239 149 149 149 236 236 236 F F F
164722024-02-01T17:34:35.693Z 12 240-259 152 152 152 244 244 242 F F F <---
164732024-02-01T17:34:35.693Z 13 260-279 151 151 151 240 240 240 F F F
164742024-02-01T17:34:35.693Z 14 280-299 151 151 151 240 240 240 F F F
164752024-02-01T17:34:35.693Z 15 300-319 152 152 151 244 244 240 F F F <---
164762024-02-01T17:34:35.694Z 16 320-339 147 147 147 234 234 234 F F F
164772024-02-01T17:34:35.694Z 17 340-359 152 152 150 244 244 238 F F F <---
164782024-02-01T17:34:35.694Z 18 360-379 152 152 148 244 244 235 F F F <---
164792024-02-01T17:34:35.694Z 19 380-399 151 151 151 240 240 240 F F F
164802024-02-01T17:34:35.694Z 20 400-419 152 152 148 245 245 235 F F F <---
164812024-02-01T17:34:35.694Z 21 420-439 145 145 145 232 232 232 F F F
164822024-02-01T17:34:35.694Z 22 440-459 151 151 151 240 240 240 F F F
164832024-02-01T17:34:35.694Z 23 460-479 148 148 148 235 235 235 F F F
164842024-02-01T17:34:35.694Z 24 480-499 152 152 150 244 244 239 F F F <---
164852024-02-01T17:34:35.695Z 25 500-519 152 152 150 244 244 238 F F F <---
164862024-02-01T17:34:35.695Z 26 520-539 150 150 150 239 239 239 F F F
164872024-02-01T17:34:35.695Z 27 540-559 152 152 149 244 244 236 F F F <---
164882024-02-01T17:34:35.695Z 28 560-579 152 152 150 245 245 239 F F F <---
164892024-02-01T17:34:35.695Z 29 580-599 152 152 150 244 244 239 F F F <---
164902024-02-01T17:34:35.717ZMax gen: 152, Max flush: 245
164912024-02-01T17:34:35.717ZError: Difference in extent metadata found!
164922024-02-01T17:34:35.717ZA Difference in extent metadata is expected here
164932024-02-01T17:34:35.717ZOn loop 76
164942024-02-01T17:34:35.718Z
164952024-02-01T17:34:35.723ZVerifying data now
164962024-02-01T17:34:40.647ZNo change after verify
164972024-02-01T17:34:40.647ZLoop: 76 Downstairs dump after verify (and repair):
164982024-02-01T17:34:40.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164992024-02-01T17:34:40.671Z 0 000-019 147 147 147 234 234 234 F F F
165002024-02-01T17:34:40.676Z 1 020-039 153 153 153 246 246 246 F F F
165012024-02-01T17:34:40.677Z 2 040-059 152 152 152 243 243 243 F F F
165022024-02-01T17:34:40.677Z 3 060-079 151 151 151 240 240 240 F F F
165032024-02-01T17:34:40.677Z 4 080-099 147 147 147 234 234 234 F F F
165042024-02-01T17:34:40.677Z 5 100-119 147 147 147 234 234 234 F F F
165052024-02-01T17:34:40.677Z 6 120-139 150 150 150 239 239 239 F F F
165062024-02-01T17:34:40.677Z 7 140-159 148 148 148 235 235 235 F F F
165072024-02-01T17:34:40.678Z 8 160-179 148 148 148 235 235 235 F F F
165082024-02-01T17:34:40.678Z 9 180-199 147 147 147 234 234 234 F F F
165092024-02-01T17:34:40.678Z 10 200-219 152 152 152 241 241 241 F F F
165102024-02-01T17:34:40.678Z 11 220-239 149 149 149 236 236 236 F F F
165112024-02-01T17:34:40.678Z 12 240-259 152 152 152 244 244 244 F F F
165122024-02-01T17:34:40.678Z 13 260-279 151 151 151 240 240 240 F F F
165132024-02-01T17:34:40.678Z 14 280-299 151 151 151 240 240 240 F F F
165142024-02-01T17:34:40.678Z 15 300-319 152 152 152 244 244 244 F F F
165152024-02-01T17:34:40.678Z 16 320-339 147 147 147 234 234 234 F F F
165162024-02-01T17:34:40.678Z 17 340-359 152 152 152 244 244 244 F F F
165172024-02-01T17:34:40.678Z 18 360-379 152 152 152 244 244 244 F F F
165182024-02-01T17:34:40.678Z 19 380-399 151 151 151 240 240 240 F F F
165192024-02-01T17:34:40.678Z 20 400-419 152 152 152 245 245 245 F F F
165202024-02-01T17:34:40.678Z 21 420-439 145 145 145 232 232 232 F F F
165212024-02-01T17:34:40.678Z 22 440-459 151 151 151 240 240 240 F F F
165222024-02-01T17:34:40.678Z 23 460-479 148 148 148 235 235 235 F F F
165232024-02-01T17:34:40.678Z 24 480-499 152 152 152 244 244 244 F F F
165242024-02-01T17:34:40.679Z 25 500-519 152 152 152 244 244 244 F F F
165252024-02-01T17:34:40.679Z 26 520-539 150 150 150 239 239 239 F F F
165262024-02-01T17:34:40.679Z 27 540-559 152 152 152 244 244 244 F F F
165272024-02-01T17:34:40.679Z 28 560-579 152 152 152 245 245 245 F F F
165282024-02-01T17:34:40.679Z 29 580-599 152 152 152 244 244 244 F F F
165292024-02-01T17:34:40.680ZMax gen: 153, Max flush: 246
165302024-02-01T17:34:40.680Z
165312024-02-01T17:34:40.680ZBegin loop 77
165322024-02-01T17:34:40.680ZDownstairs to restart: 2
165332024-02-01T17:34:40.705Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.985873881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
165342024-02-01T17:34:40.705Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986343898Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
165352024-02-01T17:34:40.709Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986419406Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
165362024-02-01T17:34:40.709Z{"msg":"Crucible 4f2890e0-0bc0-47d7-b998-6deca38079df has session id: 736a0d47-6f04-43ed-8fac-63b6a56055f2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986502138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165372024-02-01T17:34:40.709Z{"msg":"Crucible runtime is spawned
165382024-02-01T17:34:40.709ZUpstairs opts: Upstairs UUID: 4f2890e0-0bc0-47d7-b998-6deca38079df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986629275Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165392024-02-01T17:34:40.709Z{"msg":"Crucible stats registered with UUID: 4f2890e0-0bc0-47d7-b998-6deca38079df","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986777431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165402024-02-01T17:34:40.709Z{"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986819809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
165412024-02-01T17:34:40.709Z{"msg":"4f2890e0-0bc0-47d7-b998-6deca38079df active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986890045Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165422024-02-01T17:34:40.709Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.986931357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165432024-02-01T17:34:40.709Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.98700306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165442024-02-01T17:34:40.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987051163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165452024-02-01T17:34:40.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987090817Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165462024-02-01T17:34:40.710Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987164002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165472024-02-01T17:34:40.710Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.98720757Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165482024-02-01T17:34:40.710Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.98734871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165492024-02-01T17:34:40.710Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987530552Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165502024-02-01T17:34:40.710Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987613776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165512024-02-01T17:34:40.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987698068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165522024-02-01T17:34:40.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987740993Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165532024-02-01T17:34:40.710Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.987814393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"io task","client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165542024-02-01T17:34:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.989141186Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165552024-02-01T17:34:40.711Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.989206623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165562024-02-01T17:34:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.989448341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165572024-02-01T17:34:40.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.98952384Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165582024-02-01T17:34:40.711Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.989569807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165592024-02-01T17:34:40.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.989610547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165602024-02-01T17:34:40.729Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.990116862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165612024-02-01T17:34:40.729Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.990592185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165622024-02-01T17:34:40.729Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.990836102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165632024-02-01T17:34:40.729Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991221439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165642024-02-01T17:34:40.729Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991290857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165652024-02-01T17:34:40.729Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991330134Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165662024-02-01T17:34:40.730Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991376181Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165672024-02-01T17:34:40.730Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991401419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165682024-02-01T17:34:40.730Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991442269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165692024-02-01T17:34:40.730Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991746546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165702024-02-01T17:34:40.730Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991796675Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165712024-02-01T17:34:40.730Z{"msg":"[0]R flush_numbers[0..12]: [234, 246, 243, 240, 234, 234, 239, 235, 235, 234, 241, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.99182771Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165722024-02-01T17:34:40.730Z{"msg":"[0]R generation[0..12]: [147, 153, 152, 151, 147, 147, 150, 148, 148, 147, 152, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991870512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165732024-02-01T17:34:40.730Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991896345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165742024-02-01T17:34:40.730Z{"msg":"[1]R flush_numbers[0..12]: [234, 246, 243, 240, 234, 234, 239, 235, 235, 234, 241, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991937662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165752024-02-01T17:34:40.730Z{"msg":"[1]R generation[0..12]: [147, 153, 152, 151, 147, 147, 150, 148, 148, 147, 152, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.991962837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165762024-02-01T17:34:40.730Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.99200455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165772024-02-01T17:34:40.730Z{"msg":"[2]R flush_numbers[0..12]: [234, 246, 243, 240, 234, 234, 239, 235, 235, 234, 241, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992029686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165782024-02-01T17:34:40.730Z{"msg":"[2]R generation[0..12]: [147, 153, 152, 151, 147, 147, 150, 148, 148, 147, 152, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992071006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"Wait for a query_work_queue command to finish before sending IO
165792024-02-01T17:34:40.731Z":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165802024-02-01T17:34:40.731Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992100721Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165812024-02-01T17:34:40.731Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992123761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165822024-02-01T17:34:40.731Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992146219Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165832024-02-01T17:34:40.731Z{"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992167286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165842024-02-01T17:34:40.731Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992207208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"":"downstairs","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165852024-02-01T17:34:40.731Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992264143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165862024-02-01T17:34:40.731Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992288129Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"0","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165872024-02-01T17:34:40.731Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992312961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"1","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165882024-02-01T17:34:40.731Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992341294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"client":"2","":"downstairs","session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165892024-02-01T17:34:40.731Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992392139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165902024-02-01T17:34:40.732Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992441233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165912024-02-01T17:34:40.732Z{"msg":"4f2890e0-0bc0-47d7-b998-6deca38079df is now active with session: 736a0d47-6f04-43ed-8fac-63b6a56055f2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992466667Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165922024-02-01T17:34:40.732Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.992523301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748,"session_id":"736a0d47-6f04-43ed-8fac-63b6a56055f2"}
165932024-02-01T17:34:40.732Z{"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:39.99258046Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
165942024-02-01T17:34:40.732ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
165952024-02-01T17:34:40.732ZLoading write count information from file "/tmp/test_repair_verify.data"
165962024-02-01T17:34:40.732ZRun Repair workload
165972024-02-01T17:34:40.732Z01/30 Write block 36 len 12288 data: 13 14 15
165982024-02-01T17:34:40.733Z02/30 Read block 6 len 40960
165992024-02-01T17:34:40.733Z03/30 Read block 135 len 40960
166002024-02-01T17:34:40.733Z04/30 Read block 173 len 28672
166012024-02-01T17:34:40.733Z05/30 Write block 176 len 8192 data: 16 14
166022024-02-01T17:34:40.733Z06/30 Write block 172 len 36864 data: 18 16 12 13 17 15 14 14 12
166032024-02-01T17:34:40.734Z07/30 Write block 502 len 32768 data: 11 10 9 8 8 8 11 12
166042024-02-01T17:34:40.734Z08/30 Write block 143 len 32768 data: 11 10 10 10 10 14 14 10
166052024-02-01T17:34:40.734Z09/30 Write block 452 len 32768 data: 7 7 7 7 11 16 16 17
166062024-02-01T17:34:40.734Z10/30 Write block 201 len 24576 data: 12 15 16 17 14 12
166072024-02-01T17:34:40.738Z11/30 Read block 217 len 4096
166082024-02-01T17:34:40.738Z12/30 Write block 211 len 12288 data: 14 12 11
166092024-02-01T17:34:40.738Z13/30 Read block 218 len 16384
166102024-02-01T17:34:40.738Z14/30 Flush
166112024-02-01T17:34:40.738Z15/30 Write block 464 len 28672 data: 11 10 12 12 11 9 11
166122024-02-01T17:34:40.738Z16/30 Read block 590 len 12288
166132024-02-01T17:34:40.738Z17/30 Read block 91 len 12288
166142024-02-01T17:34:40.741Z18/30 Read block 529 len 28672
166152024-02-01T17:34:40.741Z19/30 Read block 235 len 40960
166162024-02-01T17:34:40.744Z20/30 Read block 548 len 28672
166172024-02-01T17:34:40.744Z21/30 Write block 90 len 16384 data: 7 5 8 7
166182024-02-01T17:34:40.747Z22/30 Read block 26 len 8192
166192024-02-01T17:34:40.747Z23/30 Write block 567 len 32768 data: 8 8 9 11 8 9 9 8
166202024-02-01T17:34:40.749Z24/30 Write block 540 len 36864 data: 13 17 19 14 16 13 14 15 10
166212024-02-01T17:34:40.749Z25/30 Read block 385 len 32768
166222024-02-01T17:34:40.752Z26/30 Write block 537 len 32768 data: 14 11 9 14 18 20 15 17
166232024-02-01T17:34:40.754Z27/30 Write block 108 len 32768 data: 16 13 10 9 8 10 10 10
166242024-02-01T17:34:40.755Z28/30 Read block 222 len 8192
166252024-02-01T17:34:40.757Z29/30 Write block 76 len 28672 data: 10 8 8 7 7 5 6
166262024-02-01T17:34:40.757Z30/30 Write block 410 len 40960 data: 8 7 8 8 11 13 16 10 9 12
166272024-02-01T17:34:40.761Z----------------------------------------------------------------
166282024-02-01T17:34:40.761Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
166292024-02-01T17:34:40.761ZStates: Active Active Active
166302024-02-01T17:34:40.761ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166312024-02-01T17:34:40.761Z 1 Acked 1000 Write 3 Done Done Done false
166322024-02-01T17:34:40.761Z 2 Acked 1001 Read 10 Done Done Done false
166332024-02-01T17:34:40.761Z 3 Acked 1002 Read 10 Done Done Sent false
166342024-02-01T17:34:40.761Z 4 Acked 1003 Read 7 Done Done Sent false
166352024-02-01T17:34:40.761Z 5 Acked 1004 Write 2 Done Done Sent false
166362024-02-01T17:34:40.761Z 6 Acked 1005 Write 9 Done Done Sent false
166372024-02-01T17:34:40.761Z 7 Acked 1006 Write 8 Done Done Sent false
166382024-02-01T17:34:40.761Z 8 Acked 1007 Write 8 Done Done Sent false
166392024-02-01T17:34:40.761Z 9 Acked 1008 Write 8 Done Done Sent false
166402024-02-01T17:34:40.761Z 10 Acked 1009 Write 6 Done Done Sent false
166412024-02-01T17:34:40.761Z 11 Acked 1010 Read 1 Done Done Sent false
166422024-02-01T17:34:40.761Z 12 Acked 1011 Write 3 Done Done Sent false
166432024-02-01T17:34:40.761Z 13 Acked 1012 Read 4 Done Done Sent false
166442024-02-01T17:34:40.761Z 14 Acked 1013 Flush 0 Done Done Sent false
166452024-02-01T17:34:40.761Z 15 Acked 1014 Write 7 Done Done Sent false
166462024-02-01T17:34:40.762Z 16 Acked 1015 Read 3 Done Done Sent false
166472024-02-01T17:34:40.762Z 17 Acked 1016 Read 3 Done Done Sent false
166482024-02-01T17:34:40.762Z 18 Acked 1017 Read 7 Done Done Sent false
166492024-02-01T17:34:40.762Z 19 Acked 1018 Read 10 Done Done Sent false
166502024-02-01T17:34:40.762Z 20 Acked 1019 Read 7 Done Done Sent false
166512024-02-01T17:34:40.762Z 21 Acked 1020 Write 4 Done Done Sent false
166522024-02-01T17:34:40.762Z 22 Acked 1021 Read 2 Done Done Sent false
166532024-02-01T17:34:40.762Z 23 Acked 1022 Write 8 Done Done Sent false
166542024-02-01T17:34:40.762Z 24 Acked 1023 Write 9 Done Done Sent false
166552024-02-01T17:34:40.762Z 25 Acked 1024 Read 8 Done Done Sent false
166562024-02-01T17:34:40.762Z 26 Acked 1025 Write 8 Done Done Sent false
166572024-02-01T17:34:40.762Z 27 Acked 1026 Write 8 Done Done Sent false
166582024-02-01T17:34:40.762Z 28 Acked 1027 Read 2 Done Done Sent false
166592024-02-01T17:34:40.762Z 29 Acked 1028 Write 7 Done Done Sent false
166602024-02-01T17:34:40.762Z 30 Acked 1029 Write 10 Sent Sent Sent false
166612024-02-01T17:34:40.762Z STATES DS:0 DS:1 DS:2 TOTAL
166622024-02-01T17:34:40.762Z New 0 0 0 0
166632024-02-01T17:34:40.762Z Sent 1 1 28 30
166642024-02-01T17:34:40.762Z Done 29 29 2 60
166652024-02-01T17:34:40.762Z Skipped 0 0 0 0
166662024-02-01T17:34:40.762Z Error 0 0 0 0
166672024-02-01T17:34:40.762ZLast Flush: 1013 1013 0
166682024-02-01T17:34:40.762ZDownstairs last five completed:
166692024-02-01T17:34:40.762ZUpstairs last five completed: 30 29 28 27 26
166702024-02-01T17:34:40.763Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:40.040553467Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1748}
166712024-02-01T17:34:40.763ZWrote out file "/tmp/test_repair_verify.data"
166722024-02-01T17:34:40.766Z
166732024-02-01T17:34:40.772ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
166742024-02-01T17:34:40.800ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
166752024-02-01T17:34:40.800Z 0 000-019 147 147 147 234 234 234 F F F
166762024-02-01T17:34:40.800Z 1 020-039 154 154 153 247 247 246 F F T <---
166772024-02-01T17:34:40.800Z 2 040-059 152 152 152 243 243 243 F F F
166782024-02-01T17:34:40.800Z 3 060-079 151 151 151 240 240 240 T T F <---
166792024-02-01T17:34:40.800Z 4 080-099 147 147 147 234 234 234 T T F <---
166802024-02-01T17:34:40.800Z 5 100-119 147 147 147 234 234 234 T T F <---
166812024-02-01T17:34:40.800Z 6 120-139 150 150 150 239 239 239 F F F
166822024-02-01T17:34:40.800Z 7 140-159 154 154 148 247 247 235 F F F <---
166832024-02-01T17:34:40.800Z 8 160-179 154 154 148 247 247 235 F F F <---
166842024-02-01T17:34:40.800Z 9 180-199 154 154 147 247 247 234 F F F <---
166852024-02-01T17:34:40.800Z 10 200-219 154 154 152 247 247 241 F F F <---
166862024-02-01T17:34:40.800Z 11 220-239 149 149 149 236 236 236 F F F
166872024-02-01T17:34:40.800Z 12 240-259 152 152 152 244 244 244 F F F
166882024-02-01T17:34:40.800Z 13 260-279 151 151 151 240 240 240 F F F
166892024-02-01T17:34:40.800Z 14 280-299 151 151 151 240 240 240 F F F
166902024-02-01T17:34:40.800Z 15 300-319 152 152 152 244 244 244 F F F
166912024-02-01T17:34:40.802Z 16 320-339 147 147 147 234 234 234 F F F
166922024-02-01T17:34:40.802Z 17 340-359 152 152 152 244 244 244 F F F
166932024-02-01T17:34:40.802Z 18 360-379 152 152 152 244 244 244 F F F
166942024-02-01T17:34:40.805Z 19 380-399 151 151 151 240 240 240 F F F
166952024-02-01T17:34:40.805Z 20 400-419 152 152 152 245 245 245 T T F <---
166962024-02-01T17:34:40.805Z 21 420-439 145 145 145 232 232 232 F F F
166972024-02-01T17:34:40.805Z 22 440-459 154 154 151 247 247 240 F F F <---
166982024-02-01T17:34:40.805Z 23 460-479 148 148 148 235 235 235 T T F <---
166992024-02-01T17:34:40.805Z 24 480-499 152 152 152 244 244 244 F F F
167002024-02-01T17:34:40.805Z 25 500-519 154 154 152 247 247 244 F F F <---
167012024-02-01T17:34:40.805Z 26 520-539 150 150 150 239 239 239 T T F <---
167022024-02-01T17:34:40.805Z 27 540-559 152 152 152 244 244 244 T T F <---
167032024-02-01T17:34:40.806Z 28 560-579 152 152 152 245 245 245 T T F <---
167042024-02-01T17:34:40.806Z 29 580-599 152 152 152 244 244 244 F F F
167052024-02-01T17:34:40.806ZMax gen: 154, Max flush: 247
167062024-02-01T17:34:40.806ZError: Difference in extent metadata found!
167072024-02-01T17:34:40.806ZA Difference in extent metadata is expected here
167082024-02-01T17:34:40.806ZOn loop 77
167092024-02-01T17:34:40.806Z
167102024-02-01T17:34:40.809ZVerifying data now
167112024-02-01T17:34:46.084ZNo change after verify
167122024-02-01T17:34:46.084ZLoop: 77 Downstairs dump after verify (and repair):
167132024-02-01T17:34:46.106ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167142024-02-01T17:34:46.106Z 0 000-019 147 147 147 234 234 234 F F F
167152024-02-01T17:34:46.110Z 1 020-039 154 154 154 247 247 247 F F F
167162024-02-01T17:34:46.111Z 2 040-059 152 152 152 243 243 243 F F F
167172024-02-01T17:34:46.111Z 3 060-079 155 155 155 248 248 248 F F F
167182024-02-01T17:34:46.111Z 4 080-099 155 155 155 248 248 248 F F F
167192024-02-01T17:34:46.111Z 5 100-119 155 155 155 248 248 248 F F F
167202024-02-01T17:34:46.111Z 6 120-139 150 150 150 239 239 239 F F F
167212024-02-01T17:34:46.111Z 7 140-159 154 154 154 247 247 247 F F F
167222024-02-01T17:34:46.112Z 8 160-179 154 154 154 247 247 247 F F F
167232024-02-01T17:34:46.112Z 9 180-199 154 154 154 247 247 247 F F F
167242024-02-01T17:34:46.112Z 10 200-219 154 154 154 247 247 247 F F F
167252024-02-01T17:34:46.113Z 11 220-239 149 149 149 236 236 236 F F F
167262024-02-01T17:34:46.113Z 12 240-259 152 152 152 244 244 244 F F F
167272024-02-01T17:34:46.113Z 13 260-279 151 151 151 240 240 240 F F F
167282024-02-01T17:34:46.113Z 14 280-299 151 151 151 240 240 240 F F F
167292024-02-01T17:34:46.114Z 15 300-319 152 152 152 244 244 244 F F F
167302024-02-01T17:34:46.114Z 16 320-339 147 147 147 234 234 234 F F F
167312024-02-01T17:34:46.114Z 17 340-359 152 152 152 244 244 244 F F F
167322024-02-01T17:34:46.114Z 18 360-379 152 152 152 244 244 244 F F F
167332024-02-01T17:34:46.114Z 19 380-399 151 151 151 240 240 240 F F F
167342024-02-01T17:34:46.114Z 20 400-419 155 155 155 248 248 248 F F F
167352024-02-01T17:34:46.114Z 21 420-439 145 145 145 232 232 232 F F F
167362024-02-01T17:34:46.114Z 22 440-459 154 154 154 247 247 247 F F F
167372024-02-01T17:34:46.114Z 23 460-479 155 155 155 248 248 248 F F F
167382024-02-01T17:34:46.114Z 24 480-499 152 152 152 244 244 244 F F F
167392024-02-01T17:34:46.115Z 25 500-519 154 154 154 247 247 247 F F F
167402024-02-01T17:34:46.115Z 26 520-539 155 155 155 248 248 248 F F F
167412024-02-01T17:34:46.115Z 27 540-559 155 155 155 248 248 248 F F F
167422024-02-01T17:34:46.115Z 28 560-579 155 155 155 248 248 248 F F F
167432024-02-01T17:34:46.115Z 29 580-599 152 152 152 244 244 244 F F F
167442024-02-01T17:34:46.115ZMax gen: 155, Max flush: 248
167452024-02-01T17:34:46.119Z
167462024-02-01T17:34:46.120ZBegin loop 78
167472024-02-01T17:34:46.120ZDownstairs to restart: 1
167482024-02-01T17:34:46.142Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.423335317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
167492024-02-01T17:34:46.143Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.423824664Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
167502024-02-01T17:34:46.143Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42388159Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
167512024-02-01T17:34:46.159Z{"msg":"Crucible 997bd06b-f223-4364-8c7d-d664565fc308 has session id: 46471745-4032-4373-b8f8-72eac5af1d44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42395572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167522024-02-01T17:34:46.159Z{"msg":"Upstairs opts: Upstairs UUID: 997bd06b-f223-4364-8c7d-d664565fc308, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424064723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167532024-02-01T17:34:46.159Z{"msg":"Crucible stats registered with UUID: 997bd06b-f223-4364-8c7d-d664565fc308","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424132509Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167542024-02-01T17:34:46.159ZCrucible runtime is spawned
167552024-02-01T17:34:46.159Z{"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424333207Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
167562024-02-01T17:34:46.159Z{"msg":"997bd06b-f223-4364-8c7d-d664565fc308 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424387545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167572024-02-01T17:34:46.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424412357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167582024-02-01T17:34:46.159Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424574995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167592024-02-01T17:34:46.159Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424633568Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167602024-02-01T17:34:46.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424679124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167612024-02-01T17:34:46.160Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.4247698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167622024-02-01T17:34:46.160Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424817823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167632024-02-01T17:34:46.161Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424860771Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167642024-02-01T17:34:46.161Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.424935301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167652024-02-01T17:34:46.161Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.4252419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167662024-02-01T17:34:46.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.425328682Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167672024-02-01T17:34:46.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.425432672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167682024-02-01T17:34:46.161Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.425537747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"io task","client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167692024-02-01T17:34:46.161Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.426526077Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167702024-02-01T17:34:46.162Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.426606857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167712024-02-01T17:34:46.162Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.426810407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167722024-02-01T17:34:46.162Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.426978344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167732024-02-01T17:34:46.162Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.427110681Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167742024-02-01T17:34:46.163Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.427139526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167752024-02-01T17:34:46.163Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42759359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167762024-02-01T17:34:46.163Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.427711708Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167772024-02-01T17:34:46.163Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.427740264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167782024-02-01T17:34:46.163Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428400067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167792024-02-01T17:34:46.169Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42850475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167802024-02-01T17:34:46.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428590949Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167812024-02-01T17:34:46.170Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42863581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167822024-02-01T17:34:46.170Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428699981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167832024-02-01T17:34:46.170Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428727997Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167842024-02-01T17:34:46.172Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428768895Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167852024-02-01T17:34:46.172Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428793732Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167862024-02-01T17:34:46.172Z{"msg":"[0]R flush_numbers[0..12]: [234, 247, 243, 248, 248, 248, 239, 247, 247, 247, 247, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428835761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167872024-02-01T17:34:46.172Z{"msg":"[0]R generation[0..12]: [147, 154, 152, 155, 155, 155, 150, 154, 154, 154, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428861255Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167882024-02-01T17:34:46.172Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428903693Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167892024-02-01T17:34:46.172Z{"msg":"[1]R flush_numbers[0..12]: [234, 247, 243, 248, 248, 248, 239, 247, 247, 247, 247, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.428929141Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167902024-02-01T17:34:46.172Z{"msg":"[1]R generation[0..12]: [147, 154, 152, 155, 155, 155, 150, 154, 154, 154, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42897072Z","hostname":"Wait for a query_work_queue command to finish before sending IO
167912024-02-01T17:34:46.172Zip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167922024-02-01T17:34:46.172Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429002318Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167932024-02-01T17:34:46.172Z{"msg":"[2]R flush_numbers[0..12]: [234, 247, 243, 248, 248, 248, 239, 247, 247, 247, 247, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.4290456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167942024-02-01T17:34:46.172Z{"msg":"[2]R generation[0..12]: [147, 154, 152, 155, 155, 155, 150, 154, 154, 154, 154, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42907091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167952024-02-01T17:34:46.172Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429113065Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167962024-02-01T17:34:46.173Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429136842Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167972024-02-01T17:34:46.173Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429177266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167982024-02-01T17:34:46.173Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429200099Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
167992024-02-01T17:34:46.173Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.42924647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"":"downstairs","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168002024-02-01T17:34:46.173Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429270343Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168012024-02-01T17:34:46.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429290388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"0","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168022024-02-01T17:34:46.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429338796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"1","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168032024-02-01T17:34:46.174Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429362368Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"client":"2","":"downstairs","session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168042024-02-01T17:34:46.174Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429384518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168052024-02-01T17:34:46.174Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429423009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168062024-02-01T17:34:46.174Z{"msg":"997bd06b-f223-4364-8c7d-d664565fc308 is now active with session: 46471745-4032-4373-b8f8-72eac5af1d44","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429446081Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168072024-02-01T17:34:46.174Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429467053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756,"session_id":"46471745-4032-4373-b8f8-72eac5af1d44"}
168082024-02-01T17:34:46.174Z{"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:45.429506263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
168092024-02-01T17:34:46.174ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
168102024-02-01T17:34:46.174ZLoading write count information from file "/tmp/test_repair_verify.data"
168112024-02-01T17:34:46.174ZRun Repair workload
168122024-02-01T17:34:46.174Z01/30 Flush
168132024-02-01T17:34:46.174Z02/30 Write block 460 len 32768 data: 15 15 13 12 12 11 13 13
168142024-02-01T17:34:46.174Z03/30 Flush
168152024-02-01T17:34:46.174Z04/30 Write block 12 len 40960 data: 8 7 7 8 10 13 15 12 13 12
168162024-02-01T17:34:46.174Z05/30 Write block 364 len 4096 data: 14
168172024-02-01T17:34:46.174Z06/30 Read block 210 len 28672
168182024-02-01T17:34:46.174Z07/30 Write block 199 len 8192 data: 9 11
168192024-02-01T17:34:46.174Z08/30 Flush
168202024-02-01T17:34:46.174Z09/30 Write block 185 len 12288 data: 17 14 14
168212024-02-01T17:34:46.174Z10/30 Write block 245 len 20480 data: 12 12 11 12 9
168222024-02-01T17:34:46.174Z11/30 Read block 326 len 32768
168232024-02-01T17:34:46.174Z12/30 Read block 529 len 24576
168242024-02-01T17:34:46.179Z13/30 Read block 390 len 16384
168252024-02-01T17:34:46.179Z14/30 Flush
168262024-02-01T17:34:46.183Z15/30 Write block 172 len 36864 data: 19 17 13 14 18 16 15 15 13
168272024-02-01T17:34:46.183Z16/30 Read block 72 len 4096
168282024-02-01T17:34:46.183Z17/30 Write block 337 len 40960 data: 11 9 9 7 10 11 12 11 11 11
168292024-02-01T17:34:46.183Z18/30 Write block 158 len 20480 data: 12 12 13 14 14
168302024-02-01T17:34:46.183Z19/30 Read block 519 len 4096
168312024-02-01T17:34:46.183Z20/30 Read block 429 len 24576
168322024-02-01T17:34:46.184Z21/30 Write block 588 len 20480 data: 12 11 11 10 7
168332024-02-01T17:34:46.187Z22/30 Write block 342 len 16384 data: 12 13 12 12
168342024-02-01T17:34:46.187Z23/30 Write block 174 len 36864 data: 14 15 19 17 16 16 14 15 17
168352024-02-01T17:34:46.188Z24/30 Flush
168362024-02-01T17:34:46.191Z25/30 Write block 488 len 12288 data: 16 16 17
168372024-02-01T17:34:46.191Z26/30 Write block 489 len 24576 data: 17 18 16 18 14 12
168382024-02-01T17:34:46.196Z27/30 Read block 111 len 4096
168392024-02-01T17:34:46.196Z28/30 Read block 518 len 4096
168402024-02-01T17:34:46.196Z29/30 Read block 496 len 28672
168412024-02-01T17:34:46.200Z30/30 Write block 334 len 32768 data: 16 15 15 12 10 10 8 11
168422024-02-01T17:34:46.200Z----------------------------------------------------------------
168432024-02-01T17:34:46.200Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:29
168442024-02-01T17:34:46.200ZStates: Active Active Active
168452024-02-01T17:34:46.200ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
168462024-02-01T17:34:46.200Z 2 Acked 1001 Write 8 Done Sent Done false
168472024-02-01T17:34:46.200Z 3 Acked 1002 Flush 0 Done Sent Done false
168482024-02-01T17:34:46.201Z 4 Acked 1003 Write 10 Done Sent Done false
168492024-02-01T17:34:46.201Z 5 Acked 1004 Write 1 Done Sent Done false
168502024-02-01T17:34:46.201Z 6 Acked 1005 Read 7 Done Sent Done false
168512024-02-01T17:34:46.201Z 7 Acked 1006 Write 2 Done Sent Done false
168522024-02-01T17:34:46.201Z 8 Acked 1007 Flush 0 Done Sent Done false
168532024-02-01T17:34:46.201Z 9 Acked 1008 Write 3 Done Sent Done false
168542024-02-01T17:34:46.201Z 10 Acked 1009 Write 5 Done Sent Done false
168552024-02-01T17:34:46.201Z 11 Acked 1010 Read 8 Done Sent Done false
168562024-02-01T17:34:46.201Z 12 Acked 1011 Read 6 Done Sent Done false
168572024-02-01T17:34:46.201Z 13 Acked 1012 Read 4 Done Sent Done false
168582024-02-01T17:34:46.202Z 14 Acked 1013 Flush 0 Done Sent Done false
168592024-02-01T17:34:46.202Z 15 Acked 1014 Write 9 Done Sent Done false
168602024-02-01T17:34:46.202Z 16 Acked 1015 Read 1 Done Sent Done false
168612024-02-01T17:34:46.202Z 17 Acked 1016 Write 10 Done Sent Done false
168622024-02-01T17:34:46.202Z 18 Acked 1017 Write 5 Done Sent Done false
168632024-02-01T17:34:46.202Z 19 Acked 1018 Read 1 Done Sent Done false
168642024-02-01T17:34:46.202Z 20 Acked 1019 Read 6 Done Sent Done false
168652024-02-01T17:34:46.202Z 21 Acked 1020 Write 5 Done Sent Done false
168662024-02-01T17:34:46.203Z 22 Acked 1021 Write 4 Done Sent Done false
168672024-02-01T17:34:46.203Z 23 Acked 1022 Write 9 Done Sent Done false
168682024-02-01T17:34:46.203Z 24 Acked 1023 Flush 0 Done Sent Done false
168692024-02-01T17:34:46.203Z 25 Acked 1024 Write 3 Done Sent Done false
168702024-02-01T17:34:46.203Z 26 Acked 1025 Write 6 Done Sent Done false
168712024-02-01T17:34:46.203Z 27 Acked 1026 Read 1 Done Sent Done false
168722024-02-01T17:34:46.203Z 28 Acked 1027 Read 1 Done Sent Done false
168732024-02-01T17:34:46.203Z 29 Acked 1028 Read 7 Done Sent Done false
168742024-02-01T17:34:46.203Z 30 Acked 1029 Write 8 Sent Sent Sent false
168752024-02-01T17:34:46.203Z STATES DS:0 DS:1 DS:2 TOTAL
168762024-02-01T17:34:46.204Z New 0 0 0 0
168772024-02-01T17:34:46.205Z Sent 1 29 1 31
168782024-02-01T17:34:46.205Z Done 28 0 28 56
168792024-02-01T17:34:46.205Z Skipped 0 0 0 0
168802024-02-01T17:34:46.205Z Error 0 0 0 0
168812024-02-01T17:34:46.205ZLast Flush: 1023 1000 1023
168822024-02-01T17:34:46.205ZDownstairs last five completed: 1000
168832024-02-01T17:34:46.205ZUpstairs last five completed: 30 29 28 27 26
168842024-02-01T17:34:46.205Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:45.479228512Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1756}
168852024-02-01T17:34:46.205ZWrote out file "/tmp/test_repair_verify.data"
168862024-02-01T17:34:46.210Z
168872024-02-01T17:34:46.210ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
168882024-02-01T17:34:46.242ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168892024-02-01T17:34:46.242Z 0 000-019 156 147 156 251 234 251 F F F <---
168902024-02-01T17:34:46.257Z 1 020-039 156 154 156 251 247 251 F F F <---
168912024-02-01T17:34:46.257Z 2 040-059 152 152 152 243 243 243 F F F
168922024-02-01T17:34:46.257Z 3 060-079 155 155 155 248 248 248 F F F
168932024-02-01T17:34:46.257Z 4 080-099 155 155 155 248 248 248 F F F
168942024-02-01T17:34:46.258Z 5 100-119 155 155 155 248 248 248 F F F
168952024-02-01T17:34:46.258Z 6 120-139 150 150 150 239 239 239 F F F
168962024-02-01T17:34:46.258Z 7 140-159 156 154 156 253 247 253 F F F <---
168972024-02-01T17:34:46.258Z 8 160-179 156 154 156 253 247 253 F F F <---
168982024-02-01T17:34:46.258Z 9 180-199 156 154 156 253 247 253 F F F <---
168992024-02-01T17:34:46.258Z 10 200-219 156 154 156 251 247 251 F F F <---
169002024-02-01T17:34:46.259Z 11 220-239 149 149 149 236 236 236 F F F
169012024-02-01T17:34:46.259Z 12 240-259 156 152 156 252 244 252 F F F <---
169022024-02-01T17:34:46.259Z 13 260-279 151 151 151 240 240 240 F F F
169032024-02-01T17:34:46.259Z 14 280-299 151 151 151 240 240 240 F F F
169042024-02-01T17:34:46.259Z 15 300-319 152 152 152 244 244 244 F F F
169052024-02-01T17:34:46.259Z 16 320-339 156 147 156 253 234 253 T F T <---
169062024-02-01T17:34:46.259Z 17 340-359 156 152 156 253 244 253 T F T <---
169072024-02-01T17:34:46.260Z 18 360-379 156 152 156 251 244 251 F F F <---
169082024-02-01T17:34:46.262Z 19 380-399 151 151 151 240 240 240 F F F
169092024-02-01T17:34:46.262Z 20 400-419 155 155 155 248 248 248 F F F
169102024-02-01T17:34:46.262Z 21 420-439 145 145 145 232 232 232 F F F
169112024-02-01T17:34:46.262Z 22 440-459 154 154 154 247 247 247 F F F
169122024-02-01T17:34:46.262Z 23 460-479 156 155 156 250 248 250 F F F <---
169132024-02-01T17:34:46.262Z 24 480-499 152 152 152 244 244 244 T F T <---
169142024-02-01T17:34:46.262Z 25 500-519 154 154 154 247 247 247 F F F
169152024-02-01T17:34:46.263Z 26 520-539 155 155 155 248 248 248 F F F
169162024-02-01T17:34:46.263Z 27 540-559 155 155 155 248 248 248 F F F
169172024-02-01T17:34:46.263Z 28 560-579 155 155 155 248 248 248 F F F
169182024-02-01T17:34:46.263Z 29 580-599 156 152 156 253 244 253 F F F <---
169192024-02-01T17:34:46.263ZMax gen: 156, Max flush: 253
169202024-02-01T17:34:46.263ZError: Difference in extent metadata found!
169212024-02-01T17:34:46.263ZA Difference in extent metadata is expected here
169222024-02-01T17:34:46.264ZOn loop 78
169232024-02-01T17:34:46.264Z
169242024-02-01T17:34:46.264ZVerifying data now
169252024-02-01T17:34:51.589ZNo change after verify
169262024-02-01T17:34:51.589ZLoop: 78 Downstairs dump after verify (and repair):
169272024-02-01T17:34:51.589ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169282024-02-01T17:34:51.606Z 0 000-019 156 156 156 251 251 251 F F F
169292024-02-01T17:34:51.606Z 1 020-039 156 156 156 251 251 251 F F F
169302024-02-01T17:34:51.607Z 2 040-059 152 152 152 243 243 243 F F F
169312024-02-01T17:34:51.607Z 3 060-079 155 155 155 248 248 248 F F F
169322024-02-01T17:34:51.607Z 4 080-099 155 155 155 248 248 248 F F F
169332024-02-01T17:34:51.607Z 5 100-119 155 155 155 248 248 248 F F F
169342024-02-01T17:34:51.607Z 6 120-139 150 150 150 239 239 239 F F F
169352024-02-01T17:34:51.607Z 7 140-159 156 156 156 253 253 253 F F F
169362024-02-01T17:34:51.607Z 8 160-179 156 156 156 253 253 253 F F F
169372024-02-01T17:34:51.608Z 9 180-199 156 156 156 253 253 253 F F F
169382024-02-01T17:34:51.608Z 10 200-219 156 156 156 251 251 251 F F F
169392024-02-01T17:34:51.608Z 11 220-239 149 149 149 236 236 236 F F F
169402024-02-01T17:34:51.608Z 12 240-259 156 156 156 252 252 252 F F F
169412024-02-01T17:34:51.608Z 13 260-279 151 151 151 240 240 240 F F F
169422024-02-01T17:34:51.608Z 14 280-299 151 151 151 240 240 240 F F F
169432024-02-01T17:34:51.608Z 15 300-319 152 152 152 244 244 244 F F F
169442024-02-01T17:34:51.608Z 16 320-339 157 157 157 254 254 254 F F F
169452024-02-01T17:34:51.608Z 17 340-359 157 157 157 254 254 254 F F F
169462024-02-01T17:34:51.608Z 18 360-379 156 156 156 251 251 251 F F F
169472024-02-01T17:34:51.608Z 19 380-399 151 151 151 240 240 240 F F F
169482024-02-01T17:34:51.608Z 20 400-419 155 155 155 248 248 248 F F F
169492024-02-01T17:34:51.608Z 21 420-439 145 145 145 232 232 232 F F F
169502024-02-01T17:34:51.608Z 22 440-459 154 154 154 247 247 247 F F F
169512024-02-01T17:34:51.608Z 23 460-479 156 156 156 250 250 250 F F F
169522024-02-01T17:34:51.608Z 24 480-499 157 157 157 254 254 254 F F F
169532024-02-01T17:34:51.608Z 25 500-519 154 154 154 247 247 247 F F F
169542024-02-01T17:34:51.609Z 26 520-539 155 155 155 248 248 248 F F F
169552024-02-01T17:34:51.609Z 27 540-559 155 155 155 248 248 248 F F F
169562024-02-01T17:34:51.609Z 28 560-579 155 155 155 248 248 248 F F F
169572024-02-01T17:34:51.609Z 29 580-599 156 156 156 253 253 253 F F F
169582024-02-01T17:34:51.609ZMax gen: 157, Max flush: 254
169592024-02-01T17:34:51.609Z
169602024-02-01T17:34:51.609ZBegin loop 79
169612024-02-01T17:34:51.609ZDownstairs to restart: 1
169622024-02-01T17:34:51.609Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.67403418Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
169632024-02-01T17:34:51.610Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674487497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
169642024-02-01T17:34:51.610Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674541683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
169652024-02-01T17:34:51.610Z{"msg":"Crucible 4b879391-a451-4196-b372-44e0b8497ddf has session id: 73b3f044-eb99-4dfd-85c2-32a85b4b8236","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674599549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169662024-02-01T17:34:51.610Z{Crucible runtime is spawned
169672024-02-01T17:34:51.610Z"msg":"Upstairs opts: Upstairs UUID: 4b879391-a451-4196-b372-44e0b8497ddf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674716766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169682024-02-01T17:34:51.610Z{"msg":"Crucible stats registered with UUID: 4b879391-a451-4196-b372-44e0b8497ddf","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674832832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169692024-02-01T17:34:51.611Z{"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.674888887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
169702024-02-01T17:34:51.612Z{"msg":"4b879391-a451-4196-b372-44e0b8497ddf active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675053998Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169712024-02-01T17:34:51.612Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675106684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169722024-02-01T17:34:51.612Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.67517211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169732024-02-01T17:34:51.612Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675241976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169742024-02-01T17:34:51.612Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675318062Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169752024-02-01T17:34:51.612Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.67539Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169762024-02-01T17:34:51.612Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675435573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169772024-02-01T17:34:51.613Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675506768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169782024-02-01T17:34:51.613Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675580108Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169792024-02-01T17:34:51.613Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675653587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169802024-02-01T17:34:51.613Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675768596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169812024-02-01T17:34:51.613Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675845462Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169822024-02-01T17:34:51.613Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.675891196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"io task","client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169832024-02-01T17:34:51.613Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.676981441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169842024-02-01T17:34:51.613Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.677047439Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169852024-02-01T17:34:51.613Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.677138922Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169862024-02-01T17:34:51.613Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.677197381Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169872024-02-01T17:34:51.613Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.677782453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169882024-02-01T17:34:51.613Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.677860248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169892024-02-01T17:34:51.613Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.678172121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169902024-02-01T17:34:51.613Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.678482124Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169912024-02-01T17:34:51.613Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.678852615Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169922024-02-01T17:34:51.614Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.678921648Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169932024-02-01T17:34:51.614Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.678982849Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169942024-02-01T17:34:51.614Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679036233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169952024-02-01T17:34:51.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679181473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169962024-02-01T17:34:51.614Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679266718Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169972024-02-01T17:34:51.614Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679341338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169982024-02-01T17:34:51.614Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679836714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
169992024-02-01T17:34:51.615Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.67992426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170002024-02-01T17:34:51.615Z{"msg":"[0]R flush_numbers[0..12]: [251, 251, 243, 248, 248, 248, 239, 253, 253, 253, 251, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.679999979Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170012024-02-01T17:34:51.615Z{"msg":"[0]R generation[0..12]: [156, 156, 152, 155, 155, 155, 150, 156, 156, 156, 156, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680074201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170022024-02-01T17:34:51.615Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680146915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170032024-02-01T17:34:51.615Z{"msg":"[1]R flush_numbers[0..12]: [251, 251, 243, 248, 248, 248, 239, 253, 253, 253, 251, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680219334Z",Wait for a query_work_queue command to finish before sending IO
170042024-02-01T17:34:51.615Z"hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170052024-02-01T17:34:51.615Z{"msg":"[1]R generation[0..12]: [156, 156, 152, 155, 155, 155, 150, 156, 156, 156, 156, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680355642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170062024-02-01T17:34:51.615Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680437371Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170072024-02-01T17:34:51.615Z{"msg":"[2]R flush_numbers[0..12]: [251, 251, 243, 248, 248, 248, 239, 253, 253, 253, 251, 236]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680490315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170082024-02-01T17:34:51.615Z{"msg":"[2]R generation[0..12]: [156, 156, 152, 155, 155, 155, 150, 156, 156, 156, 156, 149]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680539143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170092024-02-01T17:34:51.617Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680582744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170102024-02-01T17:34:51.617Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680624601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170112024-02-01T17:34:51.617Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680650058Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170122024-02-01T17:34:51.617Z{"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680692409Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170132024-02-01T17:34:51.617Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680734192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"":"downstairs","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170142024-02-01T17:34:51.617Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.68077676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170152024-02-01T17:34:51.617Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680801183Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"0","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170162024-02-01T17:34:51.617Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680841215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"1","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170172024-02-01T17:34:51.617Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680866537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"client":"2","":"downstairs","session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170182024-02-01T17:34:51.617Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.680912868Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170192024-02-01T17:34:51.617Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
170202024-02-01T17:34:51.617Z,"time":"2024-02-01T17:34:50.680980112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170212024-02-01T17:34:51.617Z{"msg":"4b879391-a451-4196-b372-44e0b8497ddf is now active with session: 73b3f044-eb99-4dfd-85c2-32a85b4b8236","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.681016646Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170222024-02-01T17:34:51.617Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.681047885Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764,"session_id":"73b3f044-eb99-4dfd-85c2-32a85b4b8236"}
170232024-02-01T17:34:51.618Z{"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:50.68109094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
170242024-02-01T17:34:51.618ZLoading write count information from file "/tmp/test_repair_verify.data"
170252024-02-01T17:34:51.618ZRun Repair workload
170262024-02-01T17:34:51.618ZError: Difference in extent metadata found!
170272024-02-01T17:34:51.618Z01/30 Write block 507 len 32768 data: 9 12 13 11 12 10 11 9
170282024-02-01T17:34:51.625Z02/30 Write block 91 len 16384 data: 6 9 8 8
170292024-02-01T17:34:51.625Z03/30 Write block 230 len 24576 data: 9 10 9 8 8 7
170302024-02-01T17:34:51.625Z04/30 Read block 325 len 24576
170312024-02-01T17:34:51.625Z05/30 Read block 500 len 40960
170322024-02-01T17:34:51.625Z06/30 Read block 260 len 4096
170332024-02-01T17:34:51.625Z07/30 Read block 493 len 20480
170342024-02-01T17:34:51.625Z08/30 Read block 209 len 12288
170352024-02-01T17:34:51.626Z09/30 Write block 51 len 20480 data: 14 12 13 15 13
170362024-02-01T17:34:51.626Z10/30 Read block 229 len 40960
170372024-02-01T17:34:51.626Z11/30 Read block 29 len 12288
170382024-02-01T17:34:51.626Z12/30 Read block 171 len 32768
170392024-02-01T17:34:51.626Z13/30 Read block 507 len 28672
170402024-02-01T17:34:51.626Z14/30 Read block 325 len 20480
170412024-02-01T17:34:51.626Z15/30 Read block 442 len 8192
170422024-02-01T17:34:51.626Z16/30 Read block 393 len 4096
170432024-02-01T17:34:51.626Z17/30 Read block 302 len 4096
170442024-02-01T17:34:51.626Z18/30 Write block 313 len 16384 data: 13 11 13 14
170452024-02-01T17:34:51.627Z19/30 Flush
170462024-02-01T17:34:51.627Z20/30 Write block 351 len 12288 data: 12 14 12
170472024-02-01T17:34:51.627Z21/30 Write block 89 len 16384 data: 7 8 7 10
170482024-02-01T17:34:51.627Z22/30 Write block 100 len 40960 data: 12 13 14 12 13 16 17 18 17 14
170492024-02-01T17:34:51.627Z23/30 Read block 305 len 36864
170502024-02-01T17:34:51.627Z24/30 Read block 177 len 4096
170512024-02-01T17:34:51.627Z25/30 Write block 580 len 8192 data: 15 14
170522024-02-01T17:34:51.627Z26/30 Write block 217 len 32768 data: 7 13 16 18 16 16 16 15
170532024-02-01T17:34:51.627Z27/30 Read block 573 len 8192
170542024-02-01T17:34:51.627Z28/30 Write block 312 len 32768 data: 11 14 12 14 15 15 14 15
170552024-02-01T17:34:51.627Z29/30 Write block 264 len 12288 data: 14 15 15
170562024-02-01T17:34:51.627Z30/30 Write block 33 len 32768 data: 8 9 11 14 15 16 14 10
170572024-02-01T17:34:51.627Z----------------------------------------------------------------
170582024-02-01T17:34:51.628Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
170592024-02-01T17:34:51.628ZStates: Active Active Active
170602024-02-01T17:34:51.628ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
170612024-02-01T17:34:51.628Z 1 Acked 1000 Write 8 Done Done Done false
170622024-02-01T17:34:51.628Z 2 Acked 1001 Write 4 Done Done Done false
170632024-02-01T17:34:51.630Z 3 Acked 1002 Write 6 Done Done Done false
170642024-02-01T17:34:51.630Z 4 Acked 1003 Read 6 Done Done Done false
170652024-02-01T17:34:51.630Z 5 Acked 1004 Read 10 Done Done Done false
170662024-02-01T17:34:51.630Z 6 Acked 1005 Read 1 Done Done Done false
170672024-02-01T17:34:51.630Z 7 Acked 1006 Read 5 Done Done Done false
170682024-02-01T17:34:51.630Z 8 Acked 1007 Read 3 Done Done Done false
170692024-02-01T17:34:51.630Z 9 Acked 1008 Write 5 Done Sent Done false
170702024-02-01T17:34:51.630Z 10 Acked 1009 Read 10 Done Sent Done false
170712024-02-01T17:34:51.630Z 11 Acked 1010 Read 3 Done Sent Done false
170722024-02-01T17:34:51.630Z 12 Acked 1011 Read 8 Done Sent Done false
170732024-02-01T17:34:51.630Z 13 Acked 1012 Read 7 Done Sent Done false
170742024-02-01T17:34:51.631Z 14 Acked 1013 Read 5 Done Sent Done false
170752024-02-01T17:34:51.631Z 15 Acked 1014 Read 2 Done Sent Done false
170762024-02-01T17:34:51.631Z 16 Acked 1015 Read 1 Done Sent Done false
170772024-02-01T17:34:51.631Z 17 Acked 1016 Read 1 Done Sent Done false
170782024-02-01T17:34:51.631Z 18 Acked 1017 Write 4 Done Sent Done false
170792024-02-01T17:34:51.631Z 19 Acked 1018 Flush 0 Done Sent Done false
170802024-02-01T17:34:51.631Z 20 Acked 1019 Write 3 Done Sent Done false
170812024-02-01T17:34:51.631Z 21 Acked 1020 Write 4 Done Sent Done false
170822024-02-01T17:34:51.631Z 22 Acked 1021 Write 10 Done Sent Done false
170832024-02-01T17:34:51.631Z 23 Acked 1022 Read 9 Done Sent Done false
170842024-02-01T17:34:51.631Z 24 Acked 1023 Read 1 Done Sent Done false
170852024-02-01T17:34:51.631Z 25 Acked 1024 Write 2 Done Sent Done false
170862024-02-01T17:34:51.631Z 26 Acked 1025 Write 8 Done Sent Done false
170872024-02-01T17:34:51.632Z 27 Acked 1026 Read 2 Done Sent Done false
170882024-02-01T17:34:51.632Z 28 Acked 1027 Write 8 Done Sent Done false
170892024-02-01T17:34:51.632Z 29 Acked 1028 Write 3 Done Sent Done false
170902024-02-01T17:34:51.632Z 30 Acked 1029 Write 8 Sent Sent Sent false
170912024-02-01T17:34:51.632Z STATES DS:0 DS:1 DS:2 TOTAL
170922024-02-01T17:34:51.632Z New 0 0 0 0
170932024-02-01T17:34:51.632Z Sent 1 22 1 24
170942024-02-01T17:34:51.632Z Done 29 8 29 66
170952024-02-01T17:34:51.632Z Skipped 0 0 0 0
170962024-02-01T17:34:51.632Z Error 0 0 0 0
170972024-02-01T17:34:51.632ZLast Flush: 1018 0 1018
170982024-02-01T17:34:51.632ZDownstairs last five completed:
170992024-02-01T17:34:51.633ZUpstairs last five completed: 30 29 28 27 26
171002024-02-01T17:34:51.633Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:50.726547632Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1764}
171012024-02-01T17:34:51.633ZWrote out file "/tmp/test_repair_verify.data"
171022024-02-01T17:34:51.633Z
171032024-02-01T17:34:51.633ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
171042024-02-01T17:34:51.633ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171052024-02-01T17:34:51.633Z 0 000-019 156 156 156 251 251 251 F F F
171062024-02-01T17:34:51.633Z 1 020-039 156 156 156 251 251 251 T F T <---
171072024-02-01T17:34:51.633Z 2 040-059 158 152 158 255 243 255 T F T <---
171082024-02-01T17:34:51.633Z 3 060-079 155 155 155 248 248 248 F F F
171092024-02-01T17:34:51.633Z 4 080-099 158 155 158 255 248 255 T T T <---
171102024-02-01T17:34:51.633Z 5 100-119 155 155 155 248 248 248 T F T <---
171112024-02-01T17:34:51.634Z 6 120-139 150 150 150 239 239 239 F F F
171122024-02-01T17:34:51.634Z 7 140-159 156 156 156 253 253 253 F F F
171132024-02-01T17:34:51.634Z 8 160-179 156 156 156 253 253 253 F F F
171142024-02-01T17:34:51.634Z 9 180-199 156 156 156 253 253 253 F F F
171152024-02-01T17:34:51.634Z 10 200-219 156 156 156 251 251 251 T F T <---
171162024-02-01T17:34:51.634Z 11 220-239 158 149 158 255 236 255 T T T <---
171172024-02-01T17:34:51.634Z 12 240-259 156 156 156 252 252 252 F F F
171182024-02-01T17:34:51.634Z 13 260-279 151 151 151 240 240 240 T F T <---
171192024-02-01T17:34:51.634Z 14 280-299 151 151 151 240 240 240 F F F
171202024-02-01T17:34:51.634Z 15 300-319 158 152 158 255 244 255 T F T <---
171212024-02-01T17:34:51.634Z 16 320-339 157 157 157 254 254 254 F F F
171222024-02-01T17:34:51.634Z 17 340-359 157 157 157 254 254 254 T F T <---
171232024-02-01T17:34:51.634Z 18 360-379 156 156 156 251 251 251 F F F
171242024-02-01T17:34:51.635Z 19 380-399 151 151 151 240 240 240 F F F
171252024-02-01T17:34:51.635Z 20 400-419 155 155 155 248 248 248 F F F
171262024-02-01T17:34:51.635Z 21 420-439 145 145 145 232 232 232 F F F
171272024-02-01T17:34:51.635Z 22 440-459 154 154 154 247 247 247 F F F
171282024-02-01T17:34:51.643Z 23 460-479 156 156 156 250 250 250 F F F
171292024-02-01T17:34:51.643Z 24 480-499 157 157 157 254 254 254 F F F
171302024-02-01T17:34:51.643Z 25 500-519 158 154 158 255 247 255 F T F <---
171312024-02-01T17:34:51.644Z 26 520-539 155 155 155 248 248 248 F F F
171322024-02-01T17:34:51.644Z 27 540-559 155 155 155 248 248 248 F F F
171332024-02-01T17:34:51.644Z 28 560-579 155 155 155 248 248 248 F F F
171342024-02-01T17:34:51.644Z 29 580-599 156 156 156 253 253 253 T F T <---
171352024-02-01T17:34:51.645ZMax gen: 158, Max flush: 255
171362024-02-01T17:34:51.645ZA Difference in extent metadata is expected here
171372024-02-01T17:34:51.645ZOn loop 79
171382024-02-01T17:34:51.645Z
171392024-02-01T17:34:51.645ZVerifying data now
171402024-02-01T17:34:57.470ZNo change after verify
171412024-02-01T17:34:57.470ZLoop: 79 Downstairs dump after verify (and repair):
171422024-02-01T17:34:57.493ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
171432024-02-01T17:34:57.493Z 0 000-019 156 156 156 251 251 251 F F F
171442024-02-01T17:34:57.493Z 1 020-039 159 159 159 256 256 256 F F F
171452024-02-01T17:34:57.493Z 2 040-059 159 159 159 256 256 256 F F F
171462024-02-01T17:34:57.493Z 3 060-079 155 155 155 248 248 248 F F F
171472024-02-01T17:34:57.497Z 4 080-099 159 159 159 256 256 256 F F F
171482024-02-01T17:34:57.498Z 5 100-119 159 159 159 256 256 256 F F F
171492024-02-01T17:34:57.498Z 6 120-139 150 150 150 239 239 239 F F F
171502024-02-01T17:34:57.498Z 7 140-159 156 156 156 253 253 253 F F F
171512024-02-01T17:34:57.498Z 8 160-179 156 156 156 253 253 253 F F F
171522024-02-01T17:34:57.498Z 9 180-199 156 156 156 253 253 253 F F F
171532024-02-01T17:34:57.499Z 10 200-219 159 159 159 256 256 256 F F F
171542024-02-01T17:34:57.499Z 11 220-239 159 159 159 256 256 256 F F F
171552024-02-01T17:34:57.499Z 12 240-259 156 156 156 252 252 252 F F F
171562024-02-01T17:34:57.503Z 13 260-279 159 159 159 256 256 256 F F F
171572024-02-01T17:34:57.503Z 14 280-299 151 151 151 240 240 240 F F F
171582024-02-01T17:34:57.503Z 15 300-319 159 159 159 256 256 256 F F F
171592024-02-01T17:34:57.503Z 16 320-339 157 157 157 254 254 254 F F F
171602024-02-01T17:34:57.503Z 17 340-359 159 159 159 256 256 256 F F F
171612024-02-01T17:34:57.503Z 18 360-379 156 156 156 251 251 251 F F F
171622024-02-01T17:34:57.504Z 19 380-399 151 151 151 240 240 240 F F F
171632024-02-01T17:34:57.504Z 20 400-419 155 155 155 248 248 248 F F F
171642024-02-01T17:34:57.504Z 21 420-439 145 145 145 232 232 232 F F F
171652024-02-01T17:34:57.504Z 22 440-459 154 154 154 247 247 247 F F F
171662024-02-01T17:34:57.504Z 23 460-479 156 156 156 250 250 250 F F F
171672024-02-01T17:34:57.505Z 24 480-499 157 157 157 254 254 254 F F F
171682024-02-01T17:34:57.505Z 25 500-519 158 158 158 255 255 255 F F F
171692024-02-01T17:34:57.505Z 26 520-539 155 155 155 248 248 248 F F F
171702024-02-01T17:34:57.505Z 27 540-559 155 155 155 248 248 248 F F F
171712024-02-01T17:34:57.505Z 28 560-579 155 155 155 248 248 248 F F F
171722024-02-01T17:34:57.505Z 29 580-599 159 159 159 256 256 256 F F F
171732024-02-01T17:34:57.505ZMax gen: 159, Max flush: 256
171742024-02-01T17:34:57.506Z
171752024-02-01T17:34:57.508ZBegin loop 80
171762024-02-01T17:34:57.508ZDownstairs to restart: 2
171772024-02-01T17:34:57.529Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.81108177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
171782024-02-01T17:34:57.529Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}",Crucible runtime is spawned
171792024-02-01T17:34:57.534Z"v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.811664859Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
171802024-02-01T17:34:57.534Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.811757517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
171812024-02-01T17:34:57.535Z{"msg":"Crucible ba071869-a028-41dc-b2da-e469eaf19b74 has session id: 129d7e1d-0607-4e85-a3de-b4bfa490a426","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.81183178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171822024-02-01T17:34:57.535Z{"msg":"Upstairs opts: Upstairs UUID: ba071869-a028-41dc-b2da-e469eaf19b74, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.811917861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171832024-02-01T17:34:57.535Z{"msg":"Crucible stats registered with UUID: ba071869-a028-41dc-b2da-e469eaf19b74","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.811999824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171842024-02-01T17:34:57.535Z{"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812046259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
171852024-02-01T17:34:57.535Z{"msg":"ba071869-a028-41dc-b2da-e469eaf19b74 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812086085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171862024-02-01T17:34:57.535Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812123724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171872024-02-01T17:34:57.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812241215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171882024-02-01T17:34:57.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812290351Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171892024-02-01T17:34:57.536Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812368088Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171902024-02-01T17:34:57.536Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812409609Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171912024-02-01T17:34:57.537Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812489473Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171922024-02-01T17:34:57.537Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812642449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171932024-02-01T17:34:57.537Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812744296Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171942024-02-01T17:34:57.537Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812817511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171952024-02-01T17:34:57.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812865427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171962024-02-01T17:34:57.537Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812910121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171972024-02-01T17:34:57.539Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.812970576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"io task","client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171982024-02-01T17:34:57.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.813956776Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
171992024-02-01T17:34:57.539Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.814024583Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172002024-02-01T17:34:57.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.814070357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172012024-02-01T17:34:57.539Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.814127978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172022024-02-01T17:34:57.539Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.814662319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172032024-02-01T17:34:57.539Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.814725838Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172042024-02-01T17:34:57.552Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.81541133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172052024-02-01T17:34:57.552Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.815476716Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172062024-02-01T17:34:57.552Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.815932493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172072024-02-01T17:34:57.553Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816437063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172082024-02-01T17:34:57.554Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816502513Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172092024-02-01T17:34:57.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816529386Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172102024-02-01T17:34:57.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816550774Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172112024-02-01T17:34:57.554Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816572932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172122024-02-01T17:34:57.554Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816595113Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172132024-02-01T17:34:57.554Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816889738Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172142024-02-01T17:34:57.554Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.816971662Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172152024-02-01T17:34:57.554Z{"msg":"[0]R flush_numbers[0..12]: [251, 256, 256, 248, 256, 256, 239, 253, 253, 253, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817020412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172162024-02-01T17:34:57.554Z{"msg":"[0]R generation[0..12]: [156, 159, 159, 155, 159, 159, 150, 156, 156, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817076737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172172024-02-01T17:34:57.554Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817122927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172182024-02-01T17:34:57.554Z{"msg":"[1]R flush_numbers[0..12]: [251, 256, 256, 248, 256, 256, 239, 253, 253, 253, 256, 256]","Wait for a query_work_queue command to finish before sending IO
172192024-02-01T17:34:57.555Zv":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817195813Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172202024-02-01T17:34:57.555Z{"msg":"[1]R generation[0..12]: [156, 159, 159, 155, 159, 159, 150, 156, 156, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817240245Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172212024-02-01T17:34:57.555Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817305224Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172222024-02-01T17:34:57.555Z{"msg":"[2]R flush_numbers[0..12]: [251, 256, 256, 248, 256, 256, 239, 253, 253, 253, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.81735111Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172232024-02-01T17:34:57.555Z{"msg":"[2]R generation[0..12]: [156, 159, 159, 155, 159, 159, 150, 156, 156, 156, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817416915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172242024-02-01T17:34:57.555Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817463032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172252024-02-01T17:34:57.555Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817522079Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172262024-02-01T17:34:57.555Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817564282Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172272024-02-01T17:34:57.559Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817628586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172282024-02-01T17:34:57.559Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817670518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"":"downstairs","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172292024-02-01T17:34:57.559Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817727463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172302024-02-01T17:34:57.560Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817766634Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172312024-02-01T17:34:57.560Z","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172322024-02-01T17:34:57.560Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817903802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"1","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172332024-02-01T17:34:57.560Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.817969887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"client":"2","":"downstairs","session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172342024-02-01T17:34:57.560Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.818014643Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172352024-02-01T17:34:57.560Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.818071244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172362024-02-01T17:34:57.560Z{"msg":"ba071869-a028-41dc-b2da-e469eaf19b74 is now active with session: 129d7e1d-0607-4e85-a3de-b4bfa490a426","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.818113082Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172372024-02-01T17:34:57.561Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.818162803Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772,"session_id":"129d7e1d-0607-4e85-a3de-b4bfa490a426"}
172382024-02-01T17:34:57.562Z{"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:34:56.818202032Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
172392024-02-01T17:34:57.562ZLoading write count information from file "/tmp/test_repair_verify.data"
172402024-02-01T17:34:57.562ZRun Repair workload
172412024-02-01T17:34:57.562Z01/30 Write block 189 len 16384 data: 11 12 12 9
172422024-02-01T17:34:57.562Z02/30 Write block 488 len 8192 data: 17 18
172432024-02-01T17:34:57.562Z03/30 Read block 576 len 4096
172442024-02-01T17:34:57.562Z04/30 Write block 144 len 40960 data: 11 11 11 11 15 15 11 10 12 12
172452024-02-01T17:34:57.563Z05/30 Write block 282 len 36864 data: 8 10 11 15 15 13 15 16 14
172462024-02-01T17:34:57.568Z06/30 Read block 495 len 36864
172472024-02-01T17:34:57.568Z07/30 Flush
172482024-02-01T17:34:57.568Z08/30 Write block 318 len 28672 data: 15 16 15 14 15 13 13
172492024-02-01T17:34:57.570Z09/30 Read block 140 len 32768
172502024-02-01T17:34:57.570Z10/30 Write block 466 len 4096 data: 14
172512024-02-01T17:34:57.570Z11/30 Read block 262 len 24576
172522024-02-01T17:34:57.570Z12/30 Read block 564 len 28672
172532024-02-01T17:34:57.570Z13/30 Write block 121 len 36864 data: 8 11 11 11 13 10 11 12 12
172542024-02-01T17:34:57.571Z14/30 Flush
172552024-02-01T17:34:57.571Z15/30 Write block 1 len 28672 data: 7 8 11 13 12 13 11
172562024-02-01T17:34:57.571Z16/30 Read block 359 len 16384
172572024-02-01T17:34:57.571Z17/30 Read block 107 len 8192
172582024-02-01T17:34:57.571Z18/30 Write block 314 len 28672 data: 13 15 16 16 16 17 16
172592024-02-01T17:34:57.571Z19/30 Flush
172602024-02-01T17:34:57.576Z20/30 Write block 469 len 24576 data: 10 12 12 12 10 8
172612024-02-01T17:34:57.577Z21/30 Write block 325 len 20480 data: 7 8 10 9 11
172622024-02-01T17:34:57.577Z22/30 Read block 226 len 28672
172632024-02-01T17:34:57.577Z23/30 Read block 66 len 28672
172642024-02-01T17:34:57.581Z24/30 Read block 170 len 32768
172652024-02-01T17:34:57.582Z25/30 Write block 332 len 40960 data: 15 15 17 16 16 13 11 11 9 12
172662024-02-01T17:34:57.582Z26/30 Write block 432 len 40960 data: 14 17 15 17 15 16 14 13 13 14
172672024-02-01T17:34:57.586Z27/30 Write block 478 len 4096 data: 8
172682024-02-01T17:34:57.586Z28/30 Write block 275 len 40960 data: 11 15 15 15 11 12 10 9 11 12
172692024-02-01T17:34:57.586Z29/30 Write block 41 len 4096 data: 13
172702024-02-01T17:34:57.586Z30/30 Write block 416 len 24576 data: 17 11 10 13 10 10
172712024-02-01T17:34:57.586Z----------------------------------------------------------------
172722024-02-01T17:34:57.586Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
172732024-02-01T17:34:57.586ZStates: Active Active Active
172742024-02-01T17:34:57.587ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172752024-02-01T17:34:57.587Z 1 Acked 1000 Write 4 Done Done Done false
172762024-02-01T17:34:57.587Z 2 Acked 1001 Write 2 Done Done Done false
172772024-02-01T17:34:57.587Z 3 Acked 1002 Read 1 Done Done Done false
172782024-02-01T17:34:57.587Z 4 Acked 1003 Write 10 Done Done Done false
172792024-02-01T17:34:57.587Z 5 Acked 1004 Write 9 Done Done Done false
172802024-02-01T17:34:57.587Z 6 Acked 1005 Read 9 Done Done Done false
172812024-02-01T17:34:57.587Z 7 Acked 1006 Flush 0 Done Done Sent false
172822024-02-01T17:34:57.587Z 8 Acked 1007 Write 7 Done Done Sent false
172832024-02-01T17:34:57.587Z 9 Acked 1008 Read 8 Done Done Sent false
172842024-02-01T17:34:57.587Z 10 Acked 1009 Write 1 Done Done Sent false
172852024-02-01T17:34:57.588Z 11 Acked 1010 Read 6 Done Done Sent false
172862024-02-01T17:34:57.588Z 12 Acked 1011 Read 7 Done Done Sent false
172872024-02-01T17:34:57.588Z 13 Acked 1012 Write 9 Done Done Sent false
172882024-02-01T17:34:57.589Z 14 Acked 1013 Flush 0 Done Done Sent false
172892024-02-01T17:34:57.589Z 15 Acked 1014 Write 7 Done Done Sent false
172902024-02-01T17:34:57.589Z 16 Acked 1015 Read 4 Done Done Sent false
172912024-02-01T17:34:57.589Z 17 Acked 1016 Read 2 Done Done Sent false
172922024-02-01T17:34:57.589Z 18 Acked 1017 Write 7 Done Done Sent false
172932024-02-01T17:34:57.589Z 19 Acked 1018 Flush 0 Done Done Sent false
172942024-02-01T17:34:57.589Z 20 Acked 1019 Write 6 Done Done Sent false
172952024-02-01T17:34:57.589Z 21 Acked 1020 Write 5 Done Done Sent false
172962024-02-01T17:34:57.589Z 22 Acked 1021 Read 7 Done Done Sent false
172972024-02-01T17:34:57.589Z 23 Acked 1022 Read 7 Done Done Sent false
172982024-02-01T17:34:57.589Z 24 Acked 1023 Read 8 Done Done Sent false
172992024-02-01T17:34:57.590Z 25 Acked 1024 Write 10 Done Done Sent false
173002024-02-01T17:34:57.590Z 26 Acked 1025 Write 10 Done Done Sent false
173012024-02-01T17:34:57.590Z 27 Acked 1026 Write 1 Done Done Sent false
173022024-02-01T17:34:57.590Z 28 Acked 1027 Write 10 Done Done Sent false
173032024-02-01T17:34:57.590Z 29 Acked 1028 Write 1 Sent Sent Sent false
173042024-02-01T17:34:57.590Z 30 Acked 1029 Write 6 Sent Sent Sent false
173052024-02-01T17:34:57.590Z STATES DS:0 DS:1 DS:2 TOTAL
173062024-02-01T17:34:57.590Z New 0 0 0 0
173072024-02-01T17:34:57.590Z Sent 2 2 24 28
173082024-02-01T17:34:57.591Z Done 28 28 6 62
173092024-02-01T17:34:57.591Z Skipped 0 0 0 0
173102024-02-01T17:34:57.591Z Error 0 0 0 0
173112024-02-01T17:34:57.591ZLast Flush: 1018 1018 0
173122024-02-01T17:34:57.592ZDownstairs last five completed:
173132024-02-01T17:34:57.592ZUpstairs last five completed: 30 29 28 27 26
173142024-02-01T17:34:57.596Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:34:56.868428254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1772}
173152024-02-01T17:34:57.596ZWrote out file "/tmp/test_repair_verify.data"
173162024-02-01T17:34:57.596Z
173172024-02-01T17:34:57.600ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
173182024-02-01T17:34:57.628ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173192024-02-01T17:34:57.628Z 0 000-019 160 160 156 259 259 251 F F F <---
173202024-02-01T17:34:57.629Z 1 020-039 159 159 159 256 256 256 F F F
173212024-02-01T17:34:57.629Z 2 040-059 159 159 159 256 256 256 T T F <---
173222024-02-01T17:34:57.629Z 3 060-079 155 155 155 248 248 248 F F F
173232024-02-01T17:34:57.629Z 4 080-099 159 159 159 256 256 256 F F F
173242024-02-01T17:34:57.633Z 5 100-119 159 159 159 256 256 256 F F F
173252024-02-01T17:34:57.633Z 6 120-139 160 160 150 258 258 239 F F F <---
173262024-02-01T17:34:57.633Z 7 140-159 160 160 156 257 257 253 F F T <---
173272024-02-01T17:34:57.633Z 8 160-179 156 156 156 253 253 253 F F F
173282024-02-01T17:34:57.633Z 9 180-199 160 160 156 257 257 253 F F T <---
173292024-02-01T17:34:57.633Z 10 200-219 159 159 159 256 256 256 F F F
173302024-02-01T17:34:57.633Z 11 220-239 159 159 159 256 256 256 F F F
173312024-02-01T17:34:57.633Z 12 240-259 156 156 156 252 252 252 F F F
173322024-02-01T17:34:57.634Z 13 260-279 159 159 159 256 256 256 T T F <---
173332024-02-01T17:34:57.634Z 14 280-299 160 160 151 257 257 240 T T T <---
173342024-02-01T17:34:57.634Z 15 300-319 160 160 159 259 259 256 F F F <---
173352024-02-01T17:34:57.634Z 16 320-339 160 160 157 259 259 254 T T F <---
173362024-02-01T17:34:57.634Z 17 340-359 159 159 159 256 256 256 T T F <---
173372024-02-01T17:34:57.634Z 18 360-379 156 156 156 251 251 251 F F F
173382024-02-01T17:34:57.634Z 19 380-399 151 151 151 240 240 240 F F F
173392024-02-01T17:34:57.634Z 20 400-419 155 155 155 248 248 248 T T F <---
173402024-02-01T17:34:57.634Z 21 420-439 145 145 145 232 232 232 T T F <---
173412024-02-01T17:34:57.634Z 22 440-459 154 154 154 247 247 247 T T F <---
173422024-02-01T17:34:57.635Z 23 460-479 160 160 156 258 258 250 T T F <---
173432024-02-01T17:34:57.635Z 24 480-499 160 160 157 257 257 254 F F T <---
173442024-02-01T17:34:57.635Z 25 500-519 158 158 158 255 255 255 F F F
173452024-02-01T17:34:57.635Z 26 520-539 155 155 155 248 248 248 F F F
173462024-02-01T17:34:57.635Z 27 540-559 155 155 155 248 248 248 F F F
173472024-02-01T17:34:57.635Z 28 560-579 155 155 155 248 248 248 F F F
173482024-02-01T17:34:57.635Z 29 580-599 159 159 159 256 256 256 F F F
173492024-02-01T17:34:57.635ZMax gen: 160, Max flush: 259
173502024-02-01T17:34:57.635ZError: Difference in extent metadata found!
173512024-02-01T17:34:57.635ZA Difference in extent metadata is expected here
173522024-02-01T17:34:57.635ZOn loop 80
173532024-02-01T17:34:57.635Z
173542024-02-01T17:34:57.638ZVerifying data now
173552024-02-01T17:35:03.897ZNo change after verify
173562024-02-01T17:35:03.897ZLoop: 80 Downstairs dump after verify (and repair):
173572024-02-01T17:35:03.921ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173582024-02-01T17:35:03.921Z 0 000-019 160 160 160 259 259 259 F F F
173592024-02-01T17:35:03.927Z 1 020-039 159 159 159 256 256 256 F F F
173602024-02-01T17:35:03.927Z 2 040-059 161 161 161 260 260 260 F F F
173612024-02-01T17:35:03.927Z 3 060-079 155 155 155 248 248 248 F F F
173622024-02-01T17:35:03.927Z 4 080-099 159 159 159 256 256 256 F F F
173632024-02-01T17:35:03.927Z 5 100-119 159 159 159 256 256 256 F F F
173642024-02-01T17:35:03.927Z 6 120-139 160 160 160 258 258 258 F F F
173652024-02-01T17:35:03.927Z 7 140-159 160 160 160 257 257 257 F F F
173662024-02-01T17:35:03.928Z 8 160-179 156 156 156 253 253 253 F F F
173672024-02-01T17:35:03.928Z 9 180-199 160 160 160 257 257 257 F F F
173682024-02-01T17:35:03.928Z 10 200-219 159 159 159 256 256 256 F F F
173692024-02-01T17:35:03.928Z 11 220-239 159 159 159 256 256 256 F F F
173702024-02-01T17:35:03.928Z 12 240-259 156 156 156 252 252 252 F F F
173712024-02-01T17:35:03.929Z 13 260-279 161 161 161 260 260 260 F F F
173722024-02-01T17:35:03.929Z 14 280-299 161 161 161 260 260 260 F F F
173732024-02-01T17:35:03.929Z 15 300-319 160 160 160 259 259 259 F F F
173742024-02-01T17:35:03.929Z 16 320-339 161 161 161 260 260 260 F F F
173752024-02-01T17:35:03.929Z 17 340-359 161 161 161 260 260 260 F F F
173762024-02-01T17:35:03.929Z 18 360-379 156 156 156 251 251 251 F F F
173772024-02-01T17:35:03.930Z 19 380-399 151 151 151 240 240 240 F F F
173782024-02-01T17:35:03.930Z 20 400-419 161 161 161 260 260 260 F F F
173792024-02-01T17:35:03.930Z 21 420-439 161 161 161 260 260 260 F F F
173802024-02-01T17:35:03.930Z 22 440-459 161 161 161 260 260 260 F F F
173812024-02-01T17:35:03.930Z 23 460-479 161 161 161 260 260 260 F F F
173822024-02-01T17:35:03.930Z 24 480-499 160 160 160 257 257 257 F F F
173832024-02-01T17:35:03.930Z 25 500-519 158 158 158 255 255 255 F F F
173842024-02-01T17:35:03.930Z 26 520-539 155 155 155 248 248 248 F F F
173852024-02-01T17:35:03.931Z 27 540-559 155 155 155 248 248 248 F F F
173862024-02-01T17:35:03.936Z 28 560-579 155 155 155 248 248 248 F F F
173872024-02-01T17:35:03.936Z 29 580-599 159 159 159 256 256 256 F F F
173882024-02-01T17:35:03.936ZMax gen: 161, Max flush: 260
173892024-02-01T17:35:03.936Z
173902024-02-01T17:35:03.936ZBegin loop 81
173912024-02-01T17:35:03.936ZDownstairs to restart: 0
173922024-02-01T17:35:03.956Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.238096882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
173932024-02-01T17:35:03.957Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.23859631Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
173942024-02-01T17:35:03.957Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.238636784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
173952024-02-01T17:35:03.964Z{"msg":"Crucible 150b2225-7082-4d70-a29f-261ac5ec1b4f has session id: aed33f50-74d8-4010-9313-5a6e8f173031","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.238694661Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
173962024-02-01T17:35:03.964Z{"msg":"Upstairs opts: Upstairs UUID: 150b2225-7082-4d70-a29f-261ac5ec1b4f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.238749308Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
173972024-02-01T17:35:03.964Z{"msg":"Crucible runtime is spawned
173982024-02-01T17:35:03.964ZCrucible stats registered with UUID: 150b2225-7082-4d70-a29f-261ac5ec1b4f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.23884142Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
173992024-02-01T17:35:03.964Z{"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239014121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
174002024-02-01T17:35:03.965Z{"msg":"150b2225-7082-4d70-a29f-261ac5ec1b4f active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.23908247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174012024-02-01T17:35:03.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239148695Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174022024-02-01T17:35:03.965Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239222506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174032024-02-01T17:35:03.965Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239289563Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174042024-02-01T17:35:03.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239317211Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174052024-02-01T17:35:03.966Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.23937955Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174062024-02-01T17:35:03.966Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239407049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174072024-02-01T17:35:03.966Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239446893Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174082024-02-01T17:35:03.966Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239489186Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174092024-02-01T17:35:03.966Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239693203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174102024-02-01T17:35:03.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239795504Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174112024-02-01T17:35:03.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.239852938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174122024-02-01T17:35:03.966Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.240158734Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"io task","client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174132024-02-01T17:35:03.966Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.240987778Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174142024-02-01T17:35:03.967Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.241049187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174152024-02-01T17:35:03.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.24109797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174162024-02-01T17:35:03.967Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.241178455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174172024-02-01T17:35:03.967Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.241414916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174182024-02-01T17:35:03.967Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.241603723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174192024-02-01T17:35:03.967Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.242474349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174202024-02-01T17:35:03.968Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.242613652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174212024-02-01T17:35:03.968Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.242693978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174222024-02-01T17:35:03.968Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.24364493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174232024-02-01T17:35:03.969Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243717227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174242024-02-01T17:35:03.969Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243745398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174252024-02-01T17:35:03.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243804425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174262024-02-01T17:35:03.969Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243843865Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174272024-02-01T17:35:03.969Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.24387903Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174282024-02-01T17:35:03.969Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243932326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174292024-02-01T17:35:03.969Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.243958823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174302024-02-01T17:35:03.971Z{"msg":"[0]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 256, 258, 257, 253, 257, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244000974Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174312024-02-01T17:35:03.972Z{"msg":"[0]R generation[0..12]: [160, 159, 161, 155, 159, 159, 160, 160, 156, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244026532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174322024-02-01T17:35:03.972Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244068981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174332024-02-01T17:35:03.972Z{"msg":"[1]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 256, 258, 257, 253, 257, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244094892Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174342024-02-01T17:35:03.972Z{"msg":"[1]R generation[0..12]: [160, 159, 161, 155, 159, 159, 160, 160, 156, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244135783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174352024-02-01T17:35:03.972Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244161248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174362024-02-01T17:35:03.973Z{"msg":"[2]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 256, 258, 257, 253, 257, 256, 256]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244203033Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174372024-02-01T17:35:03.987Z{"msg":"[2]R generation[0..12]: [160, 159, 161, 155, 159, 159, 160, 160, 156, 160, 159, 159]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244227947Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174382024-02-01T17:35:03.987Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244270218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174392024-02-01T17:35:03.988Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244293872Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174402024-02-01T17:35:03.988Z{Wait for a query_work_queue command to finish before sending IO
174412024-02-01T17:35:03.988Z"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244338326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174422024-02-01T17:35:03.988Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244361723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174432024-02-01T17:35:03.988Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244401921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"":"downstairs","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174442024-02-01T17:35:03.988Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244425706Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174452024-02-01T17:35:03.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244447067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"0","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174462024-02-01T17:35:03.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244497182Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"1","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174472024-02-01T17:35:03.988Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244520665Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"client":"2","":"downstairs","session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174482024-02-01T17:35:03.988Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244545098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174492024-02-01T17:35:03.988Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244589106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174502024-02-01T17:35:03.993Z{"msg":"150b2225-7082-4d70-a29f-261ac5ec1b4f is now active with session: aed33f50-74d8-4010-9313-5a6e8f173031","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244612541Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174512024-02-01T17:35:03.993Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.244632654Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780,"session_id":"aed33f50-74d8-4010-9313-5a6e8f173031"}
174522024-02-01T17:35:03.993Z{"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:03.24467161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
174532024-02-01T17:35:03.993ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174542024-02-01T17:35:03.993ZLoading write count information from file "/tmp/test_repair_verify.data"
174552024-02-01T17:35:03.993ZRun Repair workload
174562024-02-01T17:35:03.993Z01/30 Write block 184 len 40960 data: 16 18 15 15 10 12 13 13 10 10
174572024-02-01T17:35:03.993Z02/30 Write block 535 len 32768 data: 15 11 15 12 10 15 19 21
174582024-02-01T17:35:03.993Z03/30 Read block 267 len 36864
174592024-02-01T17:35:03.993Z04/30 Read block 207 len 28672
174602024-02-01T17:35:03.993Z05/30 Read block 539 len 16384
174612024-02-01T17:35:03.993Z06/30 Write block 232 len 16384 data: 10 9 9 8
174622024-02-01T17:35:03.994Z07/30 Read block 377 len 4096
174632024-02-01T17:35:03.994Z08/30 Write block 364 len 12288 data: 15 13 14
174642024-02-01T17:35:03.994Z09/30 Write block 289 len 32768 data: 17 15 15 12 14 14 12 13
174652024-02-01T17:35:03.994Z10/30 Write block 280 len 28672 data: 13 11 10 12 13 16 16
174662024-02-01T17:35:03.994Z11/30 Read block 429 len 20480
174672024-02-01T17:35:03.998Z12/30 Write block 316 len 12288 data: 17 17 17
174682024-02-01T17:35:03.999Z13/30 Read block 362 len 4096
174692024-02-01T17:35:03.999Z14/30 Write block 308 len 36864 data: 11 9 9 7 12 15 14 16 18
174702024-02-01T17:35:03.999Z15/30 Write block 328 len 40960 data: 10 12 14 17 16 16 18 17 17 14
174712024-02-01T17:35:04.000Z16/30 Read block 315 len 28672
174722024-02-01T17:35:04.000Z17/30 Read block 182 len 16384
174732024-02-01T17:35:04.000Z18/30 Read block 37 len 8192
174742024-02-01T17:35:04.000Z19/30 Flush
174752024-02-01T17:35:04.000Z20/30 Write block 268 len 8192 data: 15 13
174762024-02-01T17:35:04.000Z21/30 Read block 8 len 8192
174772024-02-01T17:35:04.000Z22/30 Write block 454 len 28672 data: 8 8 12 17 17 18 16
174782024-02-01T17:35:04.000Z23/30 Write block 289 len 12288 data: 18 16 16
174792024-02-01T17:35:04.000Z24/30 Read block 530 len 4096
174802024-02-01T17:35:04.005Z25/30 Write block 174 len 24576 data: 15 16 20 18 17 17
174812024-02-01T17:35:04.005Z26/30 Read block 167 len 40960
174822024-02-01T17:35:04.005Z27/30 Read block 215 len 28672
174832024-02-01T17:35:04.008Z28/30 Write block 114 len 40960 data: 11 11 10 10 10 9 9 9 12 12
174842024-02-01T17:35:04.008Z29/30 Write block 171 len 40960 data: 20 20 18 16 17 21 19 18 18 15
174852024-02-01T17:35:04.008Z30/30 Write block 591 len 32768 data: 11 8 7 6 6 4 4 4
174862024-02-01T17:35:04.012Z----------------------------------------------------------------
174872024-02-01T17:35:04.012Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
174882024-02-01T17:35:04.012ZStates: Active Active Active
174892024-02-01T17:35:04.012ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174902024-02-01T17:35:04.012Z 1 Acked 1000 Write 10 Done Done Done false
174912024-02-01T17:35:04.012Z 2 Acked 1001 Write 8 Sent Done Done false
174922024-02-01T17:35:04.012Z 3 Acked 1002 Read 9 Sent Done Done false
174932024-02-01T17:35:04.013Z 4 Acked 1003 Read 7 Sent Done Done false
174942024-02-01T17:35:04.013Z 5 Acked 1004 Read 4 Sent Done Done false
174952024-02-01T17:35:04.013Z 6 Acked 1005 Write 4 Sent Done Done false
174962024-02-01T17:35:04.013Z 7 Acked 1006 Read 1 Sent Done Done false
174972024-02-01T17:35:04.013Z 8 Acked 1007 Write 3 Sent Done Done false
174982024-02-01T17:35:04.013Z 9 Acked 1008 Write 8 Sent Done Done false
174992024-02-01T17:35:04.013Z 10 Acked 1009 Write 7 Sent Done Done false
175002024-02-01T17:35:04.013Z 11 Acked 1010 Read 5 Sent Done Done false
175012024-02-01T17:35:04.013Z 12 Acked 1011 Write 3 Sent Done Done false
175022024-02-01T17:35:04.013Z 13 Acked 1012 Read 1 Sent Done Done false
175032024-02-01T17:35:04.013Z 14 Acked 1013 Write 9 Sent Done Done false
175042024-02-01T17:35:04.013Z 15 Acked 1014 Write 10 Sent Done Done false
175052024-02-01T17:35:04.013Z 16 Acked 1015 Read 7 Sent Done Done false
175062024-02-01T17:35:04.013Z 17 Acked 1016 Read 4 Sent Done Done false
175072024-02-01T17:35:04.013Z 18 Acked 1017 Read 2 Sent Done Done false
175082024-02-01T17:35:04.013Z 19 Acked 1018 Flush 0 Sent Done Done false
175092024-02-01T17:35:04.013Z 20 Acked 1019 Write 2 Sent Done Done false
175102024-02-01T17:35:04.014Z 21 Acked 1020 Read 2 Sent Done Done false
175112024-02-01T17:35:04.014Z 22 Acked 1021 Write 7 Sent Done Done false
175122024-02-01T17:35:04.014Z 23 Acked 1022 Write 3 Sent Done Done false
175132024-02-01T17:35:04.014Z 24 Acked 1023 Read 1 Sent Done Done false
175142024-02-01T17:35:04.014Z 25 Acked 1024 Write 6 Sent Done Done false
175152024-02-01T17:35:04.014Z 26 Acked 1025 Read 10 Sent Done Done false
175162024-02-01T17:35:04.015Z 27 Acked 1026 Read 7 Sent Done Done false
175172024-02-01T17:35:04.015Z 28 Acked 1027 Write 10 Sent Done Done false
175182024-02-01T17:35:04.015Z 29 Acked 1028 Write 10 Sent Done Done false
175192024-02-01T17:35:04.015Z 30 Acked 1029 Write 8 Sent Sent Sent false
175202024-02-01T17:35:04.015Z STATES DS:0 DS:1 DS:2 TOTAL
175212024-02-01T17:35:04.015Z New 0 0 0 0
175222024-02-01T17:35:04.015Z Sent 29 1 1 31
175232024-02-01T17:35:04.015Z Done 1 29 29 59
175242024-02-01T17:35:04.015Z Skipped 0 0 0 0
175252024-02-01T17:35:04.015Z Error 0 0 0 0
175262024-02-01T17:35:04.015ZLast Flush: 0 1018 1018
175272024-02-01T17:35:04.015ZDownstairs last five completed:
175282024-02-01T17:35:04.015ZUpstairs last five completed: 30 29 28 27 26
175292024-02-01T17:35:04.015Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:03.292908345Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1780}
175302024-02-01T17:35:04.015ZWrote out file "/tmp/test_repair_verify.data"
175312024-02-01T17:35:04.019Z
175322024-02-01T17:35:04.024ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175332024-02-01T17:35:04.053ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175342024-02-01T17:35:04.053Z 0 000-019 160 160 160 259 259 259 F F F
175352024-02-01T17:35:04.053Z 1 020-039 159 159 159 256 256 256 F F F
175362024-02-01T17:35:04.053Z 2 040-059 161 161 161 260 260 260 F F F
175372024-02-01T17:35:04.053Z 3 060-079 155 155 155 248 248 248 F F F
175382024-02-01T17:35:04.057Z 4 080-099 159 159 159 256 256 256 F F F
175392024-02-01T17:35:04.057Z 5 100-119 159 159 159 256 256 256 F T T <---
175402024-02-01T17:35:04.058Z 6 120-139 160 160 160 258 258 258 F T T <---
175412024-02-01T17:35:04.059Z 7 140-159 160 160 160 257 257 257 F F F
175422024-02-01T17:35:04.059Z 8 160-179 156 156 156 253 253 253 F T T <---
175432024-02-01T17:35:04.059Z 9 180-199 160 162 162 257 261 261 T T T <---
175442024-02-01T17:35:04.059Z 10 200-219 159 159 159 256 256 256 F F F
175452024-02-01T17:35:04.059Z 11 220-239 159 162 162 256 261 261 F F F <---
175462024-02-01T17:35:04.059Z 12 240-259 156 156 156 252 252 252 F F F
175472024-02-01T17:35:04.059Z 13 260-279 161 161 161 260 260 260 F T T <---
175482024-02-01T17:35:04.059Z 14 280-299 161 162 162 260 261 261 F T T <---
175492024-02-01T17:35:04.059Z 15 300-319 160 162 162 259 261 261 F F F <---
175502024-02-01T17:35:04.059Z 16 320-339 161 162 162 260 261 261 F F F <---
175512024-02-01T17:35:04.059Z 17 340-359 161 161 161 260 260 260 F F F
175522024-02-01T17:35:04.059Z 18 360-379 156 162 162 251 261 261 F F F <---
175532024-02-01T17:35:04.059Z 19 380-399 151 151 151 240 240 240 F F F
175542024-02-01T17:35:04.059Z 20 400-419 161 161 161 260 260 260 F F F
175552024-02-01T17:35:04.059Z 21 420-439 161 161 161 260 260 260 F F F
175562024-02-01T17:35:04.059Z 22 440-459 161 161 161 260 260 260 F T T <---
175572024-02-01T17:35:04.059Z 23 460-479 161 161 161 260 260 260 F T T <---
175582024-02-01T17:35:04.059Z 24 480-499 160 160 160 257 257 257 F F F
175592024-02-01T17:35:04.059Z 25 500-519 158 158 158 255 255 255 F F F
175602024-02-01T17:35:04.059Z 26 520-539 155 162 162 248 261 261 F F F <---
175612024-02-01T17:35:04.059Z 27 540-559 155 162 162 248 261 261 F F F <---
175622024-02-01T17:35:04.059Z 28 560-579 155 155 155 248 248 248 F F F
175632024-02-01T17:35:04.059Z 29 580-599 159 159 159 256 256 256 F T T <---
175642024-02-01T17:35:04.060ZMax gen: 162, Max flush: 261
175652024-02-01T17:35:04.060ZError: Difference in extent metadata found!
175662024-02-01T17:35:04.064ZA Difference in extent metadata is expected here
175672024-02-01T17:35:04.064ZOn loop 81
175682024-02-01T17:35:04.064Z
175692024-02-01T17:35:04.064ZVerifying data now
175702024-02-01T17:35:09.747ZNo change after verify
175712024-02-01T17:35:09.747ZLoop: 81 Downstairs dump after verify (and repair):
175722024-02-01T17:35:09.823ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175732024-02-01T17:35:09.824Z 0 000-019 160 160 160 259 259 259 F F F
175742024-02-01T17:35:09.824Z 1 020-039 159 159 159 256 256 256 F F F
175752024-02-01T17:35:09.824Z 2 040-059 161 161 161 260 260 260 F F F
175762024-02-01T17:35:09.824Z 3 060-079 155 155 155 248 248 248 F F F
175772024-02-01T17:35:09.824Z 4 080-099 159 159 159 256 256 256 F F F
175782024-02-01T17:35:09.824Z 5 100-119 163 163 163 262 262 262 F F F
175792024-02-01T17:35:09.824Z 6 120-139 163 163 163 262 262 262 F F F
175802024-02-01T17:35:09.824Z 7 140-159 160 160 160 257 257 257 F F F
175812024-02-01T17:35:09.824Z 8 160-179 163 163 163 262 262 262 F F F
175822024-02-01T17:35:09.825Z 9 180-199 163 163 163 262 262 262 F F F
175832024-02-01T17:35:09.826Z 10 200-219 159 159 159 256 256 256 F F F
175842024-02-01T17:35:09.826Z 11 220-239 162 162 162 261 261 261 F F F
175852024-02-01T17:35:09.826Z 12 240-259 156 156 156 252 252 252 F F F
175862024-02-01T17:35:09.826Z 13 260-279 163 163 163 262 262 262 F F F
175872024-02-01T17:35:09.826Z 14 280-299 163 163 163 262 262 262 F F F
175882024-02-01T17:35:09.826Z 15 300-319 162 162 162 261 261 261 F F F
175892024-02-01T17:35:09.826Z 16 320-339 162 162 162 261 261 261 F F F
175902024-02-01T17:35:09.826Z 17 340-359 161 161 161 260 260 260 F F F
175912024-02-01T17:35:09.827Z 18 360-379 162 162 162 261 261 261 F F F
175922024-02-01T17:35:09.827Z 19 380-399 151 151 151 240 240 240 F F F
175932024-02-01T17:35:09.827Z 20 400-419 161 161 161 260 260 260 F F F
175942024-02-01T17:35:09.827Z 21 420-439 161 161 161 260 260 260 F F F
175952024-02-01T17:35:09.827Z 22 440-459 163 163 163 262 262 262 F F F
175962024-02-01T17:35:09.827Z 23 460-479 163 163 163 262 262 262 F F F
175972024-02-01T17:35:09.827Z 24 480-499 160 160 160 257 257 257 F F F
175982024-02-01T17:35:09.828Z 25 500-519 158 158 158 255 255 255 F F F
175992024-02-01T17:35:09.828Z 26 520-539 162 162 162 261 261 261 F F F
176002024-02-01T17:35:09.828Z 27 540-559 162 162 162 261 261 261 F F F
176012024-02-01T17:35:09.828Z 28 560-579 155 155 155 248 248 248 F F F
176022024-02-01T17:35:09.828Z 29 580-599 163 163 163 262 262 262 F F F
176032024-02-01T17:35:09.828ZMax gen: 163, Max flush: 262
176042024-02-01T17:35:09.828Z
176052024-02-01T17:35:09.829ZBegin loop 82
176062024-02-01T17:35:09.829ZDownstairs to restart: 2
176072024-02-01T17:35:09.829Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.669731668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
176082024-02-01T17:35:09.829Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670242617Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
176092024-02-01T17:35:09.829Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670317976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
176102024-02-01T17:35:09.829Z{"msg":"Crucible 7c8d8982-08fd-4494-bdaa-b85bdfa864fd has session id: 2a3507d0-e2cf-47cc-9a30-14bc2c077360","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670387676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176112024-02-01T17:35:09.830Z{"msg":"Crucible runtime is spawned
176122024-02-01T17:35:09.830ZUpstairs opts: Upstairs UUID: 7c8d8982-08fd-4494-bdaa-b85bdfa864fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670481027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176132024-02-01T17:35:09.830Z{"msg":"Crucible stats registered with UUID: 7c8d8982-08fd-4494-bdaa-b85bdfa864fd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670569515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176142024-02-01T17:35:09.830Z{"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670614218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
176152024-02-01T17:35:09.830Z{"msg":"7c8d8982-08fd-4494-bdaa-b85bdfa864fd active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670677986Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176162024-02-01T17:35:09.830Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670717619Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176172024-02-01T17:35:09.831Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670801085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176182024-02-01T17:35:09.831Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670935121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176192024-02-01T17:35:09.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.670981427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176202024-02-01T17:35:09.832Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671018127Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176212024-02-01T17:35:09.832Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671060171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176222024-02-01T17:35:09.832Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671103912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176232024-02-01T17:35:09.832Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.67114737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176242024-02-01T17:35:09.832Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671212997Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176252024-02-01T17:35:09.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.67156642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176262024-02-01T17:35:09.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671717547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176272024-02-01T17:35:09.833Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.671841297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"io task","client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176282024-02-01T17:35:09.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672498824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176292024-02-01T17:35:09.833Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672563475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176302024-02-01T17:35:09.833Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672718276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176312024-02-01T17:35:09.834Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672785514Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176322024-02-01T17:35:09.834Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672830904Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176332024-02-01T17:35:09.834Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.672876386Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176342024-02-01T17:35:09.834Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.673420138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176352024-02-01T17:35:09.834Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.673911322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176362024-02-01T17:35:09.834Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674289002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176372024-02-01T17:35:09.834Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674354444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176382024-02-01T17:35:09.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674382245Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176392024-02-01T17:35:09.835Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674405366Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176402024-02-01T17:35:09.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674698735Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176412024-02-01T17:35:09.835Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674787693Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176422024-02-01T17:35:09.835Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.674864463Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176432024-02-01T17:35:09.835Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675262426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176442024-02-01T17:35:09.835Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675324951Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176452024-02-01T17:35:09.836Z{"msg":"[0]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 262, 262, 257, 262, 262, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675353354Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176462024-02-01T17:35:09.836Z{"msg":"[0]R generation[0..12]: [160, 159, 161, 155, 159, 163, 163, 160, 163, 163, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675396259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176472024-02-01T17:35:09.836Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675422647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176482024-02-01T17:35:09.836Z{"msg":"[1]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 262, 262, 257, 262, 262, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675468227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176492024-02-01T17:35:09.836Z{"msg":"[1]R generation[0..12]: [160, 159, 161, 155, 159, 163, 163, 160, 163, 163, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675493678Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176502024-02-01T17:35:09.836Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176512024-02-01T17:35:09.837Z,"time":"2024-02-01T17:35:08.675554263Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176522024-02-01T17:35:09.837Z{"msg":"[2]R flush_numbers[0..12]: [259, 256, 260, 248, 256, 262, 262, 257, 262, 262, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675605352Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176532024-02-01T17:35:09.837Z{"msg":"[2]R generation[0..12]: [160, 159, 161, 155, 159, 163, 163, 160, 163, 163, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675630722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176542024-02-01T17:35:09.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675675557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176552024-02-01T17:35:09.837Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675699506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176562024-02-01T17:35:09.837Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675745835Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176572024-02-01T17:35:09.838Z{"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675769289Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176582024-02-01T17:35:09.839Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675809557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"":"downstairs","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176592024-02-01T17:35:09.839Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675833818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176602024-02-01T17:35:09.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675853928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"0","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176612024-02-01T17:35:09.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675897147Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"1","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176622024-02-01T17:35:09.839Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.67592109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"client":"2","":"downstairs","session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176632024-02-01T17:35:09.839Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675962455Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176642024-02-01T17:35:09.840Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.675985468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176652024-02-01T17:35:09.840Z{"msg":"7c8d8982-08fd-4494-bdaa-b85bdfa864fd is now active with session: 2a3507d0-e2cf-47cc-9a30-14bc2c077360","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.676035237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176662024-02-01T17:35:09.840Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.676057576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788,"session_id":"2a3507d0-e2cf-47cc-9a30-14bc2c077360"}
176672024-02-01T17:35:09.841Z{"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:08.676096839Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
176682024-02-01T17:35:09.841ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176692024-02-01T17:35:09.841ZLoading write count information from file "/tmp/test_repair_verify.data"
176702024-02-01T17:35:09.841ZRun Repair workload
176712024-02-01T17:35:09.842Z01/30 Write block 351 len 16384 data: 13 15 13 18
176722024-02-01T17:35:10.387Z02/30 Read block 435 len 32768
176732024-02-01T17:35:10.387Z03/30 Read block 343 len 40960
176742024-02-01T17:35:10.387Z04/30 Read block 251 len 36864
176752024-02-01T17:35:10.387Z05/30 Read block 441 len 32768
176762024-02-01T17:35:10.387Z06/30 Write block 172 len 24576 data: 21 19 17 18 22 20
176772024-02-01T17:35:10.387Z07/30 Write block 495 len 28672 data: 14 12 12 13 13 12 11
176782024-02-01T17:35:10.387Z08/30 Write block 419 len 12288 data: 14 11 11
176792024-02-01T17:35:10.387Z09/30 Write block 379 len 20480 data: 12 12 14 10 10
176802024-02-01T17:35:10.387Z10/30 Flush
176812024-02-01T17:35:10.387Z11/30 Write block 49 len 36864 data: 13 14 15 13 14 16 14 12 11
176822024-02-01T17:35:10.387Z12/30 Read block 306 len 8192
176832024-02-01T17:35:10.387Z13/30 Write block 314 len 28672 data: 15 17 19 18 18 18 17
176842024-02-01T17:35:10.388Z14/30 Read block 442 len 16384
176852024-02-01T17:35:10.388Z15/30 Read block 280 len 32768
176862024-02-01T17:35:10.388Z16/30 Write block 251 len 24576 data: 9 11 9 11 13 13
176872024-02-01T17:35:10.388Z17/30 Write block 552 len 4096 data: 11
176882024-02-01T17:35:10.388Z18/30 Write block 396 len 28672 data: 8 10 9 7 8 11 10
176892024-02-01T17:35:10.388Z19/30 Write block 241 len 36864 data: 8 12 12 12 13 13 12 13 10
176902024-02-01T17:35:10.389Z20/30 Read block 367 len 24576
176912024-02-01T17:35:10.389Z21/30 Write block 134 len 36864 data: 14 15 16 16 17 14 14 13 11
176922024-02-01T17:35:10.389Z22/30 Read block 304 len 32768
176932024-02-01T17:35:10.389Z23/30 Write block 170 len 20480 data: 19 21 22 20 18
176942024-02-01T17:35:10.389Z24/30 Read block 281 len 24576
176952024-02-01T17:35:10.389Z25/30 Read block 317 len 12288
176962024-02-01T17:35:10.389Z26/30 Write block 190 len 28672 data: 14 14 11 11 12 12 13
176972024-02-01T17:35:10.389Z27/30 Read block 218 len 40960
176982024-02-01T17:35:10.389Z28/30 Write block 277 len 20480 data: 16 16 12 14 12
176992024-02-01T17:35:10.389Z29/30 Write block 482 len 12288 data: 11 11 12
177002024-02-01T17:35:10.390Z30/30 Read block 331 len 20480
177012024-02-01T17:35:10.390Z----------------------------------------------------------------
177022024-02-01T17:35:10.390Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30
177032024-02-01T17:35:10.390ZStates: Active Active Active
177042024-02-01T17:35:10.390ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177052024-02-01T17:35:10.390Z 1 Acked 1000 Write 4 Done Done Done false
177062024-02-01T17:35:10.390Z 2 Acked 1001 Read 8 Done Done Done false
177072024-02-01T17:35:10.391Z 3 Acked 1002 Read 10 Done Done Sent false
177082024-02-01T17:35:10.391Z 4 Acked 1003 Read 9 Done Done Sent false
177092024-02-01T17:35:10.391Z 5 Acked 1004 Read 8 Done Done Sent false
177102024-02-01T17:35:10.391Z 6 Acked 1005 Write 6 Done Done Sent false
177112024-02-01T17:35:10.391Z 7 Acked 1006 Write 7 Done Done Sent false
177122024-02-01T17:35:10.391Z 8 Acked 1007 Write 3 Done Done Sent false
177132024-02-01T17:35:10.391Z 9 Acked 1008 Write 5 Done Done Sent false
177142024-02-01T17:35:10.392Z 10 Acked 1009 Flush 0 Done Done Sent false
177152024-02-01T17:35:10.392Z 11 Acked 1010 Write 9 Done Done Sent false
177162024-02-01T17:35:10.392Z 12 Acked 1011 Read 2 Done Done Sent false
177172024-02-01T17:35:10.392Z 13 Acked 1012 Write 7 Done Done Sent false
177182024-02-01T17:35:10.392Z 14 Acked 1013 Read 4 Done Done Sent false
177192024-02-01T17:35:10.392Z 15 Acked 1014 Read 8 Done Done Sent false
177202024-02-01T17:35:10.392Z 16 Acked 1015 Write 6 Done Done Sent false
177212024-02-01T17:35:10.392Z 17 Acked 1016 Write 1 Done Done Sent false
177222024-02-01T17:35:10.392Z 18 Acked 1017 Write 7 Done Done Sent false
177232024-02-01T17:35:10.392Z 19 Acked 1018 Write 9 Done Done Sent false
177242024-02-01T17:35:10.393Z 20 Acked 1019 Read 6 Done Done Sent false
177252024-02-01T17:35:10.393Z 21 Acked 1020 Write 9 Done Done Sent false
177262024-02-01T17:35:10.393Z 22 Acked 1021 Read 8 Done Done Sent false
177272024-02-01T17:35:10.393Z 23 Acked 1022 Write 5 Done Done Sent false
177282024-02-01T17:35:10.393Z 24 Acked 1023 Read 6 Done Done Sent false
177292024-02-01T17:35:10.393Z 25 Acked 1024 Read 3 Done Done Sent false
177302024-02-01T17:35:10.393Z 26 Acked 1025 Write 7 Done Done Sent false
177312024-02-01T17:35:10.393Z 27 Acked 1026 Read 10 Done Done Sent false
177322024-02-01T17:35:10.393Z 28 Acked 1027 Write 5 Done Done Sent false
177332024-02-01T17:35:10.393Z 29 Acked 1028 Write 3 Done Done Sent false
177342024-02-01T17:35:10.393Z 30 Acked 1029 Read 5 Done Sent Sent false
177352024-02-01T17:35:10.393Z STATES DS:0 DS:1 DS:2 TOTAL
177362024-02-01T17:35:10.394Z New 0 0 0 0
177372024-02-01T17:35:10.394Z Sent 0 1 28 29
177382024-02-01T17:35:10.395Z Done 30 29 2 61
177392024-02-01T17:35:10.395Z Skipped 0 0 0 0
177402024-02-01T17:35:10.395Z Error 0 0 0 0
177412024-02-01T17:35:10.395ZLast Flush: 1009 1009 0
177422024-02-01T17:35:10.395ZDownstairs last five completed:
177432024-02-01T17:35:10.395ZUpstairs last five completed: 30 29 28 27 26
177442024-02-01T17:35:10.395Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:08.725744412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1788}
177452024-02-01T17:35:10.395ZWrote out file "/tmp/test_repair_verify.data"
177462024-02-01T17:35:10.395Z
177472024-02-01T17:35:10.395ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177482024-02-01T17:35:10.395ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177492024-02-01T17:35:10.395Z 0 000-019 160 160 160 259 259 259 F F F
177502024-02-01T17:35:10.395Z 1 020-039 159 159 159 256 256 256 F F F
177512024-02-01T17:35:10.396Z 2 040-059 161 161 161 260 260 260 T T F <---
177522024-02-01T17:35:10.396Z 3 060-079 155 155 155 248 248 248 F F F
177532024-02-01T17:35:10.396Z 4 080-099 159 159 159 256 256 256 F F F
177542024-02-01T17:35:10.396Z 5 100-119 163 163 163 262 262 262 F F F
177552024-02-01T17:35:10.396Z 6 120-139 163 163 163 262 262 262 T T F <---
177562024-02-01T17:35:10.396Z 7 140-159 160 160 160 257 257 257 T T F <---
177572024-02-01T17:35:10.396Z 8 160-179 164 164 163 263 263 262 T T F <---
177582024-02-01T17:35:10.396Z 9 180-199 163 163 163 262 262 262 T T F <---
177592024-02-01T17:35:10.396Z 10 200-219 159 159 159 256 256 256 F F F
177602024-02-01T17:35:10.396Z 11 220-239 162 162 162 261 261 261 F F F
177612024-02-01T17:35:10.397Z 12 240-259 156 156 156 252 252 252 T T F <---
177622024-02-01T17:35:10.397Z 13 260-279 163 163 163 262 262 262 T T F <---
177632024-02-01T17:35:10.397Z 14 280-299 163 163 163 262 262 262 T T F <---
177642024-02-01T17:35:10.397Z 15 300-319 162 162 162 261 261 261 T T F <---
177652024-02-01T17:35:10.397Z 16 320-339 162 162 162 261 261 261 T T F <---
177662024-02-01T17:35:10.397Z 17 340-359 164 164 161 263 263 260 F F T <---
177672024-02-01T17:35:10.397Z 18 360-379 164 164 162 263 263 261 F F F <---
177682024-02-01T17:35:10.397Z 19 380-399 164 164 151 263 263 240 T T F <---
177692024-02-01T17:35:10.397Z 20 400-419 164 164 161 263 263 260 T T F <---
177702024-02-01T17:35:10.397Z 21 420-439 164 164 161 263 263 260 F F F <---
177712024-02-01T17:35:10.397Z 22 440-459 163 163 163 262 262 262 F F F
177722024-02-01T17:35:10.422Z 23 460-479 163 163 163 262 262 262 F F F
177732024-02-01T17:35:10.423Z 24 480-499 164 164 160 263 263 257 T T F <---
177742024-02-01T17:35:10.423Z 25 500-519 164 164 158 263 263 255 F F F <---
177752024-02-01T17:35:10.423Z 26 520-539 162 162 162 261 261 261 F F F
177762024-02-01T17:35:10.423Z 27 540-559 162 162 162 261 261 261 T T F <---
177772024-02-01T17:35:10.423Z 28 560-579 155 155 155 248 248 248 F F F
177782024-02-01T17:35:10.423Z 29 580-599 163 163 163 262 262 262 F F F
177792024-02-01T17:35:10.423ZMax gen: 164, Max flush: 263
177802024-02-01T17:35:10.423ZError: Difference in extent metadata found!
177812024-02-01T17:35:10.423ZA Difference in extent metadata is expected here
177822024-02-01T17:35:10.423ZOn loop 82
177832024-02-01T17:35:10.423Z
177842024-02-01T17:35:10.424ZVerifying data now
177852024-02-01T17:35:15.080ZNo change after verify
177862024-02-01T17:35:15.080ZLoop: 82 Downstairs dump after verify (and repair):
177872024-02-01T17:35:15.099ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177882024-02-01T17:35:15.099Z 0 000-019 160 160 160 259 259 259 F F F
177892024-02-01T17:35:15.099Z 1 020-039 159 159 159 256 256 256 F F F
177902024-02-01T17:35:15.100Z 2 040-059 165 165 165 264 264 264 F F F
177912024-02-01T17:35:15.100Z 3 060-079 155 155 155 248 248 248 F F F
177922024-02-01T17:35:15.100Z 4 080-099 159 159 159 256 256 256 F F F
177932024-02-01T17:35:15.100Z 5 100-119 163 163 163 262 262 262 F F F
177942024-02-01T17:35:15.100Z 6 120-139 165 165 165 264 264 264 F F F
177952024-02-01T17:35:15.100Z 7 140-159 165 165 165 264 264 264 F F F
177962024-02-01T17:35:15.101Z 8 160-179 165 165 165 264 264 264 F F F
177972024-02-01T17:35:15.101Z 9 180-199 165 165 165 264 264 264 F F F
177982024-02-01T17:35:15.101Z 10 200-219 159 159 159 256 256 256 F F F
177992024-02-01T17:35:15.101Z 11 220-239 162 162 162 261 261 261 F F F
178002024-02-01T17:35:15.101Z 12 240-259 165 165 165 264 264 264 F F F
178012024-02-01T17:35:15.101Z 13 260-279 165 165 165 264 264 264 F F F
178022024-02-01T17:35:15.101Z 14 280-299 165 165 165 264 264 264 F F F
178032024-02-01T17:35:15.101Z 15 300-319 165 165 165 264 264 264 F F F
178042024-02-01T17:35:15.101Z 16 320-339 165 165 165 264 264 264 F F F
178052024-02-01T17:35:15.102Z 17 340-359 164 164 164 263 263 263 F F F
178062024-02-01T17:35:15.102Z 18 360-379 164 164 164 263 263 263 F F F
178072024-02-01T17:35:15.102Z 19 380-399 165 165 165 264 264 264 F F F
178082024-02-01T17:35:15.103Z 20 400-419 165 165 165 264 264 264 F F F
178092024-02-01T17:35:15.103Z 21 420-439 164 164 164 263 263 263 F F F
178102024-02-01T17:35:15.103Z 22 440-459 163 163 163 262 262 262 F F F
178112024-02-01T17:35:15.103Z 23 460-479 163 163 163 262 262 262 F F F
178122024-02-01T17:35:15.103Z 24 480-499 165 165 165 264 264 264 F F F
178132024-02-01T17:35:15.103Z 25 500-519 164 164 164 263 263 263 F F F
178142024-02-01T17:35:15.103Z 26 520-539 162 162 162 261 261 261 F F F
178152024-02-01T17:35:15.103Z 27 540-559 165 165 165 264 264 264 F F F
178162024-02-01T17:35:15.103Z 28 560-579 155 155 155 248 248 248 F F F
178172024-02-01T17:35:15.104Z 29 580-599 163 163 163 262 262 262 F F F
178182024-02-01T17:35:15.104ZMax gen: 165, Max flush: 264
178192024-02-01T17:35:15.104Z
178202024-02-01T17:35:15.104ZBegin loop 83
178212024-02-01T17:35:15.104ZDownstairs to restart: 1
178222024-02-01T17:35:15.104Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.352732198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
178232024-02-01T17:35:15.104Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353314301Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
178242024-02-01T17:35:15.104Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353385217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
178252024-02-01T17:35:15.104Z{"msg":"Crucible ed0c9e1c-4f72-42a8-8fdd-6712c6e2457e has session id: 9b6f78be-22b6-4dab-942e-77dd7ba8d669","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353424816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178262024-02-01T17:35:15.105Z{"msg":"Upstairs opts: Upstairs UUID: ed0c9e1c-4f72-42a8-8fdd-6712c6e2457e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353475121Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178272024-02-01T17:35:15.105Z{"msg":"Crucible stats registered with UUID: ed0c9e1c-4f72-42a8-8fdd-6712c6e2457e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353546774Z","hostname":"Crucible runtime is spawned
178282024-02-01T17:35:15.105Zip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178292024-02-01T17:35:15.105Z{"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353737258Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
178302024-02-01T17:35:15.105Z{"msg":"ed0c9e1c-4f72-42a8-8fdd-6712c6e2457e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.353894153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178312024-02-01T17:35:15.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354001328Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178322024-02-01T17:35:15.105Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354054958Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178332024-02-01T17:35:15.105Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354094651Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178342024-02-01T17:35:15.106Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354177921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178352024-02-01T17:35:15.106Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354375443Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178362024-02-01T17:35:15.106Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354470479Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178372024-02-01T17:35:15.106Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354582286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178382024-02-01T17:35:15.106Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354660316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178392024-02-01T17:35:15.106Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354758187Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178402024-02-01T17:35:15.109Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.35483472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178412024-02-01T17:35:15.109Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354917833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178422024-02-01T17:35:15.109Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.354966423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"io task","client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178432024-02-01T17:35:15.110Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.355846825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178442024-02-01T17:35:15.110Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.35590915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178452024-02-01T17:35:15.110Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.355936578Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178462024-02-01T17:35:15.110Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.356001244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178472024-02-01T17:35:15.110Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.356137588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178482024-02-01T17:35:15.110Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.35618218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178492024-02-01T17:35:15.110Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.356898829Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178502024-02-01T17:35:15.112Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.356988376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178512024-02-01T17:35:15.112Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.357110608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178522024-02-01T17:35:15.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358083725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178532024-02-01T17:35:15.112Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358143628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178542024-02-01T17:35:15.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358170138Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178552024-02-01T17:35:15.112Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358287736Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178562024-02-01T17:35:15.112Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358339394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178572024-02-01T17:35:15.112Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358365392Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178582024-02-01T17:35:15.113Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358389751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178592024-02-01T17:35:15.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358447879Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178602024-02-01T17:35:15.114Z{"msg":"[0]R flush_numbers[0..12]: [259, 256, 264, 248, 256, 262, 264, 264, 264, 264, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358475824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178612024-02-01T17:35:15.114Z{"msg":"[0]R generation[0..12]: [160, 159, 165, 155, 159, 163, 165, 165, 165, 165, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.35853306Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178622024-02-01T17:35:15.114Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358562184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178632024-02-01T17:35:15.114Z{"msg":"[1]R flush_numbers[0..12]: [259, 256, 264, 248, 256, 262, 264, 264, 264, 264, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358590626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178642024-02-01T17:35:15.115Z{"msg":"[1]R generation[0..12]: [160, 159, 165, 155, 159, 163, 165, 165, 165, 165, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358650871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178652024-02-01T17:35:15.115Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358678432Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,Wait for a query_work_queue command to finish before sending IO
178662024-02-01T17:35:15.115Z"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178672024-02-01T17:35:15.115Z{"msg":"[2]R flush_numbers[0..12]: [259, 256, 264, 248, 256, 262, 264, 264, 264, 264, 256, 261]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358721538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178682024-02-01T17:35:15.115Z{"msg":"[2]R generation[0..12]: [160, 159, 165, 155, 159, 163, 165, 165, 165, 165, 159, 162]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358774711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178692024-02-01T17:35:15.115Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358801526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178702024-02-01T17:35:15.118Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358862868Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178712024-02-01T17:35:15.118Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358889804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178722024-02-01T17:35:15.118Z{"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358922435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178732024-02-01T17:35:15.118Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358951489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"":"downstairs","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178742024-02-01T17:35:15.118Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.358981132Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178752024-02-01T17:35:15.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359075892Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"0","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178762024-02-01T17:35:15.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359102932Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"1","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178772024-02-01T17:35:15.119Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359160133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"client":"2","":"downstairs","session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178782024-02-01T17:35:15.119Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359190257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178792024-02-01T17:35:15.119Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359243978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178802024-02-01T17:35:15.119Z{"msg":"ed0c9e1c-4f72-42a8-8fdd-6712c6e2457e is now active with session: 9b6f78be-22b6-4dab-942e-77dd7ba8d669","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359275645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178812024-02-01T17:35:15.119Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359335507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796,"session_id":"9b6f78be-22b6-4dab-942e-77dd7ba8d669"}
178822024-02-01T17:35:15.120ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
178832024-02-01T17:35:15.120Z{"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:14.359368338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
178842024-02-01T17:35:15.120ZLoading write count information from file "/tmp/test_repair_verify.data"
178852024-02-01T17:35:15.120ZRun Repair workload
178862024-02-01T17:35:15.120Z01/30 Write block 47 len 36864 data: 12 15 14 15 16 14 15 17 15
178872024-02-01T17:35:15.278Z02/30 Write block 403 len 12288 data: 13 12 9
178882024-02-01T17:35:15.278Z03/30 Flush
178892024-02-01T17:35:15.278Z04/30 Write block 175 len 4096 data: 19
178902024-02-01T17:35:15.278Z05/30 Write block 98 len 32768 data: 9 11 13 14 15 13 14 17
178912024-02-01T17:35:15.279Z06/30 Read block 207 len 40960
178922024-02-01T17:35:15.279Z07/30 Write block 303 len 8192 data: 10 10
178932024-02-01T17:35:15.279Z08/30 Read block 19 len 40960
178942024-02-01T17:35:15.279Z09/30 Flush
178952024-02-01T17:35:15.279Z10/30 Write block 485 len 20480 data: 11 13 16 18 19
178962024-02-01T17:35:15.279Z11/30 Write block 495 len 24576 data: 15 13 13 14 14 13
178972024-02-01T17:35:15.279Z12/30 Read block 467 len 20480
178982024-02-01T17:35:15.279Z13/30 Flush
178992024-02-01T17:35:15.279Z14/30 Write block 560 len 12288 data: 13 12 12
179002024-02-01T17:35:15.289Z15/30 Read block 542 len 4096
179012024-02-01T17:35:15.289Z16/30 Read block 28 len 20480
179022024-02-01T17:35:15.289Z17/30 Write block 124 len 12288 data: 12 14 11
179032024-02-01T17:35:15.289Z18/30 Read block 511 len 28672
179042024-02-01T17:35:15.289Z19/30 Write block 182 len 20480 data: 18 17 17 19 16
179052024-02-01T17:35:15.289Z20/30 Flush
179062024-02-01T17:35:15.289Z21/30 Write block 281 len 24576 data: 13 11 13 14 17 17
179072024-02-01T17:35:15.289Z22/30 Write block 272 len 16384 data: 10 10 9 12
179082024-02-01T17:35:15.289Z23/30 Write block 465 len 8192 data: 12 15
179092024-02-01T17:35:15.289Z24/30 Write block 212 len 24576 data: 13 12 11 11 7 8
179102024-02-01T17:35:15.289Z25/30 Write block 232 len 4096 data: 11
179112024-02-01T17:35:15.290Z26/30 Flush
179122024-02-01T17:35:15.290Z27/30 Write block 110 len 36864 data: 11 10 9 11 12 12 11 11 11
179132024-02-01T17:35:15.290Z28/30 Write block 55 len 40960 data: 16 13 12 9 11 12 10 10 10 9
179142024-02-01T17:35:15.290Z29/30 Read block 472 len 24576
179152024-02-01T17:35:15.290Z30/30 Read block 141 len 24576
179162024-02-01T17:35:15.290Z----------------------------------------------------------------
179172024-02-01T17:35:15.290Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:27
179182024-02-01T17:35:15.290ZStates: Active Active Active
179192024-02-01T17:35:15.290ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179202024-02-01T17:35:15.290Z 4 Acked 1003 Write 1 Done Done Done false
179212024-02-01T17:35:15.290Z 5 Acked 1004 Write 8 Done Done Done false
179222024-02-01T17:35:15.290Z 6 Acked 1005 Read 10 Done Sent Done false
179232024-02-01T17:35:15.290Z 7 Acked 1006 Write 2 Done Sent Done false
179242024-02-01T17:35:15.290Z 8 Acked 1007 Read 10 Done Sent Done false
179252024-02-01T17:35:15.290Z 9 Acked 1008 Flush 0 Done Sent Done false
179262024-02-01T17:35:15.290Z 10 Acked 1009 Write 5 Done Sent Done false
179272024-02-01T17:35:15.290Z 11 Acked 1010 Write 6 Done Sent Done false
179282024-02-01T17:35:15.290Z 12 Acked 1011 Read 5 Done Sent Done false
179292024-02-01T17:35:15.290Z 13 Acked 1012 Flush 0 Done Sent Done false
179302024-02-01T17:35:15.290Z 14 Acked 1013 Write 3 Done Sent Done false
179312024-02-01T17:35:15.290Z 15 Acked 1014 Read 1 Done Sent Done false
179322024-02-01T17:35:15.290Z 16 Acked 1015 Read 5 Done Sent Done false
179332024-02-01T17:35:15.290Z 17 Acked 1016 Write 3 Done Sent Done false
179342024-02-01T17:35:15.291Z 18 Acked 1017 Read 7 Done Sent Done false
179352024-02-01T17:35:15.291Z 19 Acked 1018 Write 5 Done Sent Done false
179362024-02-01T17:35:15.291Z 20 Acked 1019 Flush 0 Done Sent Done false
179372024-02-01T17:35:15.291Z 21 Acked 1020 Write 6 Done Sent Done false
179382024-02-01T17:35:15.291Z 22 Acked 1021 Write 4 Done Sent Done false
179392024-02-01T17:35:15.291Z 23 Acked 1022 Write 2 Done Sent Done false
179402024-02-01T17:35:15.291Z 24 Acked 1023 Write 6 Done Sent Done false
179412024-02-01T17:35:15.291Z 25 Acked 1024 Write 1 Done Sent Done false
179422024-02-01T17:35:15.291Z 26 Acked 1025 Flush 0 Done Sent Done false
179432024-02-01T17:35:15.291Z 27 Acked 1026 Write 9 Done Sent Done false
179442024-02-01T17:35:15.291Z 28 Acked 1027 Write 10 Done Sent Done false
179452024-02-01T17:35:15.304Z 29 Acked 1028 Read 6 Done Sent Done false
179462024-02-01T17:35:15.304Z 30 Acked 1029 Read 6 Done Sent Sent false
179472024-02-01T17:35:15.304Z STATES DS:0 DS:1 DS:2 TOTAL
179482024-02-01T17:35:15.304Z New 0 0 0 0
179492024-02-01T17:35:15.304Z Sent 0 25 1 26
179502024-02-01T17:35:15.304Z Done 27 2 26 55
179512024-02-01T17:35:15.305Z Skipped 0 0 0 0
179522024-02-01T17:35:15.305Z Error 0 0 0 0
179532024-02-01T17:35:15.305ZLast Flush: 1025 1002 1025
179542024-02-01T17:35:15.305ZDownstairs last five completed: 1002 1001 1000
179552024-02-01T17:35:15.305ZUpstairs last five completed: 30 29 28 27 26
179562024-02-01T17:35:15.305Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:14.409256453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1796}
179572024-02-01T17:35:15.305ZWrote out file "/tmp/test_repair_verify.data"
179582024-02-01T17:35:15.305Z
179592024-02-01T17:35:15.305ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
179602024-02-01T17:35:15.305ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
179612024-02-01T17:35:15.305Z 0 000-019 160 160 160 259 259 259 F F F
179622024-02-01T17:35:15.305Z 1 020-039 159 159 159 256 256 256 F F F
179632024-02-01T17:35:15.305Z 2 040-059 166 166 166 265 265 265 T F T <---
179642024-02-01T17:35:15.305Z 3 060-079 155 155 155 248 248 248 T F T <---
179652024-02-01T17:35:15.305Z 4 080-099 166 159 166 266 256 266 F T F <---
179662024-02-01T17:35:15.305Z 5 100-119 166 163 166 266 262 266 T T T <---
179672024-02-01T17:35:15.306Z 6 120-139 166 165 166 268 264 268 F F F <---
179682024-02-01T17:35:15.306Z 7 140-159 165 165 165 264 264 264 F F F
179692024-02-01T17:35:15.306Z 8 160-179 166 165 166 266 264 266 F T F <---
179702024-02-01T17:35:15.306Z 9 180-199 166 165 166 268 264 268 F F F <---
179712024-02-01T17:35:15.306Z 10 200-219 166 159 166 269 256 269 F F F <---
179722024-02-01T17:35:15.306Z 11 220-239 166 162 166 269 261 269 F F F <---
179732024-02-01T17:35:15.306Z 12 240-259 165 165 165 264 264 264 F F F
179742024-02-01T17:35:15.306Z 13 260-279 166 165 166 269 264 269 F F F <---
179752024-02-01T17:35:15.306Z 14 280-299 166 165 166 269 264 269 F F F <---
179762024-02-01T17:35:15.306Z 15 300-319 166 165 166 266 264 266 F F F <---
179772024-02-01T17:35:15.306Z 16 320-339 165 165 165 264 264 264 F F F
179782024-02-01T17:35:15.306Z 17 340-359 164 164 164 263 263 263 F F F
179792024-02-01T17:35:15.306Z 18 360-379 164 164 164 263 263 263 F F F
179802024-02-01T17:35:15.306Z 19 380-399 165 165 165 264 264 264 F F F
179812024-02-01T17:35:15.306Z 20 400-419 166 166 166 265 265 265 F F F
179822024-02-01T17:35:15.306Z 21 420-439 164 164 164 263 263 263 F F F
179832024-02-01T17:35:15.307Z 22 440-459 163 163 163 262 262 262 F F F
179842024-02-01T17:35:15.307Z 23 460-479 166 163 166 269 262 269 F F F <---
179852024-02-01T17:35:15.307Z 24 480-499 166 165 166 267 264 267 F F F <---
179862024-02-01T17:35:15.307Z 25 500-519 166 164 166 267 263 267 F F F <---
179872024-02-01T17:35:15.307Z 26 520-539 162 162 162 261 261 261 F F F
179882024-02-01T17:35:15.307Z 27 540-559 165 165 165 264 264 264 F F F
179892024-02-01T17:35:15.307Z 28 560-579 166 155 166 268 248 268 F F F <---
179902024-02-01T17:35:15.307Z 29 580-599 163 163 163 262 262 262 F F F
179912024-02-01T17:35:15.307ZMax gen: 166, Max flush: 269
179922024-02-01T17:35:15.307ZError: Difference in extent metadata found!
179932024-02-01T17:35:15.308ZA Difference in extent metadata is expected here
179942024-02-01T17:35:15.309ZOn loop 83
179952024-02-01T17:35:15.309Z
179962024-02-01T17:35:15.309ZVerifying data now
179972024-02-01T17:35:21.482ZNo change after verify
179982024-02-01T17:35:21.482ZLoop: 83 Downstairs dump after verify (and repair):
179992024-02-01T17:35:21.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180002024-02-01T17:35:21.504Z 0 000-019 160 160 160 259 259 259 F F F
180012024-02-01T17:35:21.504Z 1 020-039 159 159 159 256 256 256 F F F
180022024-02-01T17:35:21.504Z 2 040-059 167 167 167 270 270 270 F F F
180032024-02-01T17:35:21.504Z 3 060-079 167 167 167 270 270 270 F F F
180042024-02-01T17:35:21.512Z 4 080-099 166 166 166 266 266 266 F F F
180052024-02-01T17:35:21.513Z 5 100-119 167 167 167 270 270 270 F F F
180062024-02-01T17:35:21.513Z 6 120-139 166 166 166 268 268 268 F F F
180072024-02-01T17:35:21.513Z 7 140-159 165 165 165 264 264 264 F F F
180082024-02-01T17:35:21.513Z 8 160-179 166 166 166 266 266 266 F F F
180092024-02-01T17:35:21.513Z 9 180-199 166 166 166 268 268 268 F F F
180102024-02-01T17:35:21.513Z 10 200-219 166 166 166 269 269 269 F F F
180112024-02-01T17:35:21.513Z 11 220-239 166 166 166 269 269 269 F F F
180122024-02-01T17:35:21.513Z 12 240-259 165 165 165 264 264 264 F F F
180132024-02-01T17:35:21.513Z 13 260-279 166 166 166 269 269 269 F F F
180142024-02-01T17:35:21.513Z 14 280-299 166 166 166 269 269 269 F F F
180152024-02-01T17:35:21.513Z 15 300-319 166 166 166 266 266 266 F F F
180162024-02-01T17:35:21.513Z 16 320-339 165 165 165 264 264 264 F F F
180172024-02-01T17:35:21.513Z 17 340-359 164 164 164 263 263 263 F F F
180182024-02-01T17:35:21.514Z 18 360-379 164 164 164 263 263 263 F F F
180192024-02-01T17:35:21.514Z 19 380-399 165 165 165 264 264 264 F F F
180202024-02-01T17:35:21.514Z 20 400-419 166 166 166 265 265 265 F F F
180212024-02-01T17:35:21.514Z 21 420-439 164 164 164 263 263 263 F F F
180222024-02-01T17:35:21.514Z 22 440-459 163 163 163 262 262 262 F F F
180232024-02-01T17:35:21.515Z 23 460-479 166 166 166 269 269 269 F F F
180242024-02-01T17:35:21.515Z 24 480-499 166 166 166 267 267 267 F F F
180252024-02-01T17:35:21.515Z 25 500-519 166 166 166 267 267 267 F F F
180262024-02-01T17:35:21.515Z 26 520-539 162 162 162 261 261 261 F F F
180272024-02-01T17:35:21.515Z 27 540-559 165 165 165 264 264 264 F F F
180282024-02-01T17:35:21.515Z 28 560-579 166 166 166 268 268 268 F F F
180292024-02-01T17:35:21.515Z 29 580-599 163 163 163 262 262 262 F F F
180302024-02-01T17:35:21.515ZMax gen: 167, Max flush: 270
180312024-02-01T17:35:21.515Z
180322024-02-01T17:35:21.515ZBegin loop 84
180332024-02-01T17:35:21.515ZDownstairs to restart: 2
180342024-02-01T17:35:21.942Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821176801Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
180352024-02-01T17:35:21.942Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821672489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
180362024-02-01T17:35:21.942Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821708464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
180372024-02-01T17:35:21.942Z{"msg":"Crucible 07be1e40-7229-4ed0-88a1-8ffe94ac588b has session id: c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821772977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180382024-02-01T17:35:21.943Z{"msg":"Upstairs opts: Upstairs UUID: 07be1e40-7229-4ed0-88a1-8ffe94ac588b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821825618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180392024-02-01T17:35:21.943Z{"msg":"Crucible stats registered with UUID: 07be1e40-7229-4ed0-88a1-8ffe94ac588b","v":0Crucible runtime is spawned
180402024-02-01T17:35:21.944Z,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.821955789Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180412024-02-01T17:35:21.944Z{"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.822027744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
180422024-02-01T17:35:21.944Z{"msg":"07be1e40-7229-4ed0-88a1-8ffe94ac588b active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.822259241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180432024-02-01T17:35:21.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.8223829Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180442024-02-01T17:35:21.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.822548007Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180452024-02-01T17:35:21.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82261164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180462024-02-01T17:35:21.944Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82266466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180472024-02-01T17:35:21.944Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82273861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180482024-02-01T17:35:21.945Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.822900471Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180492024-02-01T17:35:21.946Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.822970982Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180502024-02-01T17:35:21.946Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.823054526Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180512024-02-01T17:35:21.946Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.823094106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180522024-02-01T17:35:21.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.823148114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180532024-02-01T17:35:21.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.823173197Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180542024-02-01T17:35:21.946Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82319597Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"io task","client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180552024-02-01T17:35:21.946Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.824348698Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180562024-02-01T17:35:21.946Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.824457777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180572024-02-01T17:35:21.946Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.824534205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180582024-02-01T17:35:21.948Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82457978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180592024-02-01T17:35:21.948Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.824657334Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180602024-02-01T17:35:21.948Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.824702695Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180612024-02-01T17:35:21.948Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.825292875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180622024-02-01T17:35:21.948Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.825393529Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180632024-02-01T17:35:21.948Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.825527466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180642024-02-01T17:35:21.948Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826077239Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180652024-02-01T17:35:21.948Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826147233Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180662024-02-01T17:35:21.949Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826174143Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180672024-02-01T17:35:21.949Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826385227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180682024-02-01T17:35:21.954Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826496335Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180692024-02-01T17:35:21.954Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826557325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180702024-02-01T17:35:21.955Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826615673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180712024-02-01T17:35:21.955Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826660186Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180722024-02-01T17:35:21.955Z{"msg":"[0]R flush_numbers[0..12]: [259, 256, 270, 270, 266, 270, 268, 264, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826714705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180732024-02-01T17:35:21.955Z{"msg":"[0]R generation[0..12]: [160, 159, 167, 167, 166, 167, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826759999Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180742024-02-01T17:35:21.955Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82681468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180752024-02-01T17:35:21.955Z{"msg":"[1]R flush_numbers[0..12]: [259, 256, 270, 270, 266, 270, 268, 264, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826859518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180762024-02-01T17:35:21.955ZWait for a query_work_queue command to finish before sending IO
180772024-02-01T17:35:21.955Z{"msg":"[1]R generation[0..12]: [160, 159, 167, 167, 166, 167, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82691724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180782024-02-01T17:35:21.958Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.826962049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180792024-02-01T17:35:21.958Z{"msg":"[2]R flush_numbers[0..12]: [259, 256, 270, 270, 266, 270, 268, 264, 266, 268, 269, 269]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827167843Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180802024-02-01T17:35:21.958Z{"msg":"[2]R generation[0..12]: [160, 159, 167, 167, 166, 167, 166, 165, 166, 166, 166, 166]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827245391Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180812024-02-01T17:35:21.959Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827275367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180822024-02-01T17:35:21.959Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827338245Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180832024-02-01T17:35:21.959Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827369329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180842024-02-01T17:35:21.959Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827419297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180852024-02-01T17:35:21.959Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827459413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"":"downstairs","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180862024-02-01T17:35:21.959Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827510642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180872024-02-01T17:35:21.959Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82754109Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"0","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180882024-02-01T17:35:21.959Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.82760722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"1","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180892024-02-01T17:35:21.959Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
180902024-02-01T17:35:21.959Z"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827641316Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"client":"2","":"downstairs","session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180912024-02-01T17:35:21.959Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827698666Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180922024-02-01T17:35:21.960Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827728495Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180932024-02-01T17:35:21.960Z{"msg":"07be1e40-7229-4ed0-88a1-8ffe94ac588b is now active with session: c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827771254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180942024-02-01T17:35:21.960Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827793931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804,"session_id":"c62ece3c-4c0b-4c9b-9c8a-88205c8e2bdd"}
180952024-02-01T17:35:21.960Z{"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:20.827815188Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
180962024-02-01T17:35:21.960ZLoading write count information from file "/tmp/test_repair_verify.data"
180972024-02-01T17:35:21.960ZRun Repair workload
180982024-02-01T17:35:21.960Z01/30 Flush
180992024-02-01T17:35:21.960Z02/30 Write block 544 len 4096 data: 18
181002024-02-01T17:35:21.960Z03/30 Write block 42 len 24576 data: 12 14 15 14 15 13
181012024-02-01T17:35:21.960Z04/30 Flush
181022024-02-01T17:35:21.960Z05/30 Flush
181032024-02-01T17:35:21.960Z06/30 Write block 251 len 40960 data: 10 12 10 12 14 14 13 16 16 14
181042024-02-01T17:35:21.960Z07/30 Read block 340 len 40960
181052024-02-01T17:35:21.960Z08/30 Read block 559 len 28672
181062024-02-01T17:35:21.960Z09/30 Write block 449 len 20480 data: 10 8 6 8 8
181072024-02-01T17:35:21.960Z10/30 Write block 46 len 40960 data: 16 14 16 15 16 17 15 16 18 17
181082024-02-01T17:35:21.960Z11/30 Read block 386 len 40960
181092024-02-01T17:35:21.960Z12/30 Read block 168 len 8192
181102024-02-01T17:35:21.960Z13/30 Write block 430 len 24576 data: 9 9 15 18 16 18
181112024-02-01T17:35:21.960Z14/30 Flush
181122024-02-01T17:35:21.960Z15/30 Write block 230 len 28672 data: 10 11 12 10 10 9 10
181132024-02-01T17:35:21.960Z16/30 Read block 196 len 28672
181142024-02-01T17:35:21.961Z17/30 Write block 344 len 4096 data: 13
181152024-02-01T17:35:21.961Z18/30 Read block 360 len 12288
181162024-02-01T17:35:21.961Z19/30 Write block 343 len 8192 data: 14 14
181172024-02-01T17:35:21.961Z20/30 Read block 560 len 20480
181182024-02-01T17:35:21.961Z21/30 Read block 535 len 16384
181192024-02-01T17:35:21.961Z22/30 Write block 209 len 4096 data: 16
181202024-02-01T17:35:21.961Z23/30 Read block 440 len 4096
181212024-02-01T17:35:21.961Z24/30 Flush
181222024-02-01T17:35:21.961Z25/30 Write block 73 len 40960 data: 17 17 15 11 9 9 8 8 6 7
181232024-02-01T17:35:21.961Z26/30 Read block 231 len 40960
181242024-02-01T17:35:21.961Z27/30 Write block 147 len 40960 data: 12 16 16 12 11 13 13 11 14 11
181252024-02-01T17:35:21.961Z28/30 Read block 545 len 4096
181262024-02-01T17:35:21.961Z29/30 Write block 489 len 32768 data: 20 19 17 19 15 13 16 14
181272024-02-01T17:35:21.961Z30/30 Read block 234 len 40960
181282024-02-01T17:35:21.961Z----------------------------------------------------------------
181292024-02-01T17:35:21.961Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:25
181302024-02-01T17:35:21.961ZStates: Active Active Active
181312024-02-01T17:35:21.961ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
181322024-02-01T17:35:21.961Z 6 Acked 1005 Write 10 Done Done Sent false
181332024-02-01T17:35:21.961Z 7 Acked 1006 Read 10 Done Done Sent false
181342024-02-01T17:35:21.975Z 8 Acked 1007 Read 7 Done Done Sent false
181352024-02-01T17:35:21.976Z 9 Acked 1008 Write 5 Done Done Sent false
181362024-02-01T17:35:21.976Z 10 Acked 1009 Write 10 Done Done Sent false
181372024-02-01T17:35:21.976Z 11 Acked 1010 Read 10 Done Done Sent false
181382024-02-01T17:35:21.976Z 12 Acked 1011 Read 2 Done Done Sent false
181392024-02-01T17:35:21.976Z 13 Acked 1012 Write 6 Done Done Sent false
181402024-02-01T17:35:21.976Z 14 Acked 1013 Flush 0 Done Done Sent false
181412024-02-01T17:35:21.976Z 15 Acked 1014 Write 7 Done Done Sent false
181422024-02-01T17:35:21.976Z 16 Acked 1015 Read 7 Done Done Sent false
181432024-02-01T17:35:21.976Z 17 Acked 1016 Write 1 Done Done Sent false
181442024-02-01T17:35:21.976Z 18 Acked 1017 Read 3 Done Done Sent false
181452024-02-01T17:35:21.976Z 19 Acked 1018 Write 2 Done Done Sent false
181462024-02-01T17:35:21.976Z 20 Acked 1019 Read 5 Done Done Sent false
181472024-02-01T17:35:21.976Z 21 Acked 1020 Read 4 Done Done Sent false
181482024-02-01T17:35:21.976Z 22 Acked 1021 Write 1 Done Done Sent false
181492024-02-01T17:35:21.976Z 23 Acked 1022 Read 1 Done Done Sent false
181502024-02-01T17:35:21.977Z 24 Acked 1023 Flush 0 Done Done Sent false
181512024-02-01T17:35:21.977Z 25 Acked 1024 Write 10 Done Done Sent false
181522024-02-01T17:35:21.977Z 26 Acked 1025 Read 10 Done Done Sent false
181532024-02-01T17:35:21.977Z 27 Acked 1026 Write 10 Done Done Sent false
181542024-02-01T17:35:21.977Z 28 Acked 1027 Read 1 Done Done Sent false
181552024-02-01T17:35:21.977Z 29 Acked 1028 Write 8 Done Done Sent false
181562024-02-01T17:35:21.977Z 30 Acked 1029 Read 10 Sent Done Sent false
181572024-02-01T17:35:21.977Z STATES DS:0 DS:1 DS:2 TOTAL
181582024-02-01T17:35:21.977Z New 0 0 0 0
181592024-02-01T17:35:21.977Z Sent 1 0 25 26
181602024-02-01T17:35:21.977Z Done 24 25 0 49
181612024-02-01T17:35:21.977Z Skipped 0 0 0 0
181622024-02-01T17:35:21.977Z Error 0 0 0 0
181632024-02-01T17:35:21.977ZLast Flush: 1023 1023 1004
181642024-02-01T17:35:21.977ZDownstairs last five completed: 1004 1003 1002 1001 1000
181652024-02-01T17:35:21.977ZUpstairs last five completed: 30 29 28 27 26
181662024-02-01T17:35:21.978Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:20.876878628Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1804}
181672024-02-01T17:35:21.978ZWrote out file "/tmp/test_repair_verify.data"
181682024-02-01T17:35:21.978Z
181692024-02-01T17:35:21.978ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
181702024-02-01T17:35:21.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
181712024-02-01T17:35:21.978Z 0 000-019 160 160 160 259 259 259 F F F
181722024-02-01T17:35:21.978Z 1 020-039 159 159 159 256 256 256 F F F
181732024-02-01T17:35:21.978Z 2 040-059 168 168 168 274 274 272 F F F <---
181742024-02-01T17:35:21.978Z 3 060-079 167 167 167 270 270 270 T T F <---
181752024-02-01T17:35:21.978Z 4 080-099 166 166 166 266 266 266 T T F <---
181762024-02-01T17:35:21.978Z 5 100-119 167 167 167 270 270 270 F F F
181772024-02-01T17:35:21.978Z 6 120-139 166 166 166 268 268 268 F F F
181782024-02-01T17:35:21.978Z 7 140-159 165 165 165 264 264 264 T T F <---
181792024-02-01T17:35:21.979Z 8 160-179 166 166 166 266 266 266 F F F
181802024-02-01T17:35:21.979Z 9 180-199 166 166 166 268 268 268 F F F
181812024-02-01T17:35:21.979Z 10 200-219 168 168 166 275 275 269 F F F <---
181822024-02-01T17:35:21.979Z 11 220-239 168 168 166 275 275 269 F F F <---
181832024-02-01T17:35:21.979Z 12 240-259 168 168 165 274 274 264 F F F <---
181842024-02-01T17:35:21.979Z 13 260-279 168 168 166 274 274 269 F F F <---
181852024-02-01T17:35:21.979Z 14 280-299 166 166 166 269 269 269 F F F
181862024-02-01T17:35:21.979Z 15 300-319 166 166 166 266 266 266 F F F
181872024-02-01T17:35:21.979Z 16 320-339 165 165 165 264 264 264 F F F
181882024-02-01T17:35:21.979Z 17 340-359 168 168 164 275 275 263 F F F <---
181892024-02-01T17:35:21.979Z 18 360-379 164 164 164 263 263 263 F F F
181902024-02-01T17:35:21.979Z 19 380-399 165 165 165 264 264 264 F F F
181912024-02-01T17:35:21.979Z 20 400-419 166 166 166 265 265 265 F F F
181922024-02-01T17:35:21.979Z 21 420-439 168 168 164 274 274 263 F F F <---
181932024-02-01T17:35:21.980Z 22 440-459 168 168 163 274 274 262 F F F <---
181942024-02-01T17:35:21.980Z 23 460-479 166 166 166 269 269 269 F F F
181952024-02-01T17:35:21.980Z 24 480-499 166 166 166 267 267 267 T T F <---
181962024-02-01T17:35:21.980Z 25 500-519 166 166 166 267 267 267 F F F
181972024-02-01T17:35:21.980Z 26 520-539 162 162 162 261 261 261 F F F
181982024-02-01T17:35:21.981Z 27 540-559 168 168 168 272 272 272 F F F
181992024-02-01T17:35:21.981Z 28 560-579 166 166 166 268 268 268 F F F
182002024-02-01T17:35:21.981Z 29 580-599 163 163 163 262 262 262 F F F
182012024-02-01T17:35:21.981ZMax gen: 168, Max flush: 275
182022024-02-01T17:35:21.981ZError: Difference in extent metadata found!
182032024-02-01T17:35:21.981ZA Difference in extent metadata is expected here
182042024-02-01T17:35:21.981ZOn loop 84
182052024-02-01T17:35:21.981Z
182062024-02-01T17:35:21.981ZVerifying data now
182072024-02-01T17:35:22.679ZNo change after verify
182082024-02-01T17:35:22.679ZLoop: 84 Downstairs dump after verify (and repair):
182092024-02-01T17:35:22.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182102024-02-01T17:35:22.705Z 0 000-019 160 160 160 259 259 259 F F F
182112024-02-01T17:35:22.705Z 1 020-039 159 159 159 256 256 256 F F F
182122024-02-01T17:35:22.705Z 2 040-059 168 168 168 274 274 274 F F F
182132024-02-01T17:35:22.705Z 3 060-079 169 169 169 276 276 276 F F F
182142024-02-01T17:35:22.706Z 4 080-099 169 169 169 276 276 276 F F F
182152024-02-01T17:35:22.706Z 5 100-119 167 167 167 270 270 270 F F F
182162024-02-01T17:35:22.706Z 6 120-139 166 166 166 268 268 268 F F F
182172024-02-01T17:35:22.706Z 7 140-159 169 169 169 276 276 276 F F F
182182024-02-01T17:35:22.719Z 8 160-179 166 166 166 266 266 266 F F F
182192024-02-01T17:35:22.719Z 9 180-199 166 166 166 268 268 268 F F F
182202024-02-01T17:35:22.719Z 10 200-219 168 168 168 275 275 275 F F F
182212024-02-01T17:35:22.719Z 11 220-239 168 168 168 275 275 275 F F F
182222024-02-01T17:35:22.719Z 12 240-259 168 168 168 274 274 274 F F F
182232024-02-01T17:35:22.719Z 13 260-279 168 168 168 274 274 274 F F F
182242024-02-01T17:35:22.720Z 14 280-299 166 166 166 269 269 269 F F F
182252024-02-01T17:35:22.720Z 15 300-319 166 166 166 266 266 266 F F F
182262024-02-01T17:35:22.720Z 16 320-339 165 165 165 264 264 264 F F F
182272024-02-01T17:35:22.720Z 17 340-359 168 168 168 275 275 275 F F F
182282024-02-01T17:35:22.720Z 18 360-379 164 164 164 263 263 263 F F F
182292024-02-01T17:35:22.720Z 19 380-399 165 165 165 264 264 264 F F F
182302024-02-01T17:35:22.720Z 20 400-419 166 166 166 265 265 265 F F F
182312024-02-01T17:35:22.721Z 21 420-439 168 168 168 274 274 274 F F F
182322024-02-01T17:35:22.721Z 22 440-459 168 168 168 274 274 274 F F F
182332024-02-01T17:35:22.721Z 23 460-479 166 166 166 269 269 269 F F F
182342024-02-01T17:35:22.721Z 24 480-499 169 169 169 276 276 276 F F F
182352024-02-01T17:35:22.722Z 25 500-519 166 166 166 267 267 267 F F F
182362024-02-01T17:35:22.722Z 26 520-539 162 162 162 261 261 261 F F F
182372024-02-01T17:35:22.722Z 27 540-559 168 168 168 272 272 272 F F F
182382024-02-01T17:35:22.722Z 28 560-579 166 166 166 268 268 268 F F F
182392024-02-01T17:35:22.723Z 29 580-599 163 163 163 262 262 262 F F F
182402024-02-01T17:35:22.723ZMax gen: 169, Max flush: 276
182412024-02-01T17:35:22.723Z
182422024-02-01T17:35:22.723ZBegin loop 85
182432024-02-01T17:35:22.723ZDownstairs to restart: 2
182442024-02-01T17:35:22.745Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.01907625Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
182452024-02-01T17:35:22.763Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019589725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
182462024-02-01T17:35:22.763ZCrucible runtime is spawned
182472024-02-01T17:35:22.763Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019750804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
182482024-02-01T17:35:22.764Z{"msg":"Crucible 3306a593-d115-4cbd-8d55-eafb2d37b64e has session id: dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019871784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182492024-02-01T17:35:22.764Z{"msg":"Upstairs opts: Upstairs UUID: 3306a593-d115-4cbd-8d55-eafb2d37b64e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019920832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182502024-02-01T17:35:22.764Z{"msg":"Crucible stats registered with UUID: 3306a593-d115-4cbd-8d55-eafb2d37b64e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019945572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182512024-02-01T17:35:22.764Z{"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.019990851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
182522024-02-01T17:35:22.764Z{"msg":"3306a593-d115-4cbd-8d55-eafb2d37b64e active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020033209Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182532024-02-01T17:35:22.764Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020080105Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182542024-02-01T17:35:22.764Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020127712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182552024-02-01T17:35:22.764Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020174104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182562024-02-01T17:35:22.766Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020214433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182572024-02-01T17:35:22.766Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020257645Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182582024-02-01T17:35:22.766Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020297831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182592024-02-01T17:35:22.766Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02036458Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182602024-02-01T17:35:22.766Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020394097Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182612024-02-01T17:35:22.767Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020483811Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182622024-02-01T17:35:22.767Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020694133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182632024-02-01T17:35:22.767Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020782875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182642024-02-01T17:35:22.767Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.020827827Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"io task","client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182652024-02-01T17:35:22.767Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.021869472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182662024-02-01T17:35:22.768Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.021992269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182672024-02-01T17:35:22.769Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.022134573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182682024-02-01T17:35:22.769Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.022242533Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182692024-02-01T17:35:22.769Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02297694Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182702024-02-01T17:35:22.769Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023343712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182712024-02-01T17:35:22.769Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023481053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182722024-02-01T17:35:22.771Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023533195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182732024-02-01T17:35:22.771Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023734486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182742024-02-01T17:35:22.771Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023791854Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182752024-02-01T17:35:22.771Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023817367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182762024-02-01T17:35:22.772Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.023989222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182772024-02-01T17:35:22.773Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.024072794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182782024-02-01T17:35:22.773Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.024141469Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182792024-02-01T17:35:22.773Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.024501741Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182802024-02-01T17:35:22.773Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025194365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182812024-02-01T17:35:22.773Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025257239Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182822024-02-01T17:35:22.773Z{"msg":"[0]R flush_numbers[0..12]: [259, 256, 274, 276, 276, 270, 268, 276, 266, 268, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02528601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182832024-02-01T17:35:22.773Z{"msg":"[0]R generation[0..12]: [160, 159, 168, 169, 169, 167, 166, 169, 166, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025346822Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182842024-02-01T17:35:22.774Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02538527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182852024-02-01T17:35:22.774Z{"msg":"[1]R flush_numbers[0..12]: [259, 256, 274, 276, 276, 270, 268, 276, 266, 268, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025438946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182862024-02-01T17:35:22.775Z{"msg":"[1]R generation[0..12]: [160, 159, 168, 169, 169, 167, 166, 169, 166, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025469312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182872024-02-01T17:35:22.775Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
182882024-02-01T17:35:22.776Z,"time":"2024-02-01T17:35:22.025543325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182892024-02-01T17:35:22.776Z{"msg":"[2]R flush_numbers[0..12]: [259, 256, 274, 276, 276, 270, 268, 276, 266, 268, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025610089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182902024-02-01T17:35:22.776Z{"msg":"[2]R generation[0..12]: [160, 159, 168, 169, 169, 167, 166, 169, 166, 166, 168, 168]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025643976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182912024-02-01T17:35:22.776Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025700176Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182922024-02-01T17:35:22.776Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025726229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182932024-02-01T17:35:22.776Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025796423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182942024-02-01T17:35:22.777Z{"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025831702Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182952024-02-01T17:35:22.777Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025892325Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"":"downstairs","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182962024-02-01T17:35:22.777Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.025918508Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182972024-02-01T17:35:22.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02597264Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"0","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182982024-02-01T17:35:22.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02600466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"1","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
182992024-02-01T17:35:22.777Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.02606743Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"client":"2","":"downstairs","session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
183002024-02-01T17:35:22.778Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.026096984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
183012024-02-01T17:35:22.778Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.026152933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
183022024-02-01T17:35:22.778Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
183032024-02-01T17:35:22.778Z3306a593-d115-4cbd-8d55-eafb2d37b64e is now active with session: dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.026183882Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
183042024-02-01T17:35:22.778Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.026237236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812,"session_id":"dbbcc6d2-8999-4869-abd1-fd7a5f5f09cd"}
183052024-02-01T17:35:22.778Z{"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:22.026267474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
183062024-02-01T17:35:22.779ZLoading write count information from file "/tmp/test_repair_verify.data"
183072024-02-01T17:35:22.779ZRun Repair workload
183082024-02-01T17:35:22.780Z01/30 Write block 405 len 16384 data: 10 10 12 15
183092024-02-01T17:35:22.780Z02/30 Read block 547 len 28672
183102024-02-01T17:35:22.780Z03/30 Flush
183112024-02-01T17:35:22.780Z04/30 Write block 29 len 4096 data: 6
183122024-02-01T17:35:22.781Z05/30 Read block 464 len 8192
183132024-02-01T17:35:22.781Z06/30 Write block 26 len 20480 data: 8 6 5 7 8
183142024-02-01T17:35:22.781Z07/30 Write block 102 len 24576 data: 16 14 15 18 18 19
183152024-02-01T17:35:22.781Z08/30 Write block 417 len 36864 data: 12 11 15 12 12 12 11 11 14
183162024-02-01T17:35:22.788Z09/30 Flush
183172024-02-01T17:35:22.788Z10/30 Write block 223 len 32768 data: 17 16 11 9 7 7 11 11
183182024-02-01T17:35:22.788Z11/30 Write block 282 len 16384 data: 12 14 15 18
183192024-02-01T17:35:22.789Z12/30 Write block 475 len 20480 data: 9 8 7 9 8
183202024-02-01T17:35:22.789Z13/30 Write block 452 len 32768 data: 9 9 9 9 13 18 18 19
183212024-02-01T17:35:22.789Z14/30 Write block 197 len 8192 data: 11 8
183222024-02-01T17:35:22.789Z15/30 Read block 376 len 24576
183232024-02-01T17:35:22.789Z16/30 Read block 204 len 32768
183242024-02-01T17:35:22.790Z17/30 Write block 400 len 36864 data: 9 12 11 14 13 11 11 13 16
183252024-02-01T17:35:22.790Z18/30 Write block 253 len 24576 data: 11 13 15 15 14 17
183262024-02-01T17:35:22.790Z19/30 Read block 265 len 24576
183272024-02-01T17:35:22.791Z20/30 Read block 115 len 12288
183282024-02-01T17:35:22.791Z21/30 Read block 186 len 8192
183292024-02-01T17:35:22.791Z22/30 Write block 518 len 8192 data: 12 10
183302024-02-01T17:35:22.791Z23/30 Write block 377 len 20480 data: 14 11 13 13 15
183312024-02-01T17:35:22.791Z24/30 Write block 79 len 36864 data: 9 9 7 8 9 9 9 9 8
183322024-02-01T17:35:22.792Z25/30 Write block 546 len 32768 data: 15 16 11 9 9 11 12 10
183332024-02-01T17:35:22.793Z26/30 Read block 258 len 28672
183342024-02-01T17:35:22.793Z27/30 Read block 409 len 12288
183352024-02-01T17:35:22.793Z28/30 Write block 9 len 8192 data: 11 13
183362024-02-01T17:35:22.793Z29/30 Read block 356 len 28672
183372024-02-01T17:35:22.816Z30/30 Read block 125 len 40960
183382024-02-01T17:35:22.817Z----------------------------------------------------------------
183392024-02-01T17:35:22.817Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
183402024-02-01T17:35:22.817ZStates: Active Active Active
183412024-02-01T17:35:22.817ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
183422024-02-01T17:35:22.818Z 1 Acked 1000 Write 4 Done Done Sent false
183432024-02-01T17:35:22.818Z 2 Acked 1001 Read 7 Done Done Sent false
183442024-02-01T17:35:22.818Z 3 Acked 1002 Flush 0 Done Done Sent false
183452024-02-01T17:35:22.818Z 4 Acked 1003 Write 1 Done Done Sent false
183462024-02-01T17:35:22.818Z 5 Acked 1004 Read 2 Done Done Sent false
183472024-02-01T17:35:22.819Z 6 Acked 1005 Write 5 Done Done Sent false
183482024-02-01T17:35:22.819Z 7 Acked 1006 Write 6 Done Done Sent false
183492024-02-01T17:35:22.819Z 8 Acked 1007 Write 9 Done Done Sent false
183502024-02-01T17:35:22.819Z 9 Acked 1008 Flush 0 Done Done Sent false
183512024-02-01T17:35:22.819Z 10 Acked 1009 Write 8 Done Done Sent false
183522024-02-01T17:35:22.819Z 11 Acked 1010 Write 4 Done Done Sent false
183532024-02-01T17:35:22.820Z 12 Acked 1011 Write 5 Done Done Sent false
183542024-02-01T17:35:22.820Z 13 Acked 1012 Write 8 Done Done Sent false
183552024-02-01T17:35:22.820Z 14 Acked 1013 Write 2 Done Done Sent false
183562024-02-01T17:35:22.820Z 15 Acked 1014 Read 6 Done Done Sent false
183572024-02-01T17:35:22.828Z 16 Acked 1015 Read 8 Done Done Sent false
183582024-02-01T17:35:22.828Z 17 Acked 1016 Write 9 Done Done Sent false
183592024-02-01T17:35:22.828Z 18 Acked 1017 Write 6 Done Done Sent false
183602024-02-01T17:35:22.828Z 19 Acked 1018 Read 6 Done Done Sent false
183612024-02-01T17:35:22.828Z 20 Acked 1019 Read 3 Done Done Sent false
183622024-02-01T17:35:22.829Z 21 Acked 1020 Read 2 Done Done Sent false
183632024-02-01T17:35:22.829Z 22 Acked 1021 Write 2 Done Done Sent false
183642024-02-01T17:35:22.829Z 23 Acked 1022 Write 5 Done Done Sent false
183652024-02-01T17:35:22.830Z 24 Acked 1023 Write 9 Done Done Sent false
183662024-02-01T17:35:22.830Z 25 Acked 1024 Write 8 Done Done Sent false
183672024-02-01T17:35:22.830Z 26 Acked 1025 Read 7 Done Done Sent false
183682024-02-01T17:35:22.830Z 27 Acked 1026 Read 3 Done Done Sent false
183692024-02-01T17:35:22.830Z 28 Acked 1027 Write 2 Done Done Sent false
183702024-02-01T17:35:22.831Z 29 Acked 1028 Read 7 Done Done Sent false
183712024-02-01T17:35:22.831Z 30 Acked 1029 Read 10 Done Done Sent false
183722024-02-01T17:35:22.831Z STATES DS:0 DS:1 DS:2 TOTAL
183732024-02-01T17:35:22.831Z New 0 0 0 0
183742024-02-01T17:35:22.831Z Sent 0 0 30 30
183752024-02-01T17:35:22.831Z Done 30 30 0 60
183762024-02-01T17:35:22.831Z Skipped 0 0 0 0
183772024-02-01T17:35:22.831Z Error 0 0 0 0
183782024-02-01T17:35:22.831ZLast Flush: 1008 1008 0
183792024-02-01T17:35:22.831ZDownstairs last five completed:
183802024-02-01T17:35:22.831ZUpstairs last five completed: 30 29 28 27 26
183812024-02-01T17:35:22.832Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:22.073053599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1812}
183822024-02-01T17:35:22.833ZWrote out file "/tmp/test_repair_verify.data"
183832024-02-01T17:35:22.833Z
183842024-02-01T17:35:22.850ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
183852024-02-01T17:35:22.850ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183862024-02-01T17:35:22.850Z 0 000-019 160 160 160 259 259 259 T T F <---
183872024-02-01T17:35:22.850Z 1 020-039 170 170 159 278 278 256 F F F <---
183882024-02-01T17:35:22.850Z 2 040-059 168 168 168 274 274 274 F F F
183892024-02-01T17:35:22.851Z 3 060-079 169 169 169 276 276 276 T T F <---
183902024-02-01T17:35:22.851Z 4 080-099 169 169 169 276 276 276 T T F <---
183912024-02-01T17:35:22.851Z 5 100-119 170 170 167 278 278 270 F F F <---
183922024-02-01T17:35:22.851Z 6 120-139 166 166 166 268 268 268 F F F
183932024-02-01T17:35:22.851Z 7 140-159 169 169 169 276 276 276 F F F
183942024-02-01T17:35:22.851Z 8 160-179 166 166 166 266 266 266 F F F
183952024-02-01T17:35:22.851Z 9 180-199 166 166 166 268 268 268 T T F <---
183962024-02-01T17:35:22.851Z 10 200-219 168 168 168 275 275 275 F F F
183972024-02-01T17:35:22.851Z 11 220-239 168 168 168 275 275 275 T T F <---
183982024-02-01T17:35:22.851Z 12 240-259 168 168 168 274 274 274 T T F <---
183992024-02-01T17:35:22.851Z 13 260-279 168 168 168 274 274 274 F F F
184002024-02-01T17:35:22.851Z 14 280-299 166 166 166 269 269 269 T T F <---
184012024-02-01T17:35:22.851Z 15 300-319 166 166 166 266 266 266 F F F
184022024-02-01T17:35:22.852Z 16 320-339 165 165 165 264 264 264 F F F
184032024-02-01T17:35:22.852Z 17 340-359 168 168 168 275 275 275 F F F
184042024-02-01T17:35:22.853Z 18 360-379 164 164 164 263 263 263 T T F <---
184052024-02-01T17:35:22.853Z 19 380-399 165 165 165 264 264 264 T T F <---
184062024-02-01T17:35:22.853Z 20 400-419 170 170 166 278 278 265 T T F <---
184072024-02-01T17:35:22.854Z 21 420-439 170 170 168 278 278 274 F F F <---
184082024-02-01T17:35:22.854Z 22 440-459 168 168 168 274 274 274 T T F <---
184092024-02-01T17:35:22.854Z 23 460-479 166 166 166 269 269 269 T T F <---
184102024-02-01T17:35:22.854Z 24 480-499 169 169 169 276 276 276 F F F
184112024-02-01T17:35:22.854Z 25 500-519 166 166 166 267 267 267 T T F <---
184122024-02-01T17:35:22.854Z 26 520-539 162 162 162 261 261 261 F F F
184132024-02-01T17:35:22.855Z 27 540-559 168 168 168 272 272 272 T T F <---
184142024-02-01T17:35:22.855Z 28 560-579 166 166 166 268 268 268 F F F
184152024-02-01T17:35:22.855Z 29 580-599 163 163 163 262 262 262 F F F
184162024-02-01T17:35:22.855ZMax gen: 170, Max flush: 278
184172024-02-01T17:35:22.855ZError: Difference in extent metadata found!
184182024-02-01T17:35:22.855ZA Difference in extent metadata is expected here
184192024-02-01T17:35:22.855ZOn loop 85
184202024-02-01T17:35:22.855Z
184212024-02-01T17:35:22.860ZVerifying data now
184222024-02-01T17:35:28.278ZNo change after verify
184232024-02-01T17:35:28.279ZLoop: 85 Downstairs dump after verify (and repair):
184242024-02-01T17:35:28.309ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
184252024-02-01T17:35:28.309Z 0 000-019 171 171 171 279 279 279 F F F
184262024-02-01T17:35:28.336Z 1 020-039 170 170 170 278 278 278 F F F
184272024-02-01T17:35:28.336Z 2 040-059 168 168 168 274 274 274 F F F
184282024-02-01T17:35:28.336Z 3 060-079 171 171 171 279 279 279 F F F
184292024-02-01T17:35:28.336Z 4 080-099 171 171 171 279 279 279 F F F
184302024-02-01T17:35:28.336Z 5 100-119 170 170 170 278 278 278 F F F
184312024-02-01T17:35:28.336Z 6 120-139 166 166 166 268 268 268 F F F
184322024-02-01T17:35:28.336Z 7 140-159 169 169 169 276 276 276 F F F
184332024-02-01T17:35:28.336Z 8 160-179 166 166 166 266 266 266 F F F
184342024-02-01T17:35:28.336Z 9 180-199 171 171 171 279 279 279 F F F
184352024-02-01T17:35:28.336Z 10 200-219 168 168 168 275 275 275 F F F
184362024-02-01T17:35:28.336Z 11 220-239 171 171 171 279 279 279 F F F
184372024-02-01T17:35:28.336Z 12 240-259 171 171 171 279 279 279 F F F
184382024-02-01T17:35:28.337Z 13 260-279 168 168 168 274 274 274 F F F
184392024-02-01T17:35:28.337Z 14 280-299 171 171 171 279 279 279 F F F
184402024-02-01T17:35:28.337Z 15 300-319 166 166 166 266 266 266 F F F
184412024-02-01T17:35:28.337Z 16 320-339 165 165 165 264 264 264 F F F
184422024-02-01T17:35:28.337Z 17 340-359 168 168 168 275 275 275 F F F
184432024-02-01T17:35:28.337Z 18 360-379 171 171 171 279 279 279 F F F
184442024-02-01T17:35:28.337Z 19 380-399 171 171 171 279 279 279 F F F
184452024-02-01T17:35:28.337Z 20 400-419 171 171 171 279 279 279 F F F
184462024-02-01T17:35:28.337Z 21 420-439 170 170 170 278 278 278 F F F
184472024-02-01T17:35:28.337Z 22 440-459 171 171 171 279 279 279 F F F
184482024-02-01T17:35:28.337Z 23 460-479 171 171 171 279 279 279 F F F
184492024-02-01T17:35:28.337Z 24 480-499 169 169 169 276 276 276 F F F
184502024-02-01T17:35:28.338Z 25 500-519 171 171 171 279 279 279 F F F
184512024-02-01T17:35:28.338Z 26 520-539 162 162 162 261 261 261 F F F
184522024-02-01T17:35:28.338Z 27 540-559 171 171 171 279 279 279 F F F
184532024-02-01T17:35:28.338Z 28 560-579 166 166 166 268 268 268 F F F
184542024-02-01T17:35:28.338Z 29 580-599 163 163 163 262 262 262 F F F
184552024-02-01T17:35:28.338ZMax gen: 171, Max flush: 279
184562024-02-01T17:35:28.338Z
184572024-02-01T17:35:28.338ZBegin loop 86
184582024-02-01T17:35:28.338ZDownstairs to restart: 1
184592024-02-01T17:35:28.349Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.611769066Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
184602024-02-01T17:35:28.349Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612319927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
184612024-02-01T17:35:28.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612383666Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
184622024-02-01T17:35:28.349Z{"msg":"Crucible 5cfc5ea6-acba-4536-9f5d-8584981cb345 has session id: fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
184632024-02-01T17:35:28.349Z2024-02-01T17:35:27.612424805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184642024-02-01T17:35:28.349Z{"msg":"Upstairs opts: Upstairs UUID: 5cfc5ea6-acba-4536-9f5d-8584981cb345, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612553029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184652024-02-01T17:35:28.349Z{"msg":"Crucible stats registered with UUID: 5cfc5ea6-acba-4536-9f5d-8584981cb345","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61258357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184662024-02-01T17:35:28.349Z{"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612633714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
184672024-02-01T17:35:28.350Z{"msg":"5cfc5ea6-acba-4536-9f5d-8584981cb345 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61277669Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184682024-02-01T17:35:28.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612829523Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184692024-02-01T17:35:28.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612863938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184702024-02-01T17:35:28.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612933896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184712024-02-01T17:35:28.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.612973166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184722024-02-01T17:35:28.350Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613014154Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184732024-02-01T17:35:28.350Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61308938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184742024-02-01T17:35:28.350Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613134086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184752024-02-01T17:35:28.350Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613176749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184762024-02-01T17:35:28.350Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613220647Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184772024-02-01T17:35:28.350Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613425413Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184782024-02-01T17:35:28.350Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613515551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184792024-02-01T17:35:28.350Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.613561388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"io task","client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184802024-02-01T17:35:28.350Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.614794581Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184812024-02-01T17:35:28.350Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.614840702Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184822024-02-01T17:35:28.350Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.614865984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184832024-02-01T17:35:28.350Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.614948278Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184842024-02-01T17:35:28.350Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.614983818Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184852024-02-01T17:35:28.351Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.615013571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184862024-02-01T17:35:28.353Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.615619554Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184872024-02-01T17:35:28.353Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.615951064Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184882024-02-01T17:35:28.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616355067Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184892024-02-01T17:35:28.353Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616477202Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184902024-02-01T17:35:28.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616538086Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184912024-02-01T17:35:28.353Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616659312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184922024-02-01T17:35:28.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616817315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184932024-02-01T17:35:28.353Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61690488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184942024-02-01T17:35:28.353Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.616999496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184952024-02-01T17:35:28.353Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617204847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184962024-02-01T17:35:28.354Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.6172749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184972024-02-01T17:35:28.354Z{"msg":"[0]R flush_numbers[0..12]: [279, 278, 274, 279, 279, 278, 268, 276, 266, 279, 275, 279]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617351783Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184982024-02-01T17:35:28.354Z{"msg":"[0]R generation[0..12]: [171, 170, 168, 171, 171, 170, 166, 169, 166, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617396166Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
184992024-02-01T17:35:28.354Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617422493Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185002024-02-01T17:35:28.354Z{"msg":"[1]R flush_numbers[0..12]: [279, 278, 274, 279, 279, 278, 268, 276, 266, 279, 275, 279]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617464016Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185012024-02-01T17:35:28.355Z{"msg":"[1]R generation[0..12]: [171, 170, 168, 171, 171, 170, 166, 169, 166, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617489424Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185022024-02-01T17:35:28.355Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617544387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185032024-02-01T17:35:28.355Z{"msg":"[2]R flush_numbers[0..12]: [279, 278, 274, 279, 279, 278, 268, 276, 266, 279, 275, 279]","Wait for a query_work_queue command to finish before sending IO
185042024-02-01T17:35:28.355Zv":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617574155Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185052024-02-01T17:35:28.355Z{"msg":"[2]R generation[0..12]: [171, 170, 168, 171, 171, 170, 166, 169, 166, 171, 168, 171]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617616718Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185062024-02-01T17:35:28.356Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617642357Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185072024-02-01T17:35:28.356Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617696474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185082024-02-01T17:35:28.356Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617737566Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185092024-02-01T17:35:28.356Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61777901Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185102024-02-01T17:35:28.356Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617834437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"":"downstairs","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185112024-02-01T17:35:28.356Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617877475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185122024-02-01T17:35:28.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617928276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"0","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185132024-02-01T17:35:28.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.617970314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"1","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185142024-02-01T17:35:28.356Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.618068874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"client":"2","":"downstairs","session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185152024-02-01T17:35:28.356Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.618114433Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185162024-02-01T17:35:28.356Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.618151475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185172024-02-01T17:35:28.356Z{"msg":"5cfc5ea6-acba-4536-9f5d-8584981cb345 is now active with session: fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.618183355Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185182024-02-01T17:35:28.356Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:27.61820486Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820,"session_id":"fd5fb85c-2f8b-4c5a-b43c-5a603e2474d9"}
185192024-02-01T17:35:28.356Z{"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
185202024-02-01T17:35:28.356Z,"time":"2024-02-01T17:35:27.61822599Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
185212024-02-01T17:35:28.356ZLoading write count information from file "/tmp/test_repair_verify.data"
185222024-02-01T17:35:28.357ZRun Repair workload
185232024-02-01T17:35:28.357Z01/30 Flush
185242024-02-01T17:35:28.357Z02/30 Write block 457 len 4096 data: 19
185252024-02-01T17:35:28.357Z03/30 Write block 288 len 40960 data: 16 19 17 17 13 15 15 13 14 12
185262024-02-01T17:35:28.357Z04/30 Write block 162 len 24576 data: 15 14 14 12 10 10
185272024-02-01T17:35:28.357Z05/30 Write block 507 len 4096 data: 10
185282024-02-01T17:35:28.357Z06/30 Write block 466 len 8192 data: 16 14
185292024-02-01T17:35:28.357Z07/30 Write block 236 len 36864 data: 11 11 11 11 10 9 13 13 13
185302024-02-01T17:35:28.357Z08/30 Flush
185312024-02-01T17:35:28.361Z09/30 Write block 581 len 28672 data: 15 14 12 14 13 13 11
185322024-02-01T17:35:28.361Z10/30 Read block 311 len 32768
185332024-02-01T17:35:28.361Z11/30 Write block 401 len 12288 data: 13 12 15
185342024-02-01T17:35:28.361Z12/30 Write block 404 len 8192 data: 14 12
185352024-02-01T17:35:28.362Z13/30 Write block 489 len 24576 data: 21 20 18 20 16 14
185362024-02-01T17:35:28.362Z14/30 Write block 398 len 40960 data: 10 8 10 14 13 16 15 13 12 14
185372024-02-01T17:35:28.362Z15/30 Write block 304 len 4096 data: 11
185382024-02-01T17:35:28.363Z16/30 Read block 133 len 32768
185392024-02-01T17:35:28.363Z17/30 Read block 312 len 36864
185402024-02-01T17:35:28.363Z18/30 Read block 55 len 12288
185412024-02-01T17:35:28.389Z19/30 Flush
185422024-02-01T17:35:28.389Z20/30 Write block 25 len 20480 data: 11 9 7 6 8
185432024-02-01T17:35:28.389Z21/30 Read block 47 len 4096
185442024-02-01T17:35:28.390Z22/30 Read block 355 len 4096
185452024-02-01T17:35:28.390Z23/30 Read block 460 len 40960
185462024-02-01T17:35:28.390Z24/30 Write block 498 len 28672 data: 15 15 14 12 12 11 10
185472024-02-01T17:35:28.390Z25/30 Write block 24 len 12288 data: 10 12 10
185482024-02-01T17:35:28.390Z26/30 Write block 197 len 4096 data: 12
185492024-02-01T17:35:28.390Z27/30 Write block 20 len 8192 data: 14 13
185502024-02-01T17:35:28.390Z28/30 Read block 66 len 36864
185512024-02-01T17:35:28.391Z29/30 Write block 93 len 24576 data: 9 9 7 9 9 10
185522024-02-01T17:35:28.391Z30/30 Write block 511 len 16384 data: 13 11 12 10
185532024-02-01T17:35:28.392Z----------------------------------------------------------------
185542024-02-01T17:35:28.392Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:29
185552024-02-01T17:35:28.392ZStates: Active Active Active
185562024-02-01T17:35:28.392ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185572024-02-01T17:35:28.392Z 2 Acked 1001 Write 1 Done Sent Done false
185582024-02-01T17:35:28.392Z 3 Acked 1002 Write 10 Done Sent Done false
185592024-02-01T17:35:28.392Z 4 Acked 1003 Write 6 Done Sent Done false
185602024-02-01T17:35:28.392Z 5 Acked 1004 Write 1 Done Sent Done false
185612024-02-01T17:35:28.392Z 6 Acked 1005 Write 2 Done Sent Done false
185622024-02-01T17:35:28.393Z 7 Acked 1006 Write 9 Done Sent Done false
185632024-02-01T17:35:28.393Z 8 Acked 1007 Flush 0 Done Sent Done false
185642024-02-01T17:35:28.393Z 9 Acked 1008 Write 7 Done Sent Done false
185652024-02-01T17:35:28.393Z 10 Acked 1009 Read 8 Done Sent Done false
185662024-02-01T17:35:28.393Z 11 Acked 1010 Write 3 Done Sent Done false
185672024-02-01T17:35:28.393Z 12 Acked 1011 Write 2 Done Sent Done false
185682024-02-01T17:35:28.393Z 13 Acked 1012 Write 6 Done Sent Done false
185692024-02-01T17:35:28.393Z 14 Acked 1013 Write 10 Done Sent Done false
185702024-02-01T17:35:28.393Z 15 Acked 1014 Write 1 Done Sent Done false
185712024-02-01T17:35:28.393Z 16 Acked 1015 Read 8 Done Sent Done false
185722024-02-01T17:35:28.394Z 17 Acked 1016 Read 9 Done Sent Done false
185732024-02-01T17:35:28.394Z 18 Acked 1017 Read 3 Done Sent Done false
185742024-02-01T17:35:28.394Z 19 Acked 1018 Flush 0 Done Sent Done false
185752024-02-01T17:35:28.394Z 20 Acked 1019 Write 5 Done Sent Done false
185762024-02-01T17:35:28.394Z 21 Acked 1020 Read 1 Done Sent Done false
185772024-02-01T17:35:28.394Z 22 Acked 1021 Read 1 Done Sent Done false
185782024-02-01T17:35:28.394Z 23 Acked 1022 Read 10 Done Sent Done false
185792024-02-01T17:35:28.394Z 24 Acked 1023 Write 7 Done Sent Done false
185802024-02-01T17:35:28.394Z 25 Acked 1024 Write 3 Done Sent Done false
185812024-02-01T17:35:28.394Z 26 Acked 1025 Write 1 Done Sent Done false
185822024-02-01T17:35:28.394Z 27 Acked 1026 Write 2 Done Sent Done false
185832024-02-01T17:35:28.394Z 28 Acked 1027 Read 9 Done Sent Done false
185842024-02-01T17:35:28.394Z 29 Acked 1028 Write 6 Done Sent Done false
185852024-02-01T17:35:28.395Z 30 Acked 1029 Write 4 Sent Sent Sent false
185862024-02-01T17:35:28.395Z STATES DS:0 DS:1 DS:2 TOTAL
185872024-02-01T17:35:28.395Z New 0 0 0 0
185882024-02-01T17:35:28.395Z Sent 1 29 1 31
185892024-02-01T17:35:28.395Z Done 28 0 28 56
185902024-02-01T17:35:28.395Z Skipped 0 0 0 0
185912024-02-01T17:35:28.395Z Error 0 0 0 0
185922024-02-01T17:35:28.395ZLast Flush: 1018 1000 1018
185932024-02-01T17:35:28.395ZDownstairs last five completed: 1000
185942024-02-01T17:35:28.395ZUpstairs last five completed: 30 29 28 27 26
185952024-02-01T17:35:28.395Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:27.662502724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1820}
185962024-02-01T17:35:28.395ZWrote out file "/tmp/test_repair_verify.data"
185972024-02-01T17:35:28.395Z
185982024-02-01T17:35:28.404ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185992024-02-01T17:35:28.417ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186002024-02-01T17:35:28.417Z 0 000-019 171 171 171 279 279 279 F F F
186012024-02-01T17:35:28.421Z 1 020-039 170 170 170 278 278 278 T F T <---
186022024-02-01T17:35:28.421Z 2 040-059 168 168 168 274 274 274 F F F
186032024-02-01T17:35:28.421Z 3 060-079 171 171 171 279 279 279 F F F
186042024-02-01T17:35:28.422Z 4 080-099 171 171 171 279 279 279 T F T <---
186052024-02-01T17:35:28.422Z 5 100-119 170 170 170 278 278 278 F F F
186062024-02-01T17:35:28.422Z 6 120-139 166 166 166 268 268 268 F F F
186072024-02-01T17:35:28.422Z 7 140-159 169 169 169 276 276 276 F F F
186082024-02-01T17:35:28.422Z 8 160-179 172 166 172 281 266 281 F F F <---
186092024-02-01T17:35:28.422Z 9 180-199 171 171 171 279 279 279 T F T <---
186102024-02-01T17:35:28.422Z 10 200-219 168 168 168 275 275 275 F F F
186112024-02-01T17:35:28.422Z 11 220-239 172 171 172 281 279 281 F F F <---
186122024-02-01T17:35:28.422Z 12 240-259 172 171 172 281 279 281 F F F <---
186132024-02-01T17:35:28.422Z 13 260-279 168 168 168 274 274 274 F F F
186142024-02-01T17:35:28.422Z 14 280-299 172 171 172 281 279 281 F F F <---
186152024-02-01T17:35:28.422Z 15 300-319 172 166 172 282 266 282 F F F <---
186162024-02-01T17:35:28.422Z 16 320-339 165 165 165 264 264 264 F F F
186172024-02-01T17:35:28.422Z 17 340-359 168 168 168 275 275 275 F F F
186182024-02-01T17:35:28.423Z 18 360-379 171 171 171 279 279 279 F F F
186192024-02-01T17:35:28.423Z 19 380-399 172 171 172 282 279 282 F F F <---
186202024-02-01T17:35:28.423Z 20 400-419 172 171 172 282 279 282 F F F <---
186212024-02-01T17:35:28.423Z 21 420-439 170 170 170 278 278 278 F F F
186222024-02-01T17:35:28.423Z 22 440-459 172 171 172 281 279 281 F F F <---
186232024-02-01T17:35:28.423Z 23 460-479 172 171 172 281 279 281 F F F <---
186242024-02-01T17:35:28.423Z 24 480-499 172 169 172 282 276 282 T F T <---
186252024-02-01T17:35:28.423Z 25 500-519 172 171 172 281 279 281 T F T <---
186262024-02-01T17:35:28.423Z 26 520-539 162 162 162 261 261 261 F F F
186272024-02-01T17:35:28.423Z 27 540-559 171 171 171 279 279 279 F F F
186282024-02-01T17:35:28.424Z 28 560-579 166 166 166 268 268 268 F F F
186292024-02-01T17:35:28.424Z 29 580-599 172 163 172 282 262 282 F F F <---
186302024-02-01T17:35:28.424ZMax gen: 172, Max flush: 282
186312024-02-01T17:35:28.424ZError: Difference in extent metadata found!
186322024-02-01T17:35:28.428ZA Difference in extent metadata is expected here
186332024-02-01T17:35:28.428ZOn loop 86
186342024-02-01T17:35:28.428Z
186352024-02-01T17:35:28.428ZVerifying data now
186362024-02-01T17:35:33.686ZNo change after verify
186372024-02-01T17:35:33.686ZLoop: 86 Downstairs dump after verify (and repair):
186382024-02-01T17:35:33.709ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186392024-02-01T17:35:33.709Z 0 000-019 171 171 171 279 279 279 F F F
186402024-02-01T17:35:33.709Z 1 020-039 173 173 173 283 283 283 F F F
186412024-02-01T17:35:33.710Z 2 040-059 168 168 168 274 274 274 F F F
186422024-02-01T17:35:33.710Z 3 060-079 171 171 171 279 279 279 F F F
186432024-02-01T17:35:33.710Z 4 080-099 173 173 173 283 283 283 F F F
186442024-02-01T17:35:33.710Z 5 100-119 170 170 170 278 278 278 F F F
186452024-02-01T17:35:33.710Z 6 120-139 166 166 166 268 268 268 F F F
186462024-02-01T17:35:33.710Z 7 140-159 169 169 169 276 276 276 F F F
186472024-02-01T17:35:33.711Z 8 160-179 172 172 172 281 281 281 F F F
186482024-02-01T17:35:33.711Z 9 180-199 173 173 173 283 283 283 F F F
186492024-02-01T17:35:33.711Z 10 200-219 168 168 168 275 275 275 F F F
186502024-02-01T17:35:33.711Z 11 220-239 172 172 172 281 281 281 F F F
186512024-02-01T17:35:33.711Z 12 240-259 172 172 172 281 281 281 F F F
186522024-02-01T17:35:33.716Z 13 260-279 168 168 168 274 274 274 F F F
186532024-02-01T17:35:33.716Z 14 280-299 172 172 172 281 281 281 F F F
186542024-02-01T17:35:33.716Z 15 300-319 172 172 172 282 282 282 F F F
186552024-02-01T17:35:33.716Z 16 320-339 165 165 165 264 264 264 F F F
186562024-02-01T17:35:33.716Z 17 340-359 168 168 168 275 275 275 F F F
186572024-02-01T17:35:33.716Z 18 360-379 171 171 171 279 279 279 F F F
186582024-02-01T17:35:33.716Z 19 380-399 172 172 172 282 282 282 F F F
186592024-02-01T17:35:33.716Z 20 400-419 172 172 172 282 282 282 F F F
186602024-02-01T17:35:33.717Z 21 420-439 170 170 170 278 278 278 F F F
186612024-02-01T17:35:33.717Z 22 440-459 172 172 172 281 281 281 F F F
186622024-02-01T17:35:33.717Z 23 460-479 172 172 172 281 281 281 F F F
186632024-02-01T17:35:33.717Z 24 480-499 173 173 173 283 283 283 F F F
186642024-02-01T17:35:33.717Z 25 500-519 173 173 173 283 283 283 F F F
186652024-02-01T17:35:33.717Z 26 520-539 162 162 162 261 261 261 F F F
186662024-02-01T17:35:33.717Z 27 540-559 171 171 171 279 279 279 F F F
186672024-02-01T17:35:33.717Z 28 560-579 166 166 166 268 268 268 F F F
186682024-02-01T17:35:33.717Z 29 580-599 172 172 172 282 282 282 F F F
186692024-02-01T17:35:33.717ZMax gen: 173, Max flush: 283
186702024-02-01T17:35:33.717Z
186712024-02-01T17:35:33.717ZBegin loop 87
186722024-02-01T17:35:33.717ZDownstairs to restart: 0
186732024-02-01T17:35:33.748Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.02868252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
186742024-02-01T17:35:33.748Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029203212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
186752024-02-01T17:35:33.748Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029258555Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
186762024-02-01T17:35:33.763Z{"msg":"Crucible 6bc5a120-6769-4c75-8157-7b50d41f0af0 has session id: 0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029385536Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186772024-02-01T17:35:33.763Z{"msg":"Upstairs opts: Upstairs UUID: 6bc5a120-6769-4c75-8157-7b50d41f0af0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029440989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"Crucible runtime is spawned
186782024-02-01T17:35:33.763Z}
186792024-02-01T17:35:33.763Z{"msg":"Crucible stats registered with UUID: 6bc5a120-6769-4c75-8157-7b50d41f0af0","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029554587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186802024-02-01T17:35:33.763Z{"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029608193Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
186812024-02-01T17:35:33.763Z{"msg":"6bc5a120-6769-4c75-8157-7b50d41f0af0 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029791856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186822024-02-01T17:35:33.763Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029940228Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186832024-02-01T17:35:33.764Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.029998241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186842024-02-01T17:35:33.765Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030041107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186852024-02-01T17:35:33.765Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030072833Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186862024-02-01T17:35:33.765Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030096338Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186872024-02-01T17:35:33.765Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030118598Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186882024-02-01T17:35:33.765Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030166394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186892024-02-01T17:35:33.765Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030447899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186902024-02-01T17:35:33.765Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030692394Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186912024-02-01T17:35:33.765Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030767506Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186922024-02-01T17:35:33.765Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030794579Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186932024-02-01T17:35:33.766Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.030826436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"io task","client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186942024-02-01T17:35:33.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.031762951Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186952024-02-01T17:35:33.766Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.031845482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186962024-02-01T17:35:33.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.031894276Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186972024-02-01T17:35:33.766Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.031969294Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186982024-02-01T17:35:33.766Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.03295549Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
186992024-02-01T17:35:33.766Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.033026744Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187002024-02-01T17:35:33.766Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.033055719Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187012024-02-01T17:35:33.766Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.033083751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187022024-02-01T17:35:33.766Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.033871977Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187032024-02-01T17:35:33.767Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.033957006Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187042024-02-01T17:35:33.767Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.03402496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187052024-02-01T17:35:33.767Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.034112593Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187062024-02-01T17:35:33.767Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.034177381Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187072024-02-01T17:35:33.767Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.034205035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187082024-02-01T17:35:33.767Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.034246873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187092024-02-01T17:35:33.767Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.034945225Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187102024-02-01T17:35:33.767Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035016586Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187112024-02-01T17:35:33.767Z{"msg":"[0]R flush_numbers[0..12]: [279, 283, 274, 279, 283, 278, 268, 276, 281, 283, 275, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035065419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187122024-02-01T17:35:33.768Z{"msg":"[0]R generation[0..12]: [171, 173, 168, 171, 173, 170, 166, 169, 172, 173, 168, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035091727Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187132024-02-01T17:35:33.769Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035138292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187142024-02-01T17:35:33.769Z{"msg":"[1]R flush_numbers[0..12]: [279, 283, 274, 279, 283, 278, 268, 276, 281, 283, 275, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035164427Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187152024-02-01T17:35:33.769Z{"msg":"[1]R generation[0..12]: [171, 173, 168, 171, 173, 170, 166, 169, 172, 173, 168, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035205623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187162024-02-01T17:35:33.769Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035231616Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187172024-02-01T17:35:33.769Z{"msg":"Wait for a query_work_queue command to finish before sending IO
187182024-02-01T17:35:33.769Z[2]R flush_numbers[0..12]: [279, 283, 274, 279, 283, 278, 268, 276, 281, 283, 275, 281]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035281307Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187192024-02-01T17:35:33.769Z{"msg":"[2]R generation[0..12]: [171, 173, 168, 171, 173, 170, 166, 169, 172, 173, 168, 172]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035308995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187202024-02-01T17:35:33.769Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035334378Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187212024-02-01T17:35:33.769Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035383223Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187222024-02-01T17:35:33.769Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.03540784Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187232024-02-01T17:35:33.771Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035448257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187242024-02-01T17:35:33.772Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.03547227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"":"downstairs","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187252024-02-01T17:35:33.772Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035496592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187262024-02-01T17:35:33.772Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035516511Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"0","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187272024-02-01T17:35:33.772Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035538096Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"1","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187282024-02-01T17:35:33.772Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035585912Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"client":"2","":"downstairs","session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187292024-02-01T17:35:33.772Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.03561164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187302024-02-01T17:35:33.772Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035634676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187312024-02-01T17:35:33.772Z{"msg":"6bc5a120-6769-4c75-8157-7b50d41f0af0 is now active with session: 0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035657063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187322024-02-01T17:35:33.773Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035681606Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828,"session_id":"0ea7f6cf-b331-4c2e-a80d-4e9b6fe10f7a"}
187332024-02-01T17:35:33.773Z{"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:33.035727602Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
187342024-02-01T17:35:33.773ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187352024-02-01T17:35:33.774ZLoading write count information from file "/tmp/test_repair_verify.data"
187362024-02-01T17:35:33.774ZRun Repair workload
187372024-02-01T17:35:33.774Z01/30 Write block 491 len 8192 data: 19 21
187382024-02-01T17:35:33.783Z02/30 Read block 462 len 20480
187392024-02-01T17:35:33.783Z03/30 Read block 160 len 8192
187402024-02-01T17:35:33.783Z04/30 Read block 69 len 36864
187412024-02-01T17:35:33.783Z05/30 Write block 230 len 16384 data: 12 12 13 11
187422024-02-01T17:35:33.783Z06/30 Read block 372 len 32768
187432024-02-01T17:35:33.783Z07/30 Write block 534 len 8192 data: 16 16
187442024-02-01T17:35:33.784Z08/30 Read block 549 len 24576
187452024-02-01T17:35:33.784Z09/30 Flush
187462024-02-01T17:35:33.784Z10/30 Write block 256 len 12288 data: 16 15 18
187472024-02-01T17:35:33.784Z11/30 Write block 311 len 16384 data: 8 13 16 16
187482024-02-01T17:35:33.784Z12/30 Flush
187492024-02-01T17:35:33.784Z13/30 Flush
187502024-02-01T17:35:33.784Z14/30 Write block 11 len 24576 data: 10 9 8 8 9 11
187512024-02-01T17:35:33.784Z15/30 Write block 16 len 8192 data: 12 14
187522024-02-01T17:35:33.784Z16/30 Flush
187532024-02-01T17:35:33.784Z17/30 Write block 582 len 8192 data: 15 13
187542024-02-01T17:35:33.784Z18/30 Write block 31 len 20480 data: 8 9 9 10 12
187552024-02-01T17:35:33.785Z19/30 Flush
187562024-02-01T17:35:33.792Z20/30 Write block 398 len 20480 data: 11 9 11 15 14
187572024-02-01T17:35:33.792Z21/30 Flush
187582024-02-01T17:35:33.792Z22/30 Write block 59 len 24576 data: 12 13 11 11 11 10
187592024-02-01T17:35:33.792Z23/30 Read block 516 len 32768
187602024-02-01T17:35:33.799Z24/30 Read block 574 len 24576
187612024-02-01T17:35:33.799Z25/30 Read block 104 len 20480
187622024-02-01T17:35:33.799Z26/30 Read block 60 len 4096
187632024-02-01T17:35:33.799Z27/30 Read block 182 len 4096
187642024-02-01T17:35:33.799Z28/30 Read block 476 len 20480
187652024-02-01T17:35:33.808Z29/30 Write block 337 len 28672 data: 15 12 12 10 13 13 15
187662024-02-01T17:35:33.808Z30/30 Read block 497 len 4096
187672024-02-01T17:35:33.809Z----------------------------------------------------------------
187682024-02-01T17:35:33.809Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
187692024-02-01T17:35:33.809ZStates: Active Active Active
187702024-02-01T17:35:33.809ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187712024-02-01T17:35:33.809Z 1 Acked 1000 Write 2 Done Done Done false
187722024-02-01T17:35:33.809Z 2 Acked 1001 Read 5 Done Done Done false
187732024-02-01T17:35:33.809Z 3 Acked 1002 Read 2 Sent Done Done false
187742024-02-01T17:35:33.809Z 4 Acked 1003 Read 9 Sent Done Done false
187752024-02-01T17:35:33.809Z 5 Acked 1004 Write 4 Sent Done Done false
187762024-02-01T17:35:33.809Z 6 Acked 1005 Read 8 Sent Done Done false
187772024-02-01T17:35:33.809Z 7 Acked 1006 Write 2 Sent Done Done false
187782024-02-01T17:35:33.809Z 8 Acked 1007 Read 6 Sent Done Done false
187792024-02-01T17:35:33.809Z 9 Acked 1008 Flush 0 Sent Done Done false
187802024-02-01T17:35:33.809Z 10 Acked 1009 Write 3 Sent Done Done false
187812024-02-01T17:35:33.809Z 11 Acked 1010 Write 4 Sent Done Done false
187822024-02-01T17:35:33.810Z 12 Acked 1011 Flush 0 Sent Done Done false
187832024-02-01T17:35:33.810Z 13 Acked 1012 Flush 0 Sent Done Done false
187842024-02-01T17:35:33.810Z 14 Acked 1013 Write 6 Sent Done Done false
187852024-02-01T17:35:33.811Z 15 Acked 1014 Write 2 Sent Done Done false
187862024-02-01T17:35:33.811Z 16 Acked 1015 Flush 0 Sent Done Done false
187872024-02-01T17:35:33.811Z 17 Acked 1016 Write 2 Sent Done Done false
187882024-02-01T17:35:33.811Z 18 Acked 1017 Write 5 Sent Done Done false
187892024-02-01T17:35:33.811Z 19 Acked 1018 Flush 0 Sent Done Done false
187902024-02-01T17:35:33.811Z 20 Acked 1019 Write 5 Sent Done Done false
187912024-02-01T17:35:33.811Z 21 Acked 1020 Flush 0 Sent Done Done false
187922024-02-01T17:35:33.811Z 22 Acked 1021 Write 6 Sent Done Done false
187932024-02-01T17:35:33.811Z 23 Acked 1022 Read 8 Sent Done Done false
187942024-02-01T17:35:33.811Z 24 Acked 1023 Read 6 Sent Done Done false
187952024-02-01T17:35:33.811Z 25 Acked 1024 Read 5 Sent Done Done false
187962024-02-01T17:35:33.812Z 26 Acked 1025 Read 1 Sent Done Done false
187972024-02-01T17:35:33.812Z 27 Acked 1026 Read 1 Sent Done Done false
187982024-02-01T17:35:33.812Z 28 Acked 1027 Read 5 Sent Done Done false
187992024-02-01T17:35:33.812Z 29 Acked 1028 Write 7 Sent Done Done false
188002024-02-01T17:35:33.812Z 30 Acked 1029 Read 1 Sent Done Done false
188012024-02-01T17:35:33.812Z STATES DS:0 DS:1 DS:2 TOTAL
188022024-02-01T17:35:33.812Z New 0 0 0 0
188032024-02-01T17:35:33.812Z Sent 28 0 0 28
188042024-02-01T17:35:33.812Z Done 2 30 30 62
188052024-02-01T17:35:33.813Z Skipped 0 0 0 0
188062024-02-01T17:35:33.813Z Error 0 0 0 0
188072024-02-01T17:35:33.813ZLast Flush: 0 1020 1020
188082024-02-01T17:35:33.813ZDownstairs last five completed:
188092024-02-01T17:35:33.813ZUpstairs last five completed: 30 29 28 27 26
188102024-02-01T17:35:33.814Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:33.085144684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1828}
188112024-02-01T17:35:33.814ZWrote out file "/tmp/test_repair_verify.data"
188122024-02-01T17:35:33.820Z
188132024-02-01T17:35:33.820ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188142024-02-01T17:35:33.840ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188152024-02-01T17:35:33.840Z 0 000-019 171 174 174 279 287 287 F F F <---
188162024-02-01T17:35:33.850Z 1 020-039 173 174 174 283 288 288 F F F <---
188172024-02-01T17:35:33.850Z 2 040-059 168 168 168 274 274 274 F T T <---
188182024-02-01T17:35:33.850Z 3 060-079 171 171 171 279 279 279 F T T <---
188192024-02-01T17:35:33.850Z 4 080-099 173 173 173 283 283 283 F F F
188202024-02-01T17:35:33.850Z 5 100-119 170 170 170 278 278 278 F F F
188212024-02-01T17:35:33.850Z 6 120-139 166 166 166 268 268 268 F F F
188222024-02-01T17:35:33.850Z 7 140-159 169 169 169 276 276 276 F F F
188232024-02-01T17:35:33.851Z 8 160-179 172 172 172 281 281 281 F F F
188242024-02-01T17:35:33.851Z 9 180-199 173 173 173 283 283 283 F F F
188252024-02-01T17:35:33.851Z 10 200-219 168 168 168 275 275 275 F F F
188262024-02-01T17:35:33.851Z 11 220-239 172 174 174 281 284 284 F F F <---
188272024-02-01T17:35:33.851Z 12 240-259 172 174 174 281 285 285 F F F <---
188282024-02-01T17:35:33.851Z 13 260-279 168 168 168 274 274 274 F F F
188292024-02-01T17:35:33.852Z 14 280-299 172 172 172 281 281 281 F F F
188302024-02-01T17:35:33.853Z 15 300-319 172 174 174 282 285 285 F F F <---
188312024-02-01T17:35:33.853Z 16 320-339 165 165 165 264 264 264 F T T <---
188322024-02-01T17:35:33.853Z 17 340-359 168 168 168 275 275 275 F T T <---
188332024-02-01T17:35:33.853Z 18 360-379 171 171 171 279 279 279 F F F
188342024-02-01T17:35:33.853Z 19 380-399 172 174 174 282 289 289 F F F <---
188352024-02-01T17:35:33.853Z 20 400-419 172 174 174 282 289 289 F F F <---
188362024-02-01T17:35:33.853Z 21 420-439 170 170 170 278 278 278 F F F
188372024-02-01T17:35:33.853Z 22 440-459 172 172 172 281 281 281 F F F
188382024-02-01T17:35:33.854Z 23 460-479 172 172 172 281 281 281 F F F
188392024-02-01T17:35:33.854Z 24 480-499 173 174 174 283 284 284 T F F <---
188402024-02-01T17:35:33.854Z 25 500-519 173 173 173 283 283 283 F F F
188412024-02-01T17:35:33.854Z 26 520-539 162 174 174 261 284 284 F F F <---
188422024-02-01T17:35:33.854Z 27 540-559 171 171 171 279 279 279 F F F
188432024-02-01T17:35:33.854Z 28 560-579 166 166 166 268 268 268 F F F
188442024-02-01T17:35:33.854Z 29 580-599 172 174 174 282 288 288 F F F <---
188452024-02-01T17:35:33.854ZMax gen: 174, Max flush: 289
188462024-02-01T17:35:33.854ZError: Difference in extent metadata found!
188472024-02-01T17:35:33.854ZA Difference in extent metadata is expected here
188482024-02-01T17:35:33.854ZOn loop 87
188492024-02-01T17:35:33.854Z
188502024-02-01T17:35:33.854ZVerifying data now
188512024-02-01T17:35:42.467ZNo change after verify
188522024-02-01T17:35:42.468ZLoop: 87 Downstairs dump after verify (and repair):
188532024-02-01T17:35:42.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188542024-02-01T17:35:42.617Z 0 000-019 174 174 174 287 287 287 F F F
188552024-02-01T17:35:42.617Z 1 020-039 174 174 174 288 288 288 F F F
188562024-02-01T17:35:42.617Z 2 040-059 175 175 175 290 290 290 F F F
188572024-02-01T17:35:42.617Z 3 060-079 175 175 175 290 290 290 F F F
188582024-02-01T17:35:42.617Z 4 080-099 173 173 173 283 283 283 F F F
188592024-02-01T17:35:42.617Z 5 100-119 170 170 170 278 278 278 F F F
188602024-02-01T17:35:42.617Z 6 120-139 166 166 166 268 268 268 F F F
188612024-02-01T17:35:42.618Z 7 140-159 169 169 169 276 276 276 F F F
188622024-02-01T17:35:42.618Z 8 160-179 172 172 172 281 281 281 F F F
188632024-02-01T17:35:42.618Z 9 180-199 173 173 173 283 283 283 F F F
188642024-02-01T17:35:42.618Z 10 200-219 168 168 168 275 275 275 F F F
188652024-02-01T17:35:42.618Z 11 220-239 174 174 174 284 284 284 F F F
188662024-02-01T17:35:42.618Z 12 240-259 174 174 174 285 285 285 F F F
188672024-02-01T17:35:42.618Z 13 260-279 168 168 168 274 274 274 F F F
188682024-02-01T17:35:42.618Z 14 280-299 172 172 172 281 281 281 F F F
188692024-02-01T17:35:42.618Z 15 300-319 174 174 174 285 285 285 F F F
188702024-02-01T17:35:42.619Z 16 320-339 175 175 175 290 290 290 F F F
188712024-02-01T17:35:42.619Z 17 340-359 175 175 175 290 290 290 F F F
188722024-02-01T17:35:42.619Z 18 360-379 171 171 171 279 279 279 F F F
188732024-02-01T17:35:42.619Z 19 380-399 174 174 174 289 289 289 F F F
188742024-02-01T17:35:42.619Z 20 400-419 174 174 174 289 289 289 F F F
188752024-02-01T17:35:42.620Z 21 420-439 170 170 170 278 278 278 F F F
188762024-02-01T17:35:42.620Z 22 440-459 172 172 172 281 281 281 F F F
188772024-02-01T17:35:42.620Z 23 460-479 172 172 172 281 281 281 F F F
188782024-02-01T17:35:42.620Z 24 480-499 174 174 174 284 284 284 F F F
188792024-02-01T17:35:42.621Z 25 500-519 173 173 173 283 283 283 F F F
188802024-02-01T17:35:42.621Z 26 520-539 174 174 174 284 284 284 F F F
188812024-02-01T17:35:42.621Z 27 540-559 171 171 171 279 279 279 F F F
188822024-02-01T17:35:42.621Z 28 560-579 166 166 166 268 268 268 F F F
188832024-02-01T17:35:42.621Z 29 580-599 174 174 174 288 288 288 F F F
188842024-02-01T17:35:42.621ZMax gen: 175, Max flush: 290
188852024-02-01T17:35:42.621Z
188862024-02-01T17:35:42.621ZBegin loop 88
188872024-02-01T17:35:42.621ZDownstairs to restart: 2
188882024-02-01T17:35:42.622Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.358740825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
188892024-02-01T17:35:42.622Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359242658Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
188902024-02-01T17:35:42.622Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
188912024-02-01T17:35:42.622Z,"time":"2024-02-01T17:35:38.359369726Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
188922024-02-01T17:35:42.622Z{"msg":"Crucible 19a15629-7390-4626-97b9-acbb241a1f3c has session id: 2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359424422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188932024-02-01T17:35:42.622Z{"msg":"Upstairs opts: Upstairs UUID: 19a15629-7390-4626-97b9-acbb241a1f3c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359502747Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188942024-02-01T17:35:42.623Z{"msg":"Crucible stats registered with UUID: 19a15629-7390-4626-97b9-acbb241a1f3c","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.35956375Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188952024-02-01T17:35:42.623Z{"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359597056Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
188962024-02-01T17:35:42.623Z{"msg":"19a15629-7390-4626-97b9-acbb241a1f3c active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359638404Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188972024-02-01T17:35:42.623Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359661475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188982024-02-01T17:35:42.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359723917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
188992024-02-01T17:35:42.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359750434Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189002024-02-01T17:35:42.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359773237Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189012024-02-01T17:35:42.624Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359801021Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189022024-02-01T17:35:42.624Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359863188Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189032024-02-01T17:35:42.624Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359906725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189042024-02-01T17:35:42.624Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.359983852Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189052024-02-01T17:35:42.624Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.360061232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189062024-02-01T17:35:42.624Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.360130229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189072024-02-01T17:35:42.624Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.360418933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189082024-02-01T17:35:42.624Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.360531931Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"io task","client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189092024-02-01T17:35:42.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.361499857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189102024-02-01T17:35:42.625Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.361541274Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189112024-02-01T17:35:42.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.361565259Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189122024-02-01T17:35:42.625Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.361656806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189132024-02-01T17:35:42.625Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.361915466Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189142024-02-01T17:35:42.626Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.362031202Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189152024-02-01T17:35:42.626Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.362721888Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189162024-02-01T17:35:42.627Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.36282663Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189172024-02-01T17:35:42.627Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363049809Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189182024-02-01T17:35:42.627Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363700972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189192024-02-01T17:35:42.627Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363818354Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189202024-02-01T17:35:42.627Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363865834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189212024-02-01T17:35:42.627Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363893975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189222024-02-01T17:35:42.627Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363938279Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189232024-02-01T17:35:42.627Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363963269Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189242024-02-01T17:35:42.627Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.363983673Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189252024-02-01T17:35:42.627Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364016149Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189262024-02-01T17:35:42.628Z{"msg":"[0]R flush_numbers[0..12]: [287, 288, 290, 290, 283, 278, 268, 276, 281, 283, 275, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364058421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189272024-02-01T17:35:42.628Z{"msg":"[0]R generation[0..12]: [174, 174, 175, 175, 173, 170, 166, 169, 172, 173, 168, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364110508Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189282024-02-01T17:35:42.628Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
189292024-02-01T17:35:42.628Z,"time":"2024-02-01T17:35:38.364136492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189302024-02-01T17:35:42.628Z{"msg":"[1]R flush_numbers[0..12]: [287, 288, 290, 290, 283, 278, 268, 276, 281, 283, 275, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364274232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189312024-02-01T17:35:42.628Z{"msg":"[1]R generation[0..12]: [174, 174, 175, 175, 173, 170, 166, 169, 172, 173, 168, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364346215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189322024-02-01T17:35:42.628Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364374146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189332024-02-01T17:35:42.628Z{"msg":"[2]R flush_numbers[0..12]: [287, 288, 290, 290, 283, 278, 268, 276, 281, 283, 275, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364403254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189342024-02-01T17:35:42.629Z{"msg":"[2]R generation[0..12]: [174, 174, 175, 175, 173, 170, 166, 169, 172, 173, 168, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.36444092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189352024-02-01T17:35:42.629Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364467078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189362024-02-01T17:35:42.629Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364509483Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189372024-02-01T17:35:42.630Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364534098Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189382024-02-01T17:35:42.630Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364555802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189392024-02-01T17:35:42.630Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364577262Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"":"downstairs","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189402024-02-01T17:35:42.630Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364623013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189412024-02-01T17:35:42.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.36464548Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"0","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189422024-02-01T17:35:42.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364667601Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"1","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189432024-02-01T17:35:42.630Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364690635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"client":"2","":"downstairs","session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189442024-02-01T17:35:42.632Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364712798Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189452024-02-01T17:35:42.632Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364757218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189462024-02-01T17:35:42.633Z{"msg":"19a15629-7390-4626-97b9-acbb241a1f3c is now active with session: 2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364780796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189472024-02-01T17:35:42.633Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364800942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836,"session_id":"2bd2911d-3a2f-41e7-99c3-4c0f83a16d6f"}
189482024-02-01T17:35:42.633Z{"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:38.364821712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
189492024-02-01T17:35:42.633ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189502024-02-01T17:35:42.633ZLoading write count information from file "/tmp/test_repair_verify.data"
189512024-02-01T17:35:42.633ZRun Repair workload
189522024-02-01T17:35:42.633Z01/30 Flush
189532024-02-01T17:35:42.649Z02/30 Write block 75 len 24576 data: 16 12 10 10 10 10
189542024-02-01T17:35:42.649Z03/30 Read block 568 len 28672
189552024-02-01T17:35:42.649Z04/30 Flush
189562024-02-01T17:35:42.649Z05/30 Write block 567 len 40960 data: 9 9 10 12 9 10 10 9 12 11
189572024-02-01T17:35:42.649Z06/30 Write block 322 len 36864 data: 16 14 14 8 9 11 11 13 15
189582024-02-01T17:35:42.650Z07/30 Read block 305 len 28672
189592024-02-01T17:35:42.650Z08/30 Read block 365 len 36864
189602024-02-01T17:35:42.650Z09/30 Write block 340 len 8192 data: 11 14
189612024-02-01T17:35:42.651Z10/30 Write block 209 len 16384 data: 17 13 15 14
189622024-02-01T17:35:42.651Z11/30 Write block 348 len 4096 data: 14
189632024-02-01T17:35:42.651Z12/30 Read block 148 len 4096
189642024-02-01T17:35:42.651Z13/30 Read block 547 len 8192
189652024-02-01T17:35:42.651Z14/30 Write block 26 len 32768 data: 11 8 7 9 9 9 10 10
189662024-02-01T17:35:42.651Z15/30 Read block 428 len 28672
189672024-02-01T17:35:42.651Z16/30 Read block 361 len 8192
189682024-02-01T17:35:42.651Z17/30 Write block 467 len 36864 data: 15 12 11 13 13 13 11 9 10
189692024-02-01T17:35:42.651Z18/30 Read block 103 len 28672
189702024-02-01T17:35:42.652Z19/30 Write block 290 len 8192 data: 18 18
189712024-02-01T17:35:42.652Z20/30 Write block 529 len 28672 data: 6 7 8 10 11 17 17
189722024-02-01T17:35:42.652Z21/30 Write block 512 len 8192 data: 12 13
189732024-02-01T17:35:42.652Z22/30 Read block 205 len 4096
189742024-02-01T17:35:42.652Z23/30 Read block 246 len 40960
189752024-02-01T17:35:42.652Z24/30 Read block 572 len 16384
189762024-02-01T17:35:42.652Z25/30 Read block 414 len 16384
189772024-02-01T17:35:42.652Z26/30 Read block 456 len 32768
189782024-02-01T17:35:42.652Z27/30 Write block 62 len 16384 data: 12 12 11 10
189792024-02-01T17:35:42.652Z28/30 Write block 317 len 8192 data: 19 19
189802024-02-01T17:35:42.652Z29/30 Read block 463 len 24576
189812024-02-01T17:35:42.652Z30/30 Read block 397 len 12288
189822024-02-01T17:35:42.652Z----------------------------------------------------------------
189832024-02-01T17:35:42.653Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:29
189842024-02-01T17:35:42.653ZStates: Active Active Active
189852024-02-01T17:35:42.653ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189862024-02-01T17:35:42.653Z 2 Acked 1001 Write 6 Done Done Sent false
189872024-02-01T17:35:42.653Z 3 Acked 1002 Read 7 Done Done Sent false
189882024-02-01T17:35:42.653Z 4 Acked 1003 Flush 0 Done Done Sent false
189892024-02-01T17:35:42.653Z 5 Acked 1004 Write 10 Done Done Sent false
189902024-02-01T17:35:42.653Z 6 Acked 1005 Write 9 Done Done Sent false
189912024-02-01T17:35:42.654Z 7 Acked 1006 Read 7 Done Done Sent false
189922024-02-01T17:35:42.654Z 8 Acked 1007 Read 9 Done Done Sent false
189932024-02-01T17:35:42.654Z 9 Acked 1008 Write 2 Done Done Sent false
189942024-02-01T17:35:42.654Z 10 Acked 1009 Write 4 Done Done Sent false
189952024-02-01T17:35:42.654Z 11 Acked 1010 Write 1 Done Done Sent false
189962024-02-01T17:35:42.654Z 12 Acked 1011 Read 1 Done Done Sent false
189972024-02-01T17:35:42.654Z 13 Acked 1012 Read 2 Done Done Sent false
189982024-02-01T17:35:42.654Z 14 Acked 1013 Write 8 Done Done Sent false
189992024-02-01T17:35:42.654Z 15 Acked 1014 Read 7 Done Done Sent false
190002024-02-01T17:35:42.654Z 16 Acked 1015 Read 2 Done Done Sent false
190012024-02-01T17:35:42.654Z 17 Acked 1016 Write 9 Done Done Sent false
190022024-02-01T17:35:42.654Z 18 Acked 1017 Read 7 Done Done Sent false
190032024-02-01T17:35:42.654Z 19 Acked 1018 Write 2 Done Done Sent false
190042024-02-01T17:35:42.654Z 20 Acked 1019 Write 7 Done Done Sent false
190052024-02-01T17:35:42.655Z 21 Acked 1020 Write 2 Done Done Sent false
190062024-02-01T17:35:42.655Z 22 Acked 1021 Read 1 Done Done Sent false
190072024-02-01T17:35:42.655Z 23 Acked 1022 Read 10 Done Done Sent false
190082024-02-01T17:35:42.655Z 24 Acked 1023 Read 4 Done Done Sent false
190092024-02-01T17:35:42.655Z 25 Acked 1024 Read 4 Done Done Sent false
190102024-02-01T17:35:42.656Z 26 Acked 1025 Read 8 Done Done Sent false
190112024-02-01T17:35:42.657Z 27 Acked 1026 Write 4 Done Done Sent false
190122024-02-01T17:35:42.657Z 28 Acked 1027 Write 2 Done Done Sent false
190132024-02-01T17:35:42.657Z 29 Acked 1028 Read 6 Done Done Sent false
190142024-02-01T17:35:42.657Z 30 Acked 1029 Read 3 Done Done Sent false
190152024-02-01T17:35:42.657Z STATES DS:0 DS:1 DS:2 TOTAL
190162024-02-01T17:35:42.657Z New 0 0 0 0
190172024-02-01T17:35:42.657Z Sent 0 0 29 29
190182024-02-01T17:35:42.658Z Done 29 29 0 58
190192024-02-01T17:35:42.658Z Skipped 0 0 0 0
190202024-02-01T17:35:42.658Z Error 0 0 0 0
190212024-02-01T17:35:42.658ZLast Flush: 1003 1003 1000
190222024-02-01T17:35:42.658ZDownstairs last five completed: 1000
190232024-02-01T17:35:42.658ZUpstairs last five completed: 30 29 28 27 26
190242024-02-01T17:35:42.658Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:38.408403212Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1836}
190252024-02-01T17:35:42.658ZWrote out file "/tmp/test_repair_verify.data"
190262024-02-01T17:35:42.658Z
190272024-02-01T17:35:42.658ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190282024-02-01T17:35:42.658ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190292024-02-01T17:35:42.658Z 0 000-019 174 174 174 287 287 287 F F F
190302024-02-01T17:35:42.659Z 1 020-039 174 174 174 288 288 288 T T F <---
190312024-02-01T17:35:42.659Z 2 040-059 175 175 175 290 290 290 F F F
190322024-02-01T17:35:42.659Z 3 060-079 176 176 175 292 292 290 T T F <---
190332024-02-01T17:35:42.659Z 4 080-099 176 176 173 292 292 283 F F F <---
190342024-02-01T17:35:42.659Z 5 100-119 170 170 170 278 278 278 F F F
190352024-02-01T17:35:42.659Z 6 120-139 166 166 166 268 268 268 F F F
190362024-02-01T17:35:42.659Z 7 140-159 169 169 169 276 276 276 F F F
190372024-02-01T17:35:42.659Z 8 160-179 172 172 172 281 281 281 F F F
190382024-02-01T17:35:42.660Z 9 180-199 173 173 173 283 283 283 F F F
190392024-02-01T17:35:42.660Z 10 200-219 168 168 168 275 275 275 T T F <---
190402024-02-01T17:35:42.660Z 11 220-239 174 174 174 284 284 284 F F F
190412024-02-01T17:35:42.660Z 12 240-259 174 174 174 285 285 285 F F F
190422024-02-01T17:35:42.660Z 13 260-279 168 168 168 274 274 274 F F F
190432024-02-01T17:35:42.660Z 14 280-299 172 172 172 281 281 281 T T F <---
190442024-02-01T17:35:42.660Z 15 300-319 174 174 174 285 285 285 T T F <---
190452024-02-01T17:35:42.660Z 16 320-339 175 175 175 290 290 290 T T F <---
190462024-02-01T17:35:42.660Z 17 340-359 175 175 175 290 290 290 T T F <---
190472024-02-01T17:35:42.660Z 18 360-379 171 171 171 279 279 279 F F F
190482024-02-01T17:35:42.660Z 19 380-399 174 174 174 289 289 289 F F F
190492024-02-01T17:35:42.660Z 20 400-419 174 174 174 289 289 289 F F F
190502024-02-01T17:35:42.661Z 21 420-439 170 170 170 278 278 278 F F F
190512024-02-01T17:35:42.661Z 22 440-459 172 172 172 281 281 281 F F F
190522024-02-01T17:35:42.661Z 23 460-479 172 172 172 281 281 281 T T F <---
190532024-02-01T17:35:42.668Z 24 480-499 174 174 174 284 284 284 F F F
190542024-02-01T17:35:42.668Z 25 500-519 173 173 173 283 283 283 T T F <---
190552024-02-01T17:35:42.668Z 26 520-539 174 174 174 284 284 284 T T F <---
190562024-02-01T17:35:42.669Z 27 540-559 171 171 171 279 279 279 F F F
190572024-02-01T17:35:42.669Z 28 560-579 166 166 166 268 268 268 T T F <---
190582024-02-01T17:35:42.669Z 29 580-599 174 174 174 288 288 288 F F F
190592024-02-01T17:35:42.669ZMax gen: 176, Max flush: 292
190602024-02-01T17:35:42.669ZError: Difference in extent metadata found!
190612024-02-01T17:35:42.669ZA Difference in extent metadata is expected here
190622024-02-01T17:35:42.669ZOn loop 88
190632024-02-01T17:35:42.670Z
190642024-02-01T17:35:42.670ZVerifying data now
190652024-02-01T17:35:44.248ZNo change after verify
190662024-02-01T17:35:44.248ZLoop: 88 Downstairs dump after verify (and repair):
190672024-02-01T17:35:44.265ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190682024-02-01T17:35:44.266Z 0 000-019 174 174 174 287 287 287 F F F
190692024-02-01T17:35:44.284Z 1 020-039 177 177 177 293 293 293 F F F
190702024-02-01T17:35:44.285Z 2 040-059 175 175 175 290 290 290 F F F
190712024-02-01T17:35:44.285Z 3 060-079 177 177 177 293 293 293 F F F
190722024-02-01T17:35:44.285Z 4 080-099 176 176 176 292 292 292 F F F
190732024-02-01T17:35:44.285Z 5 100-119 170 170 170 278 278 278 F F F
190742024-02-01T17:35:44.286Z 6 120-139 166 166 166 268 268 268 F F F
190752024-02-01T17:35:44.286Z 7 140-159 169 169 169 276 276 276 F F F
190762024-02-01T17:35:44.286Z 8 160-179 172 172 172 281 281 281 F F F
190772024-02-01T17:35:44.286Z 9 180-199 173 173 173 283 283 283 F F F
190782024-02-01T17:35:44.286Z 10 200-219 177 177 177 293 293 293 F F F
190792024-02-01T17:35:44.287Z 11 220-239 174 174 174 284 284 284 F F F
190802024-02-01T17:35:44.287Z 12 240-259 174 174 174 285 285 285 F F F
190812024-02-01T17:35:44.287Z 13 260-279 168 168 168 274 274 274 F F F
190822024-02-01T17:35:44.288Z 14 280-299 177 177 177 293 293 293 F F F
190832024-02-01T17:35:44.288Z 15 300-319 177 177 177 293 293 293 F F F
190842024-02-01T17:35:44.288Z 16 320-339 177 177 177 293 293 293 F F F
190852024-02-01T17:35:44.289Z 17 340-359 177 177 177 293 293 293 F F F
190862024-02-01T17:35:44.289Z 18 360-379 171 171 171 279 279 279 F F F
190872024-02-01T17:35:44.289Z 19 380-399 174 174 174 289 289 289 F F F
190882024-02-01T17:35:44.289Z 20 400-419 174 174 174 289 289 289 F F F
190892024-02-01T17:35:44.289Z 21 420-439 170 170 170 278 278 278 F F F
190902024-02-01T17:35:44.290Z 22 440-459 172 172 172 281 281 281 F F F
190912024-02-01T17:35:44.290Z 23 460-479 177 177 177 293 293 293 F F F
190922024-02-01T17:35:44.290Z 24 480-499 174 174 174 284 284 284 F F F
190932024-02-01T17:35:44.291Z 25 500-519 177 177 177 293 293 293 F F F
190942024-02-01T17:35:44.291Z 26 520-539 177 177 177 293 293 293 F F F
190952024-02-01T17:35:44.291Z 27 540-559 171 171 171 279 279 279 F F F
190962024-02-01T17:35:44.291Z 28 560-579 177 177 177 293 293 293 F F F
190972024-02-01T17:35:44.291Z 29 580-599 174 174 174 288 288 288 F F F
190982024-02-01T17:35:44.292ZMax gen: 177, Max flush: 293
190992024-02-01T17:35:44.292Z
191002024-02-01T17:35:44.292ZBegin loop 89
191012024-02-01T17:35:44.292ZDownstairs to restart: 0
191022024-02-01T17:35:44.334Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.579929475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
191032024-02-01T17:35:44.334Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.580453799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
191042024-02-01T17:35:44.335Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.580516198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
191052024-02-01T17:35:44.336Z{"msg":"Crucible 5a09aa99-660d-4af3-aadc-e94de2586f0d has session id: 9911936e-db86-437c-b289-43ca9eff45b9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.580557298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191062024-02-01T17:35:44.336Z{"msg":"Upstairs opts: Upstairs UUID: 5a09aa99-660d-4af3-aadc-e94de2586f0d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.580611468Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191072024-02-01T17:35:44.336Z{"msg":"Crucible stats registered with UUID: 5a09aa99-660d-4af3-aadc-e94de2586f0d","v":0,"name":"crucible","level":30Crucible runtime is spawned
191082024-02-01T17:35:44.336Z,"time":"2024-02-01T17:35:43.580655887Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191092024-02-01T17:35:44.337Z{"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.580797961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
191102024-02-01T17:35:44.337Z{"msg":"5a09aa99-660d-4af3-aadc-e94de2586f0d active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581069524Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191112024-02-01T17:35:44.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581138543Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191122024-02-01T17:35:44.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581179456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191132024-02-01T17:35:44.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581249925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191142024-02-01T17:35:44.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581294298Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191152024-02-01T17:35:44.338Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581494452Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191162024-02-01T17:35:44.338Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581550441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191172024-02-01T17:35:44.338Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581592805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191182024-02-01T17:35:44.338Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.58167859Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191192024-02-01T17:35:44.338Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581760518Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191202024-02-01T17:35:44.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581805292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191212024-02-01T17:35:44.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.581952553Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191222024-02-01T17:35:44.339Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.582060573Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"io task","client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191232024-02-01T17:35:44.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.582748666Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191242024-02-01T17:35:44.339Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.582884626Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191252024-02-01T17:35:44.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.582984227Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191262024-02-01T17:35:44.339Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.583104923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191272024-02-01T17:35:44.339Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.583541435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191282024-02-01T17:35:44.340Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.583624684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191292024-02-01T17:35:44.340Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.58367793Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191302024-02-01T17:35:44.340Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.583708053Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191312024-02-01T17:35:44.340Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584618679Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191322024-02-01T17:35:44.340Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584672557Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191332024-02-01T17:35:44.340Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584697172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191342024-02-01T17:35:44.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584755767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191352024-02-01T17:35:44.341Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584781475Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191362024-02-01T17:35:44.341Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584803896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191372024-02-01T17:35:44.341Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.584825696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191382024-02-01T17:35:44.341Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585523516Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191392024-02-01T17:35:44.341Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585590791Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191402024-02-01T17:35:44.341Z{"msg":"[0]R flush_numbers[0..12]: [287, 293, 290, 293, 292, 278, 268, 276, 281, 283, 293, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.58563928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191412024-02-01T17:35:44.342Z{"msg":"[0]R generation[0..12]: [174, 177, 175, 177, 176, 170, 166, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585691459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191422024-02-01T17:35:44.342Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585719372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191432024-02-01T17:35:44.342Z{"msg":"[1]R flush_numbers[0..12]: [287, 293, 290, 293, 292, 278, 268, 276, 281, 283, 293, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585743421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191442024-02-01T17:35:44.343Z{"msg":"[1]R generation[0..12]: [174, 177, 175, 177, 176, 170, 166, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585767589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191452024-02-01T17:35:44.343Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585809482Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191462024-02-01T17:35:44.343Z{"msg":"[2]R flush_numbers[0..12]: [287, 293, 290, 293, 292, 278, 268, 276, 281, 283, 293, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585835488Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191472024-02-01T17:35:44.344Z{"msg":"[2]R generation[0..12]: [174, 177, 175, 177, 176, 170, 166, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585858667Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191482024-02-01T17:35:44.344Z{Wait for a query_work_queue command to finish before sending IO
191492024-02-01T17:35:44.344Z"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585902793Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191502024-02-01T17:35:44.344Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585926945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191512024-02-01T17:35:44.344Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.585949191Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191522024-02-01T17:35:44.344Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586014322Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191532024-02-01T17:35:44.344Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586041915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"":"downstairs","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191542024-02-01T17:35:44.345Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586064349Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191552024-02-01T17:35:44.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586105167Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"0","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191562024-02-01T17:35:44.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.58612976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"1","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191572024-02-01T17:35:44.346Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586151976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"client":"2","":"downstairs","session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191582024-02-01T17:35:44.346Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586173817Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191592024-02-01T17:35:44.346Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586194044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191602024-02-01T17:35:44.346Z{"msg":"5a09aa99-660d-4af3-aadc-e94de2586f0d is now active with session: 9911936e-db86-437c-b289-43ca9eff45b9","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586259877Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191612024-02-01T17:35:44.346Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.58628614Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844,"session_id":"9911936e-db86-437c-b289-43ca9eff45b9"}
191622024-02-01T17:35:44.346Z{"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:43.586334215Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
191632024-02-01T17:35:44.346ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191642024-02-01T17:35:44.382ZLoading write count information from file "/tmp/test_repair_verify.data"
191652024-02-01T17:35:44.383ZRun Repair workload
191662024-02-01T17:35:44.383Z01/30 Write block 337 len 32768 data: 16 13 13 12 15 14 16 15
191672024-02-01T17:35:44.383Z02/30 Read block 456 len 36864
191682024-02-01T17:35:44.383Z03/30 Write block 303 len 4096 data: 11
191692024-02-01T17:35:44.383Z04/30 Write block 464 len 32768 data: 13 13 17 16 13 12 14 14
191702024-02-01T17:35:44.383Z05/30 Write block 548 len 40960 data: 12 10 10 12 13 11 9 11 10 11
191712024-02-01T17:35:44.384Z06/30 Read block 290 len 40960
191722024-02-01T17:35:44.384Z07/30 Read block 584 len 40960
191732024-02-01T17:35:44.384Z08/30 Write block 11 len 16384 data: 11 10 9 9
191742024-02-01T17:35:44.384Z09/30 Write block 597 len 8192 data: 5 5
191752024-02-01T17:35:44.384Z10/30 Write block 305 len 8192 data: 11 8
191762024-02-01T17:35:44.384Z11/30 Write block 466 len 16384 data: 18 17 14 13
191772024-02-01T17:35:44.384Z12/30 Write block 54 len 12288 data: 19 18 14
191782024-02-01T17:35:44.384Z13/30 Read block 368 len 28672
191792024-02-01T17:35:44.385Z14/30 Read block 165 len 20480
191802024-02-01T17:35:44.385Z15/30 Write block 39 len 16384 data: 15 11 14 13
191812024-02-01T17:35:44.385Z16/30 Read block 277 len 28672
191822024-02-01T17:35:44.385Z17/30 Read block 135 len 24576
191832024-02-01T17:35:44.385Z18/30 Read block 340 len 40960
191842024-02-01T17:35:44.385Z19/30 Write block 410 len 40960 data: 9 8 9 9 12 14 18 13 12 16
191852024-02-01T17:35:44.385Z20/30 Read block 305 len 20480
191862024-02-01T17:35:44.385Z21/30 Read block 224 len 12288
191872024-02-01T17:35:44.385Z22/30 Write block 492 len 16384 data: 22 17 15 17
191882024-02-01T17:35:44.385Z23/30 Read block 489 len 12288
191892024-02-01T17:35:44.385Z24/30 Read block 212 len 28672
191902024-02-01T17:35:44.385Z25/30 Read block 442 len 4096
191912024-02-01T17:35:44.386Z26/30 Read block 166 len 40960
191922024-02-01T17:35:44.386Z27/30 Write block 118 len 32768 data: 12 10 10 10 13 13 13 15
191932024-02-01T17:35:44.387Z28/30 Write block 411 len 24576 data: 9 10 10 13 15 19
191942024-02-01T17:35:44.387Z29/30 Write block 291 len 40960 data: 19 14 16 16 14 15 13 15 14 14
191952024-02-01T17:35:44.387Z30/30 Read block 129 len 40960
191962024-02-01T17:35:44.387Z----------------------------------------------------------------
191972024-02-01T17:35:44.387Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
191982024-02-01T17:35:44.387ZStates: Active Active Active
191992024-02-01T17:35:44.387ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192002024-02-01T17:35:44.387Z 1 Acked 1000 Write 8 Sent Done Done false
192012024-02-01T17:35:44.387Z 2 Acked 1001 Read 9 Sent Done Done false
192022024-02-01T17:35:44.388Z 3 Acked 1002 Write 1 Sent Done Done false
192032024-02-01T17:35:44.388Z 4 Acked 1003 Write 8 Sent Done Done false
192042024-02-01T17:35:44.388Z 5 Acked 1004 Write 10 Sent Done Done false
192052024-02-01T17:35:44.388Z 6 Acked 1005 Read 10 Sent Done Done false
192062024-02-01T17:35:44.388Z 7 Acked 1006 Read 10 Sent Done Done false
192072024-02-01T17:35:44.389Z 8 Acked 1007 Write 4 Sent Done Done false
192082024-02-01T17:35:44.389Z 9 Acked 1008 Write 2 Sent Done Done false
192092024-02-01T17:35:44.389Z 10 Acked 1009 Write 2 Sent Done Done false
192102024-02-01T17:35:44.389Z 11 Acked 1010 Write 4 Sent Done Done false
192112024-02-01T17:35:44.389Z 12 Acked 1011 Write 3 Sent Done Done false
192122024-02-01T17:35:44.389Z 13 Acked 1012 Read 7 Sent Done Done false
192132024-02-01T17:35:44.389Z 14 Acked 1013 Read 5 Sent Done Done false
192142024-02-01T17:35:44.389Z 15 Acked 1014 Write 4 Sent Done Done false
192152024-02-01T17:35:44.390Z 16 Acked 1015 Read 7 Sent Done Done false
192162024-02-01T17:35:44.390Z 17 Acked 1016 Read 6 Sent Done Done false
192172024-02-01T17:35:44.390Z 18 Acked 1017 Read 10 Sent Done Done false
192182024-02-01T17:35:44.390Z 19 Acked 1018 Write 10 Sent Done Done false
192192024-02-01T17:35:44.391Z 20 Acked 1019 Read 5 Sent Done Done false
192202024-02-01T17:35:44.391Z 21 Acked 1020 Read 3 Sent Done Done false
192212024-02-01T17:35:44.391Z 22 Acked 1021 Write 4 Sent Done Done false
192222024-02-01T17:35:44.391Z 23 Acked 1022 Read 3 Sent Done Done false
192232024-02-01T17:35:44.391Z 24 Acked 1023 Read 7 Sent Done Done false
192242024-02-01T17:35:44.391Z 25 Acked 1024 Read 1 Sent Done Done false
192252024-02-01T17:35:44.391Z 26 Acked 1025 Read 10 Sent Done Done false
192262024-02-01T17:35:44.391Z 27 Acked 1026 Write 8 Sent Done Done false
192272024-02-01T17:35:44.391Z 28 Acked 1027 Write 6 Sent Done Done false
192282024-02-01T17:35:44.391Z 29 Acked 1028 Write 10 Sent Done Done false
192292024-02-01T17:35:44.391Z 30 Acked 1029 Read 10 Sent Sent Done false
192302024-02-01T17:35:44.391Z STATES DS:0 DS:1 DS:2 TOTAL
192312024-02-01T17:35:44.391Z New 0 0 0 0
192322024-02-01T17:35:44.391Z Sent 30 1 0 31
192332024-02-01T17:35:44.392Z Done 0 29 30 59
192342024-02-01T17:35:44.392Z Skipped 0 0 0 0
192352024-02-01T17:35:44.392Z Error 0 0 0 0
192362024-02-01T17:35:44.392ZLast Flush: 0 0 0
192372024-02-01T17:35:44.392ZDownstairs last five completed:
192382024-02-01T17:35:44.392ZUpstairs last five completed: 30 29 28 27 26
192392024-02-01T17:35:44.392Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:43.632960114Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1844}
192402024-02-01T17:35:44.392ZWrote out file "/tmp/test_repair_verify.data"
192412024-02-01T17:35:44.405Z
192422024-02-01T17:35:44.405ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192432024-02-01T17:35:44.406ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192442024-02-01T17:35:44.406Z 0 000-019 174 174 174 287 287 287 F T T <---
192452024-02-01T17:35:44.406Z 1 020-039 177 177 177 293 293 293 F T T <---
192462024-02-01T17:35:44.406Z 2 040-059 175 175 175 290 290 290 F T T <---
192472024-02-01T17:35:44.406Z 3 060-079 177 177 177 293 293 293 F F F
192482024-02-01T17:35:44.406Z 4 080-099 176 176 176 292 292 292 F F F
192492024-02-01T17:35:44.406Z 5 100-119 170 170 170 278 278 278 F T T <---
192502024-02-01T17:35:44.406Z 6 120-139 166 166 166 268 268 268 F T T <---
192512024-02-01T17:35:44.407Z 7 140-159 169 169 169 276 276 276 F F F
192522024-02-01T17:35:44.407Z 8 160-179 172 172 172 281 281 281 F F F
192532024-02-01T17:35:44.407Z 9 180-199 173 173 173 283 283 283 F F F
192542024-02-01T17:35:44.407Z 10 200-219 177 177 177 293 293 293 F F F
192552024-02-01T17:35:44.407Z 11 220-239 174 174 174 284 284 284 F F F
192562024-02-01T17:35:44.407Z 12 240-259 174 174 174 285 285 285 F F F
192572024-02-01T17:35:44.407Z 13 260-279 168 168 168 274 274 274 F F F
192582024-02-01T17:35:44.407Z 14 280-299 177 177 177 293 293 293 F T T <---
192592024-02-01T17:35:44.408Z 15 300-319 177 177 177 293 293 293 F T T <---
192602024-02-01T17:35:44.408Z 16 320-339 177 177 177 293 293 293 F T T <---
192612024-02-01T17:35:44.408Z 17 340-359 177 177 177 293 293 293 F T T <---
192622024-02-01T17:35:44.408Z 18 360-379 171 171 171 279 279 279 F F F
192632024-02-01T17:35:44.408Z 19 380-399 174 174 174 289 289 289 F F F
192642024-02-01T17:35:44.408Z 20 400-419 174 174 174 289 289 289 F T T <---
192652024-02-01T17:35:44.408Z 21 420-439 170 170 170 278 278 278 F F F
192662024-02-01T17:35:44.408Z 22 440-459 172 172 172 281 281 281 F F F
192672024-02-01T17:35:44.409Z 23 460-479 177 177 177 293 293 293 F T T <---
192682024-02-01T17:35:44.409Z 24 480-499 174 174 174 284 284 284 F T T <---
192692024-02-01T17:35:44.409Z 25 500-519 177 177 177 293 293 293 F F F
192702024-02-01T17:35:44.409Z 26 520-539 177 177 177 293 293 293 F F F
192712024-02-01T17:35:44.409Z 27 540-559 171 171 171 279 279 279 F T T <---
192722024-02-01T17:35:44.409Z 28 560-579 177 177 177 293 293 293 F F F
192732024-02-01T17:35:44.409Z 29 580-599 174 174 174 288 288 288 F T T <---
192742024-02-01T17:35:44.409ZMax gen: 177, Max flush: 293
192752024-02-01T17:35:44.409ZError: Difference in extent metadata found!
192762024-02-01T17:35:44.410ZA Difference in extent metadata is expected here
192772024-02-01T17:35:44.410ZOn loop 89
192782024-02-01T17:35:44.410Z
192792024-02-01T17:35:44.410ZVerifying data now
192802024-02-01T17:35:49.599ZNo change after verify
192812024-02-01T17:35:49.599ZLoop: 89 Downstairs dump after verify (and repair):
192822024-02-01T17:35:49.622ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192832024-02-01T17:35:49.622Z 0 000-019 178 178 178 294 294 294 F F F
192842024-02-01T17:35:49.622Z 1 020-039 178 178 178 294 294 294 F F F
192852024-02-01T17:35:49.622Z 2 040-059 178 178 178 294 294 294 F F F
192862024-02-01T17:35:49.626Z 3 060-079 177 177 177 293 293 293 F F F
192872024-02-01T17:35:49.626Z 4 080-099 176 176 176 292 292 292 F F F
192882024-02-01T17:35:49.626Z 5 100-119 178 178 178 294 294 294 F F F
192892024-02-01T17:35:49.626Z 6 120-139 178 178 178 294 294 294 F F F
192902024-02-01T17:35:49.626Z 7 140-159 169 169 169 276 276 276 F F F
192912024-02-01T17:35:49.626Z 8 160-179 172 172 172 281 281 281 F F F
192922024-02-01T17:35:49.626Z 9 180-199 173 173 173 283 283 283 F F F
192932024-02-01T17:35:49.626Z 10 200-219 177 177 177 293 293 293 F F F
192942024-02-01T17:35:49.626Z 11 220-239 174 174 174 284 284 284 F F F
192952024-02-01T17:35:49.627Z 12 240-259 174 174 174 285 285 285 F F F
192962024-02-01T17:35:49.627Z 13 260-279 168 168 168 274 274 274 F F F
192972024-02-01T17:35:49.627Z 14 280-299 178 178 178 294 294 294 F F F
192982024-02-01T17:35:49.627Z 15 300-319 178 178 178 294 294 294 F F F
192992024-02-01T17:35:49.627Z 16 320-339 178 178 178 294 294 294 F F F
193002024-02-01T17:35:49.627Z 17 340-359 178 178 178 294 294 294 F F F
193012024-02-01T17:35:49.627Z 18 360-379 171 171 171 279 279 279 F F F
193022024-02-01T17:35:49.627Z 19 380-399 174 174 174 289 289 289 F F F
193032024-02-01T17:35:49.627Z 20 400-419 178 178 178 294 294 294 F F F
193042024-02-01T17:35:49.628Z 21 420-439 170 170 170 278 278 278 F F F
193052024-02-01T17:35:49.628Z 22 440-459 172 172 172 281 281 281 F F F
193062024-02-01T17:35:49.628Z 23 460-479 178 178 178 294 294 294 F F F
193072024-02-01T17:35:49.628Z 24 480-499 178 178 178 294 294 294 F F F
193082024-02-01T17:35:49.628Z 25 500-519 177 177 177 293 293 293 F F F
193092024-02-01T17:35:49.628Z 26 520-539 177 177 177 293 293 293 F F F
193102024-02-01T17:35:49.628Z 27 540-559 178 178 178 294 294 294 F F F
193112024-02-01T17:35:49.628Z 28 560-579 177 177 177 293 293 293 F F F
193122024-02-01T17:35:49.628Z 29 580-599 178 178 178 294 294 294 F F F
193132024-02-01T17:35:49.628ZMax gen: 178, Max flush: 294
193142024-02-01T17:35:49.632Z
193152024-02-01T17:35:49.632ZBegin loop 90
193162024-02-01T17:35:49.632ZDownstairs to restart: 0
193172024-02-01T17:35:49.658Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.942644991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
193182024-02-01T17:35:49.658Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943162029Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
193192024-02-01T17:35:49.658Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943194008Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
193202024-02-01T17:35:49.663Z{"msg":"Crucible 9d759e6e-b4bb-46b1-adbf-5098deaf1022 has session id: ef66936c-6d0a-4a19-9d4b-84389e1bf908","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943306327Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193212024-02-01T17:35:49.663Z{"msg":"Upstairs opts: Upstairs UUID: 9d759e6e-b4bb-46b1-adbf-5098deaf1022, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943351314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193222024-02-01T17:35:49.663Z{"msg":"Crucible stats registered with UUID: 9d759e6e-b4bb-46b1-adbf-5098deaf1022","v":0,"name":"crucible","level":30Crucible runtime is spawned
193232024-02-01T17:35:49.663Z,"time":"2024-02-01T17:35:48.943442496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193242024-02-01T17:35:49.663Z{"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943647996Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
193252024-02-01T17:35:49.663Z{"msg":"9d759e6e-b4bb-46b1-adbf-5098deaf1022 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943738826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193262024-02-01T17:35:49.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943803369Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193272024-02-01T17:35:49.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943880013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193282024-02-01T17:35:49.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943925407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193292024-02-01T17:35:49.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.943978134Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193302024-02-01T17:35:49.664Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944004013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193312024-02-01T17:35:49.664Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944043108Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193322024-02-01T17:35:49.664Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944166814Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193332024-02-01T17:35:49.664Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944299794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193342024-02-01T17:35:49.664Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944408364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193352024-02-01T17:35:49.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944500853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193362024-02-01T17:35:49.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944601928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193372024-02-01T17:35:49.664Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.944650474Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"io task","client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193382024-02-01T17:35:49.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.945671055Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193392024-02-01T17:35:49.665Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.945712042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193402024-02-01T17:35:49.665Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.94582024Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193412024-02-01T17:35:49.665Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.945966532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193422024-02-01T17:35:49.667Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.946074637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193432024-02-01T17:35:49.668Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.946160988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193442024-02-01T17:35:49.668Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.946798561Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193452024-02-01T17:35:49.668Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.946884112Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193462024-02-01T17:35:49.668Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.947134203Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193472024-02-01T17:35:49.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.947738749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193482024-02-01T17:35:49.673Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.947798567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193492024-02-01T17:35:49.673Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.947825001Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193502024-02-01T17:35:49.673Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.947948766Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193512024-02-01T17:35:49.674Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948005642Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193522024-02-01T17:35:49.674Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948034715Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193532024-02-01T17:35:49.674Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.94810848Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193542024-02-01T17:35:49.674Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948158538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193552024-02-01T17:35:49.674Z{"msg":"[0]R flush_numbers[0..12]: [294, 294, 294, 293, 292, 294, 294, 276, 281, 283, 293, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948253684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193562024-02-01T17:35:49.674Z{"msg":"[0]R generation[0..12]: [178, 178, 178, 177, 176, 178, 178, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948305388Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193572024-02-01T17:35:49.674Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948332472Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193582024-02-01T17:35:49.674Z{"msg":"[1]R flush_numbers[0..12]: [294, 294, 294, 293, 292, 294, 294, 276, 281, 283, 293, 284]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948374356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193592024-02-01T17:35:49.674Z{"msg":"[1]R generation[0..12]: [178, 178, 178, 177, 176, 178, 178, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948399751Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193602024-02-01T17:35:49.674Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948461089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193612024-02-01T17:35:49.674Z{"msg":"[2]R flush_numbers[0..12]: [294, 294, 294, 293, 292, 294, 294, 276, 281, 283, 293, 284]","v":0,"Wait for a query_work_queue command to finish before sending IO
193622024-02-01T17:35:49.674Zname":"crucible","level":30,"time":"2024-02-01T17:35:48.94855588Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193632024-02-01T17:35:49.675Z{"msg":"[2]R generation[0..12]: [178, 178, 178, 177, 176, 178, 178, 169, 172, 173, 177, 174]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948627008Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193642024-02-01T17:35:49.675Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948695324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193652024-02-01T17:35:49.675Z{"msg":"Max found gen is 179","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948733163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193662024-02-01T17:35:49.675Z{"msg":"Generation requested: 180 >= found:179","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948770347Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193672024-02-01T17:35:49.675Z{"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948808995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193682024-02-01T17:35:49.675Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948872826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"":"downstairs","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193692024-02-01T17:35:49.675Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948914326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193702024-02-01T17:35:49.675Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.948966094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"0","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193712024-02-01T17:35:49.678Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.94900876Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"1","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193722024-02-01T17:35:49.679Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.9491048Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"client":"2","":"downstairs","session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193732024-02-01T17:35:49.679Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.949177205Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193742024-02-01T17:35:49.679Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.949252991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193752024-02-01T17:35:49.679Z{"msg":"9d759e6e-b4bb-46b1-adbf-5098deaf1022 is now active with session: ef66936c-6d0a-4a19-9d4b-84389e1bf908","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.949318828Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193762024-02-01T17:35:49.679Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.94937846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852,"session_id":"ef66936c-6d0a-4a19-9d4b-84389e1bf908"}
193772024-02-01T17:35:49.679Z{"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:48.949417414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
193782024-02-01T17:35:49.679ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
193792024-02-01T17:35:49.679ZLoading write count information from file "/tmp/test_repair_verify.data"
193802024-02-01T17:35:49.679ZRun Repair workload
193812024-02-01T17:35:49.679Z01/30 Flush
193822024-02-01T17:35:49.679Z02/30 Write block 135 len 12288 data: 16 17 17
193832024-02-01T17:35:49.679Z03/30 Write block 430 len 36864 data: 10 10 16 19 17 19 16 17 15
193842024-02-01T17:35:49.679Z04/30 Read block 82 len 8192
193852024-02-01T17:35:49.679Z05/30 Read block 307 len 16384
193862024-02-01T17:35:49.683Z06/30 Flush
193872024-02-01T17:35:49.683Z07/30 Write block 306 len 36864 data: 9 8 12 10 10 9 14 17 17
193882024-02-01T17:35:49.683Z08/30 Write block 48 len 40960 data: 17 16 17 18 16 17 20 19 15 13
193892024-02-01T17:35:49.683Z09/30 Write block 38 len 32768 data: 17 16 12 15 14 15 16 15
193902024-02-01T17:35:49.683Z10/30 Write block 46 len 16384 data: 17 15 18 17
193912024-02-01T17:35:49.683Z11/30 Write block 223 len 8192 data: 18 17
193922024-02-01T17:35:49.683Z12/30 Write block 104 len 8192 data: 16 19
193932024-02-01T17:35:49.694Z13/30 Read block 445 len 28672
193942024-02-01T17:35:49.694Z14/30 Read block 113 len 8192
193952024-02-01T17:35:49.694Z15/30 Read block 465 len 24576
193962024-02-01T17:35:49.694Z16/30 Write block 66 len 16384 data: 12 13 14 17
193972024-02-01T17:35:49.694Z17/30 Read block 333 len 40960
193982024-02-01T17:35:49.711Z18/30 Read block 518 len 12288
193992024-02-01T17:35:49.711Z19/30 Write block 218 len 8192 data: 14 17
194002024-02-01T17:35:49.711Z20/30 Read block 45 len 8192
194012024-02-01T17:35:49.711Z21/30 Read block 162 len 20480
194022024-02-01T17:35:49.711Z22/30 Write block 168 len 4096 data: 13
194032024-02-01T17:35:49.711Z23/30 Write block 212 len 32768 data: 15 13 12 12 8 9 15 18
194042024-02-01T17:35:49.712Z24/30 Write block 156 len 20480 data: 12 11 13 13 14
194052024-02-01T17:35:49.712Z25/30 Flush
194062024-02-01T17:35:50.135Z26/30 Write block 408 len 20480 data: 17 11 10 10 11
194072024-02-01T17:35:50.135Z27/30 Write block 151 len 8192 data: 12 14
194082024-02-01T17:35:50.135Z28/30 Read block 79 len 40960
194092024-02-01T17:35:50.135Z29/30 Read block 191 len 16384
194102024-02-01T17:35:50.135Z30/30 Write block 458 len 32768 data: 19 20 17 16 14 13 14 14
194112024-02-01T17:35:50.135Z----------------------------------------------------------------
194122024-02-01T17:35:50.135Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:24
194132024-02-01T17:35:50.135ZStates: Active Active Active
194142024-02-01T17:35:50.135ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194152024-02-01T17:35:50.135Z 7 Acked 1006 Write 9 Done Done Done false
194162024-02-01T17:35:50.135Z 8 Acked 1007 Write 10 Done Done Done false
194172024-02-01T17:35:50.135Z 9 Acked 1008 Write 8 Done Done Done false
194182024-02-01T17:35:50.135Z 10 Acked 1009 Write 4 Sent Done Done false
194192024-02-01T17:35:50.135Z 11 Acked 1010 Write 2 Sent Done Done false
194202024-02-01T17:35:50.135Z 12 Acked 1011 Write 2 Sent Done Done false
194212024-02-01T17:35:50.135Z 13 Acked 1012 Read 7 Sent Done Done false
194222024-02-01T17:35:50.135Z 14 Acked 1013 Read 2 Sent Done Done false
194232024-02-01T17:35:50.136Z 15 Acked 1014 Read 6 Sent Done Done false
194242024-02-01T17:35:50.136Z 16 Acked 1015 Write 4 Sent Done Done false
194252024-02-01T17:35:50.136Z 17 Acked 1016 Read 10 Sent Done Done false
194262024-02-01T17:35:50.136Z 18 Acked 1017 Read 3 Sent Done Done false
194272024-02-01T17:35:50.136Z 19 Acked 1018 Write 2 Sent Done Done false
194282024-02-01T17:35:50.136Z 20 Acked 1019 Read 2 Sent Done Done false
194292024-02-01T17:35:50.136Z 21 Acked 1020 Read 5 Sent Done Done false
194302024-02-01T17:35:50.136Z 22 Acked 1021 Write 1 Sent Done Done false
194312024-02-01T17:35:50.136Z 23 Acked 1022 Write 8 Sent Done Done false
194322024-02-01T17:35:50.136Z 24 Acked 1023 Write 5 Sent Done Done false
194332024-02-01T17:35:50.136Z 25 Acked 1024 Flush 0 Sent Done Done false
194342024-02-01T17:35:50.136Z 26 Acked 1025 Write 5 Sent Done Done false
194352024-02-01T17:35:50.136Z 27 Acked 1026 Write 2 Sent Done Done false
194362024-02-01T17:35:50.136Z 28 Acked 1027 Read 10 Sent Done Done false
194372024-02-01T17:35:50.136Z 29 Acked 1028 Read 4 Sent Done Done false
194382024-02-01T17:35:50.136Z 30 Acked 1029 Write 8 Sent Sent Sent false
194392024-02-01T17:35:50.136Z STATES DS:0 DS:1 DS:2 TOTAL
194402024-02-01T17:35:50.136Z New 0 0 0 0
194412024-02-01T17:35:50.136Z Sent 21 1 1 23
194422024-02-01T17:35:50.136Z Done 3 23 23 49
194432024-02-01T17:35:50.136Z Skipped 0 0 0 0
194442024-02-01T17:35:50.136Z Error 0 0 0 0
194452024-02-01T17:35:50.136ZLast Flush: 1005 1024 1024
194462024-02-01T17:35:50.136ZDownstairs last five completed: 1005 1004 1003 1002 1001
194472024-02-01T17:35:50.137ZUpstairs last five completed: 30 29 28 27 26
194482024-02-01T17:35:50.137Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:48.996238587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1852}
194492024-02-01T17:35:50.137ZWrote out file "/tmp/test_repair_verify.data"
194502024-02-01T17:35:50.137Z
194512024-02-01T17:35:50.148ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194522024-02-01T17:35:50.148ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194532024-02-01T17:35:50.148Z 0 000-019 178 178 178 294 294 294 F F F
194542024-02-01T17:35:50.148Z 1 020-039 178 180 180 294 297 297 T F F <---
194552024-02-01T17:35:50.148Z 2 040-059 178 180 180 294 297 297 T F F <---
194562024-02-01T17:35:50.148Z 3 060-079 177 180 180 293 297 297 F F F <---
194572024-02-01T17:35:50.148Z 4 080-099 176 176 176 292 292 292 F F F
194582024-02-01T17:35:50.148Z 5 100-119 178 180 180 294 297 297 F F F <---
194592024-02-01T17:35:50.151Z 6 120-139 180 180 180 296 296 296 F F F
194602024-02-01T17:35:50.151Z 7 140-159 169 180 180 276 297 297 F T T <---
194612024-02-01T17:35:50.151Z 8 160-179 172 180 180 281 297 297 F F F <---
194622024-02-01T17:35:50.151Z 9 180-199 173 173 173 283 283 283 F F F
194632024-02-01T17:35:50.151Z 10 200-219 177 180 180 293 297 297 F F F <---
194642024-02-01T17:35:50.151Z 11 220-239 174 180 180 284 297 297 F F F <---
194652024-02-01T17:35:50.151Z 12 240-259 174 174 174 285 285 285 F F F
194662024-02-01T17:35:50.151Z 13 260-279 168 168 168 274 274 274 F F F
194672024-02-01T17:35:50.151Z 14 280-299 178 178 178 294 294 294 F F F
194682024-02-01T17:35:50.152Z 15 300-319 178 180 180 294 297 297 T F F <---
194692024-02-01T17:35:50.152Z 16 320-339 178 178 178 294 294 294 F F F
194702024-02-01T17:35:50.152Z 17 340-359 178 178 178 294 294 294 F F F
194712024-02-01T17:35:50.152Z 18 360-379 171 171 171 279 279 279 F F F
194722024-02-01T17:35:50.152Z 19 380-399 174 174 174 289 289 289 F F F
194732024-02-01T17:35:50.152Z 20 400-419 178 178 178 294 294 294 F T T <---
194742024-02-01T17:35:50.152Z 21 420-439 180 180 180 296 296 296 F F F
194752024-02-01T17:35:50.152Z 22 440-459 172 172 172 281 281 281 F T T <---
194762024-02-01T17:35:50.152Z 23 460-479 178 178 178 294 294 294 F T T <---
194772024-02-01T17:35:50.153Z 24 480-499 178 178 178 294 294 294 F F F
194782024-02-01T17:35:50.153Z 25 500-519 177 177 177 293 293 293 F F F
194792024-02-01T17:35:50.153Z 26 520-539 177 177 177 293 293 293 F F F
194802024-02-01T17:35:50.153Z 27 540-559 178 178 178 294 294 294 F F F
194812024-02-01T17:35:50.153Z 28 560-579 177 177 177 293 293 293 F F F
194822024-02-01T17:35:50.153Z 29 580-599 178 178 178 294 294 294 F F F
194832024-02-01T17:35:50.153ZMax gen: 180, Max flush: 297
194842024-02-01T17:35:50.153ZError: Difference in extent metadata found!
194852024-02-01T17:35:50.154ZA Difference in extent metadata is expected here
194862024-02-01T17:35:50.154ZOn loop 90
194872024-02-01T17:35:50.154Z
194882024-02-01T17:35:50.154ZVerifying data now
194892024-02-01T17:35:51.801ZNo change after verify
194902024-02-01T17:35:51.801ZLoop: 90 Downstairs dump after verify (and repair):
194912024-02-01T17:35:51.823ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194922024-02-01T17:35:51.823Z 0 000-019 178 178 178 294 294 294 F F F
194932024-02-01T17:35:51.839Z 1 020-039 180 180 180 297 297 297 F F F
194942024-02-01T17:35:51.839Z 2 040-059 180 180 180 297 297 297 F F F
194952024-02-01T17:35:51.839Z 3 060-079 180 180 180 297 297 297 F F F
194962024-02-01T17:35:51.839Z 4 080-099 176 176 176 292 292 292 F F F
194972024-02-01T17:35:51.839Z 5 100-119 180 180 180 297 297 297 F F F
194982024-02-01T17:35:51.839Z 6 120-139 180 180 180 296 296 296 F F F
194992024-02-01T17:35:51.839Z 7 140-159 181 181 181 298 298 298 F F F
195002024-02-01T17:35:51.839Z 8 160-179 180 180 180 297 297 297 F F F
195012024-02-01T17:35:51.839Z 9 180-199 173 173 173 283 283 283 F F F
195022024-02-01T17:35:51.839Z 10 200-219 180 180 180 297 297 297 F F F
195032024-02-01T17:35:51.839Z 11 220-239 180 180 180 297 297 297 F F F
195042024-02-01T17:35:51.839Z 12 240-259 174 174 174 285 285 285 F F F
195052024-02-01T17:35:51.839Z 13 260-279 168 168 168 274 274 274 F F F
195062024-02-01T17:35:51.839Z 14 280-299 178 178 178 294 294 294 F F F
195072024-02-01T17:35:51.839Z 15 300-319 180 180 180 297 297 297 F F F
195082024-02-01T17:35:51.839Z 16 320-339 178 178 178 294 294 294 F F F
195092024-02-01T17:35:51.839Z 17 340-359 178 178 178 294 294 294 F F F
195102024-02-01T17:35:51.839Z 18 360-379 171 171 171 279 279 279 F F F
195112024-02-01T17:35:51.839Z 19 380-399 174 174 174 289 289 289 F F F
195122024-02-01T17:35:51.840Z 20 400-419 181 181 181 298 298 298 F F F
195132024-02-01T17:35:51.840Z 21 420-439 180 180 180 296 296 296 F F F
195142024-02-01T17:35:51.840Z 22 440-459 181 181 181 298 298 298 F F F
195152024-02-01T17:35:51.840Z 23 460-479 181 181 181 298 298 298 F F F
195162024-02-01T17:35:51.840Z 24 480-499 178 178 178 294 294 294 F F F
195172024-02-01T17:35:51.840Z 25 500-519 177 177 177 293 293 293 F F F
195182024-02-01T17:35:51.840Z 26 520-539 177 177 177 293 293 293 F F F
195192024-02-01T17:35:51.840Z 27 540-559 178 178 178 294 294 294 F F F
195202024-02-01T17:35:51.840Z 28 560-579 177 177 177 293 293 293 F F F
195212024-02-01T17:35:51.840Z 29 580-599 178 178 178 294 294 294 F F F
195222024-02-01T17:35:51.840ZMax gen: 181, Max flush: 298
195232024-02-01T17:35:51.840Z
195242024-02-01T17:35:51.840ZBegin loop 91
195252024-02-01T17:35:51.840ZDownstairs to restart: 1
195262024-02-01T17:35:52.269Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.137948896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
195272024-02-01T17:35:52.270Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138452157Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
195282024-02-01T17:35:52.705Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138564656Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
195292024-02-01T17:35:52.705ZCrucible runtime is spawned
195302024-02-01T17:35:52.705Z{"msg":"Crucible b5c49dd9-f0f0-40c0-a59f-d7c3698a4130 has session id: 91c189a7-3d15-4f28-8508-8f6c6600cf90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138680417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195312024-02-01T17:35:52.705Z{"msg":"Upstairs opts: Upstairs UUID: b5c49dd9-f0f0-40c0-a59f-d7c3698a4130, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138757087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195322024-02-01T17:35:52.705Z{"msg":"Crucible stats registered with UUID: b5c49dd9-f0f0-40c0-a59f-d7c3698a4130","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138830252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195332024-02-01T17:35:52.705Z{"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138871816Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
195342024-02-01T17:35:52.705Z{"msg":"b5c49dd9-f0f0-40c0-a59f-d7c3698a4130 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.138973268Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195352024-02-01T17:35:52.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139031254Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195362024-02-01T17:35:52.705Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139085796Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195372024-02-01T17:35:52.705Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139160797Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195382024-02-01T17:35:52.707Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139210591Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195392024-02-01T17:35:52.707Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139251951Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195402024-02-01T17:35:52.707Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139338851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195412024-02-01T17:35:52.707Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139384397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195422024-02-01T17:35:52.707Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.13942817Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195432024-02-01T17:35:52.707Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139567618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195442024-02-01T17:35:52.707Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139618989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195452024-02-01T17:35:52.707Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139694926Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195462024-02-01T17:35:52.707Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.139739437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"io task","client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195472024-02-01T17:35:52.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.140912214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195482024-02-01T17:35:52.707Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.141006749Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195492024-02-01T17:35:52.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.141050158Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195502024-02-01T17:35:52.707Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.141184405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195512024-02-01T17:35:52.707Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.141352003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195522024-02-01T17:35:52.707Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.14142019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195532024-02-01T17:35:52.707Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.142077126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195542024-02-01T17:35:52.707Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.142138933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195552024-02-01T17:35:52.707Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.142176834Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195562024-02-01T17:35:52.707Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143073022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195572024-02-01T17:35:52.707Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143161799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195582024-02-01T17:35:52.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143189285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195592024-02-01T17:35:52.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143312244Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195602024-02-01T17:35:52.708Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143373432Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195612024-02-01T17:35:52.708Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143400091Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195622024-02-01T17:35:52.708Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143440042Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195632024-02-01T17:35:52.708Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143464594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195642024-02-01T17:35:52.708Z{"msg":"[0]R flush_numbers[0..12]: [294, 297, 297, 297, 292, 297, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143505984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195652024-02-01T17:35:52.708Z{"msg":"[0]R generation[0..12]: [178, 180, 180, 180, 176, 180, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143531346Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195662024-02-01T17:35:52.708Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143555412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195672024-02-01T17:35:52.708Z{"msg":"[1]R flush_numbers[0..12]: [294, 297, 297, 297, 292, 297, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143598329Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195682024-02-01T17:35:52.708Z{"msg":"[1]R generation[0..12]: [178, 180, 180, 180, 176, 180, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143623389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195692024-02-01T17:35:52.708Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
195702024-02-01T17:35:52.708Z,"time":"2024-02-01T17:35:51.143692576Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195712024-02-01T17:35:52.708Z{"msg":"[2]R flush_numbers[0..12]: [294, 297, 297, 297, 292, 297, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143738723Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195722024-02-01T17:35:52.708Z{"msg":"[2]R generation[0..12]: [178, 180, 180, 180, 176, 180, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.14376431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195732024-02-01T17:35:52.708Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143817823Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195742024-02-01T17:35:52.709Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143841393Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195752024-02-01T17:35:52.709Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143884613Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195762024-02-01T17:35:52.709Z{"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.14391605Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195772024-02-01T17:35:52.709Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.1439625Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"":"downstairs","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195782024-02-01T17:35:52.709Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.143987498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195792024-02-01T17:35:52.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144038331Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"0","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195802024-02-01T17:35:52.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144064525Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"1","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195812024-02-01T17:35:52.709Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144086285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"client":"2","":"downstairs","session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195822024-02-01T17:35:52.709Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144134944Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195832024-02-01T17:35:52.709Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144157421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195842024-02-01T17:35:52.709Z{"msg":"b5c49dd9-f0f0-40c0-a59f-d7c3698a4130 is now active with session: 91c189a7-3d15-4f28-8508-8f6c6600cf90","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.14418049Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195852024-02-01T17:35:52.709Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144225467Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860,"session_id":"91c189a7-3d15-4f28-8508-8f6c6600cf90"}
195862024-02-01T17:35:52.709Z{"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195872024-02-01T17:35:52.710Zmsg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:51.144252013Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
195882024-02-01T17:35:52.710ZLoading write count information from file "/tmp/test_repair_verify.data"
195892024-02-01T17:35:52.710ZRun Repair workload
195902024-02-01T17:35:52.710Z01/30 Write block 492 len 24576 data: 23 18 16 18 15 14
195912024-02-01T17:35:52.710Z02/30 Write block 102 len 12288 data: 17 15 17
195922024-02-01T17:35:52.710Z03/30 Read block 46 len 28672
195932024-02-01T17:35:52.710Z04/30 Read block 591 len 28672
195942024-02-01T17:35:52.710Z05/30 Read block 433 len 20480
195952024-02-01T17:35:52.710Z06/30 Flush
195962024-02-01T17:35:52.710Z07/30 Write block 544 len 12288 data: 19 14 16
195972024-02-01T17:35:52.710Z08/30 Read block 129 len 32768
195982024-02-01T17:35:52.710Z09/30 Read block 290 len 4096
195992024-02-01T17:35:52.711Z10/30 Read block 139 len 28672
196002024-02-01T17:35:52.711Z11/30 Write block 83 len 36864 data: 10 10 10 10 9 9 8 9 8
196012024-02-01T17:35:52.711Z12/30 Flush
196022024-02-01T17:35:52.711Z13/30 Flush
196032024-02-01T17:35:52.711Z14/30 Write block 500 len 40960 data: 15 13 13 12 11 9 9 11 13 14
196042024-02-01T17:35:52.711Z15/30 Write block 430 len 28672 data: 11 11 17 20 18 20 17
196052024-02-01T17:35:52.711Z16/30 Read block 210 len 12288
196062024-02-01T17:35:52.711Z17/30 Write block 430 len 12288 data: 12 12 18
196072024-02-01T17:35:52.711Z18/30 Read block 567 len 12288
196082024-02-01T17:35:52.711Z19/30 Write block 523 len 4096 data: 10
196092024-02-01T17:35:52.711Z20/30 Write block 527 len 8192 data: 6 6
196102024-02-01T17:35:52.711Z21/30 Read block 156 len 12288
196112024-02-01T17:35:52.711Z22/30 Write block 268 len 12288 data: 16 14 14
196122024-02-01T17:35:52.711Z23/30 Flush
196132024-02-01T17:35:52.711Z24/30 Write block 391 len 20480 data: 12 14 13 11 9
196142024-02-01T17:35:52.711Z25/30 Read block 562 len 24576
196152024-02-01T17:35:52.711Z26/30 Read block 91 len 40960
196162024-02-01T17:35:52.711Z27/30 Write block 386 len 16384 data: 8 9 9 11
196172024-02-01T17:35:52.712Z28/30 Write block 362 len 28672 data: 20 17 16 14 15 11 9
196182024-02-01T17:35:52.712Z29/30 Read block 66 len 36864
196192024-02-01T17:35:52.712Z30/30 Read block 211 len 24576
196202024-02-01T17:35:52.712Z----------------------------------------------------------------
196212024-02-01T17:35:52.712Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
196222024-02-01T17:35:52.712ZStates: Active Active Active
196232024-02-01T17:35:52.712ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196242024-02-01T17:35:52.712Z 1 Acked 1000 Write 6 Done Sent Done false
196252024-02-01T17:35:52.712Z 2 Acked 1001 Write 3 Done Sent Done false
196262024-02-01T17:35:52.712Z 3 Acked 1002 Read 7 Done Sent Done false
196272024-02-01T17:35:52.712Z 4 Acked 1003 Read 7 Done Sent Done false
196282024-02-01T17:35:52.718Z 5 Acked 1004 Read 5 Done Sent Done false
196292024-02-01T17:35:52.718Z 6 Acked 1005 Flush 0 Done Sent Done false
196302024-02-01T17:35:52.718Z 7 Acked 1006 Write 3 Done Sent Done false
196312024-02-01T17:35:52.719Z 8 Acked 1007 Read 8 Done Sent Done false
196322024-02-01T17:35:52.719Z 9 Acked 1008 Read 1 Done Sent Done false
196332024-02-01T17:35:52.719Z 10 Acked 1009 Read 7 Done Sent Done false
196342024-02-01T17:35:52.719Z 11 Acked 1010 Write 9 Done Sent Done false
196352024-02-01T17:35:52.719Z 12 Acked 1011 Flush 0 Done Sent Done false
196362024-02-01T17:35:52.719Z 13 Acked 1012 Flush 0 Done Sent Done false
196372024-02-01T17:35:52.719Z 14 Acked 1013 Write 10 Done Sent Done false
196382024-02-01T17:35:52.719Z 15 Acked 1014 Write 7 Done Sent Done false
196392024-02-01T17:35:52.719Z 16 Acked 1015 Read 3 Done Sent Done false
196402024-02-01T17:35:52.719Z 17 Acked 1016 Write 3 Done Sent Done false
196412024-02-01T17:35:52.719Z 18 Acked 1017 Read 3 Done Sent Done false
196422024-02-01T17:35:52.719Z 19 Acked 1018 Write 1 Done Sent Done false
196432024-02-01T17:35:52.719Z 20 Acked 1019 Write 2 Done Sent Done false
196442024-02-01T17:35:52.719Z 21 Acked 1020 Read 3 Done Sent Done false
196452024-02-01T17:35:52.719Z 22 Acked 1021 Write 3 Done Sent Done false
196462024-02-01T17:35:52.719Z 23 Acked 1022 Flush 0 Done Sent Done false
196472024-02-01T17:35:52.719Z 24 Acked 1023 Write 5 Done Sent Done false
196482024-02-01T17:35:52.719Z 25 Acked 1024 Read 6 Done Sent Done false
196492024-02-01T17:35:52.719Z 26 Acked 1025 Read 10 Done Sent Done false
196502024-02-01T17:35:52.719Z 27 Acked 1026 Write 4 Done Sent Done false
196512024-02-01T17:35:52.719Z 28 Acked 1027 Write 7 Done Sent Done false
196522024-02-01T17:35:52.719Z 29 Acked 1028 Read 9 Done Sent Done false
196532024-02-01T17:35:52.719Z 30 Acked 1029 Read 6 Sent Sent Done false
196542024-02-01T17:35:52.720Z STATES DS:0 DS:1 DS:2 TOTAL
196552024-02-01T17:35:52.720Z New 0 0 0 0
196562024-02-01T17:35:52.720Z Sent 1 30 0 31
196572024-02-01T17:35:52.720Z Done 29 0 30 59
196582024-02-01T17:35:52.720Z Skipped 0 0 0 0
196592024-02-01T17:35:52.720Z Error 0 0 0 0
196602024-02-01T17:35:52.720ZLast Flush: 1022 0 1022
196612024-02-01T17:35:52.720ZDownstairs last five completed:
196622024-02-01T17:35:52.720ZUpstairs last five completed: 30 29 28 27 26
196632024-02-01T17:35:52.720Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:51.191286691Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1860}
196642024-02-01T17:35:52.720ZWrote out file "/tmp/test_repair_verify.data"
196652024-02-01T17:35:52.721Z
196662024-02-01T17:35:52.721ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
196672024-02-01T17:35:52.721ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196682024-02-01T17:35:52.721Z 0 000-019 178 178 178 294 294 294 F F F
196692024-02-01T17:35:52.721Z 1 020-039 180 180 180 297 297 297 F F F
196702024-02-01T17:35:52.721Z 2 040-059 180 180 180 297 297 297 F F F
196712024-02-01T17:35:52.721Z 3 060-079 180 180 180 297 297 297 F F F
196722024-02-01T17:35:52.721Z 4 080-099 182 176 182 300 292 300 F F F <---
196732024-02-01T17:35:52.721Z 5 100-119 182 180 182 299 297 299 F F F <---
196742024-02-01T17:35:52.721Z 6 120-139 180 180 180 296 296 296 F F F
196752024-02-01T17:35:52.722Z 7 140-159 181 181 181 298 298 298 F F F
196762024-02-01T17:35:52.722Z 8 160-179 180 180 180 297 297 297 F F F
196772024-02-01T17:35:52.722Z 9 180-199 173 173 173 283 283 283 F F F
196782024-02-01T17:35:52.722Z 10 200-219 180 180 180 297 297 297 F F F
196792024-02-01T17:35:52.722Z 11 220-239 180 180 180 297 297 297 F F F
196802024-02-01T17:35:52.722Z 12 240-259 174 174 174 285 285 285 F F F
196812024-02-01T17:35:52.722Z 13 260-279 182 168 182 302 274 302 F F F <---
196822024-02-01T17:35:52.722Z 14 280-299 178 178 178 294 294 294 F F F
196832024-02-01T17:35:52.722Z 15 300-319 180 180 180 297 297 297 F F F
196842024-02-01T17:35:52.722Z 16 320-339 178 178 178 294 294 294 F F F
196852024-02-01T17:35:52.722Z 17 340-359 178 178 178 294 294 294 F F F
196862024-02-01T17:35:52.722Z 18 360-379 171 171 171 279 279 279 T F T <---
196872024-02-01T17:35:52.722Z 19 380-399 174 174 174 289 289 289 T F T <---
196882024-02-01T17:35:52.722Z 20 400-419 181 181 181 298 298 298 F F F
196892024-02-01T17:35:52.722Z 21 420-439 182 180 182 302 296 302 F F F <---
196902024-02-01T17:35:52.722Z 22 440-459 181 181 181 298 298 298 F F F
196912024-02-01T17:35:52.722Z 23 460-479 181 181 181 298 298 298 F F F
196922024-02-01T17:35:52.722Z 24 480-499 182 178 182 299 294 299 F F F <---
196932024-02-01T17:35:52.722Z 25 500-519 182 177 182 302 293 302 F F F <---
196942024-02-01T17:35:52.722Z 26 520-539 182 177 182 302 293 302 F F F <---
196952024-02-01T17:35:52.722Z 27 540-559 182 178 182 300 294 300 F F F <---
196962024-02-01T17:35:52.722Z 28 560-579 177 177 177 293 293 293 F F F
196972024-02-01T17:35:52.722Z 29 580-599 178 178 178 294 294 294 F F F
196982024-02-01T17:35:52.722ZMax gen: 182, Max flush: 302
196992024-02-01T17:35:52.722ZError: Difference in extent metadata found!
197002024-02-01T17:35:52.723ZA Difference in extent metadata is expected here
197012024-02-01T17:35:52.723ZOn loop 91
197022024-02-01T17:35:52.723Z
197032024-02-01T17:35:52.723ZVerifying data now
197042024-02-01T17:35:56.859ZNo change after verify
197052024-02-01T17:35:56.860ZLoop: 91 Downstairs dump after verify (and repair):
197062024-02-01T17:35:56.880ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197072024-02-01T17:35:56.880Z 0 000-019 178 178 178 294 294 294 F F F
197082024-02-01T17:35:56.885Z 1 020-039 180 180 180 297 297 297 F F F
197092024-02-01T17:35:56.885Z 2 040-059 180 180 180 297 297 297 F F F
197102024-02-01T17:35:56.885Z 3 060-079 180 180 180 297 297 297 F F F
197112024-02-01T17:35:56.885Z 4 080-099 182 182 182 300 300 300 F F F
197122024-02-01T17:35:56.885Z 5 100-119 182 182 182 299 299 299 F F F
197132024-02-01T17:35:56.885Z 6 120-139 180 180 180 296 296 296 F F F
197142024-02-01T17:35:56.885Z 7 140-159 181 181 181 298 298 298 F F F
197152024-02-01T17:35:56.886Z 8 160-179 180 180 180 297 297 297 F F F
197162024-02-01T17:35:56.886Z 9 180-199 173 173 173 283 283 283 F F F
197172024-02-01T17:35:56.886Z 10 200-219 180 180 180 297 297 297 F F F
197182024-02-01T17:35:56.886Z 11 220-239 180 180 180 297 297 297 F F F
197192024-02-01T17:35:56.886Z 12 240-259 174 174 174 285 285 285 F F F
197202024-02-01T17:35:56.886Z 13 260-279 182 182 182 302 302 302 F F F
197212024-02-01T17:35:56.886Z 14 280-299 178 178 178 294 294 294 F F F
197222024-02-01T17:35:56.886Z 15 300-319 180 180 180 297 297 297 F F F
197232024-02-01T17:35:56.886Z 16 320-339 178 178 178 294 294 294 F F F
197242024-02-01T17:35:56.886Z 17 340-359 178 178 178 294 294 294 F F F
197252024-02-01T17:35:56.886Z 18 360-379 183 183 183 303 303 303 F F F
197262024-02-01T17:35:56.886Z 19 380-399 183 183 183 303 303 303 F F F
197272024-02-01T17:35:56.886Z 20 400-419 181 181 181 298 298 298 F F F
197282024-02-01T17:35:56.886Z 21 420-439 182 182 182 302 302 302 F F F
197292024-02-01T17:35:56.886Z 22 440-459 181 181 181 298 298 298 F F F
197302024-02-01T17:35:56.886Z 23 460-479 181 181 181 298 298 298 F F F
197312024-02-01T17:35:56.886Z 24 480-499 182 182 182 299 299 299 F F F
197322024-02-01T17:35:56.886Z 25 500-519 182 182 182 302 302 302 F F F
197332024-02-01T17:35:56.886Z 26 520-539 182 182 182 302 302 302 F F F
197342024-02-01T17:35:56.886Z 27 540-559 182 182 182 300 300 300 F F F
197352024-02-01T17:35:56.886Z 28 560-579 177 177 177 293 293 293 F F F
197362024-02-01T17:35:56.887Z 29 580-599 178 178 178 294 294 294 F F F
197372024-02-01T17:35:56.887ZMax gen: 183, Max flush: 303
197382024-02-01T17:35:56.887Z
197392024-02-01T17:35:56.887ZBegin loop 92
197402024-02-01T17:35:56.887ZDownstairs to restart: 2
197412024-02-01T17:35:57.323Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.200114364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
197422024-02-01T17:35:57.323Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.200651289Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
197432024-02-01T17:35:57.332Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.200718232Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
197442024-02-01T17:35:57.332Z{"msg":"Crucible 24f530d6-e897-4795-88d0-baaf3e3f1565 has session id: 7bd9dacb-ad51-4c96-866c-6f21fa36354d","v":0,"name":"crucible","level":30Crucible runtime is spawned
197452024-02-01T17:35:57.332Z,"time":"2024-02-01T17:35:56.200871822Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197462024-02-01T17:35:57.332Z{"msg":"Upstairs opts: Upstairs UUID: 24f530d6-e897-4795-88d0-baaf3e3f1565, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201170347Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197472024-02-01T17:35:57.332Z{"msg":"Crucible stats registered with UUID: 24f530d6-e897-4795-88d0-baaf3e3f1565","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201243421Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197482024-02-01T17:35:57.332Z{"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201315768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
197492024-02-01T17:35:57.333Z{"msg":"24f530d6-e897-4795-88d0-baaf3e3f1565 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.20143957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197502024-02-01T17:35:57.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201522938Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197512024-02-01T17:35:57.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201610113Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197522024-02-01T17:35:57.333Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201697324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197532024-02-01T17:35:57.333Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201745916Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197542024-02-01T17:35:57.334Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201902596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197552024-02-01T17:35:57.334Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.201934815Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197562024-02-01T17:35:57.334Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.202009178Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197572024-02-01T17:35:57.334Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.202086863Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197582024-02-01T17:35:57.334Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.202127481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197592024-02-01T17:35:57.334Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.202186372Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197602024-02-01T17:35:57.336Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.202392922Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197612024-02-01T17:35:57.336Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.20246028Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"io task","client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197622024-02-01T17:35:57.336Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203337545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197632024-02-01T17:35:57.336Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203409621Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197642024-02-01T17:35:57.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203587485Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197652024-02-01T17:35:57.337Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203646897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197662024-02-01T17:35:57.337Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203691367Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197672024-02-01T17:35:57.337Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.203731671Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197682024-02-01T17:35:57.337Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.204391991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197692024-02-01T17:35:57.337Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.204627266Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197702024-02-01T17:35:57.337Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205126546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197712024-02-01T17:35:57.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205461405Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197722024-02-01T17:35:57.337Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205549419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197732024-02-01T17:35:57.337Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205619608Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197742024-02-01T17:35:57.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205648414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197752024-02-01T17:35:57.337Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.20569092Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197762024-02-01T17:35:57.337Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205715857Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197772024-02-01T17:35:57.337Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.205951847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197782024-02-01T17:35:57.337Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206024368Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197792024-02-01T17:35:57.339Z{"msg":"[0]R flush_numbers[0..12]: [294, 297, 297, 297, 300, 299, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206055161Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197802024-02-01T17:35:57.339Z{"msg":"[0]R generation[0..12]: [178, 180, 180, 180, 182, 182, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206099094Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197812024-02-01T17:35:57.339Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206125537Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197822024-02-01T17:35:57.339Z{"msg":"[1]R flush_numbers[0..12]: [294, 297, 297, 297, 300, 299, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206167558Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197832024-02-01T17:35:57.339Z{"msg":"[1]R generation[0..12]: [178, 180, 180, 180, 182, 182, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206192428Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197842024-02-01T17:35:57.339Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206270762Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197852024-02-01T17:35:57.339ZWait for a query_work_queue command to finish before sending IO
197862024-02-01T17:35:57.339Z{"msg":"[2]R flush_numbers[0..12]: [294, 297, 297, 297, 300, 299, 296, 298, 297, 283, 297, 297]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206319416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197872024-02-01T17:35:57.339Z{"msg":"[2]R generation[0..12]: [178, 180, 180, 180, 182, 182, 180, 181, 180, 173, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206376875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197882024-02-01T17:35:57.339Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206421806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197892024-02-01T17:35:57.339Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206475435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197902024-02-01T17:35:57.339Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206517618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197912024-02-01T17:35:57.339Z{"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206582946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197922024-02-01T17:35:57.339Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206622709Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"":"downstairs","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197932024-02-01T17:35:57.339Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206695164Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197942024-02-01T17:35:57.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206734862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"0","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197952024-02-01T17:35:57.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206788517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"1","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197962024-02-01T17:35:57.339Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206830507Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"client":"2","":"downstairs","session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197972024-02-01T17:35:57.339Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197982024-02-01T17:35:57.339Z,"time":"2024-02-01T17:35:56.206892923Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
197992024-02-01T17:35:57.339Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206935572Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
198002024-02-01T17:35:57.340Z{"msg":"24f530d6-e897-4795-88d0-baaf3e3f1565 is now active with session: 7bd9dacb-ad51-4c96-866c-6f21fa36354d","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.206991862Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
198012024-02-01T17:35:57.340Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.20703139Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868,"session_id":"7bd9dacb-ad51-4c96-866c-6f21fa36354d"}
198022024-02-01T17:35:57.340Z{"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:35:56.20708929Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
198032024-02-01T17:35:57.340ZLoading write count information from file "/tmp/test_repair_verify.data"
198042024-02-01T17:35:57.340ZRun Repair workload
198052024-02-01T17:35:57.340Z01/30 Write block 594 len 12288 data: 7 7 5
198062024-02-01T17:35:57.340Z02/30 Read block 243 len 12288
198072024-02-01T17:35:57.340Z03/30 Write block 167 len 12288 data: 11 14 17
198082024-02-01T17:35:57.340Z04/30 Read block 421 len 36864
198092024-02-01T17:35:57.340Z05/30 Write block 9 len 40960 data: 12 14 12 11 10 10 10 13 15 16
198102024-02-01T17:35:57.340Z06/30 Read block 66 len 36864
198112024-02-01T17:35:57.340Z07/30 Read block 580 len 4096
198122024-02-01T17:35:57.340Z08/30 Read block 85 len 24576
198132024-02-01T17:35:57.340Z09/30 Read block 587 len 32768
198142024-02-01T17:35:57.340Z10/30 Read block 16 len 20480
198152024-02-01T17:35:57.340Z11/30 Read block 444 len 20480
198162024-02-01T17:35:57.340Z12/30 Read block 482 len 40960
198172024-02-01T17:35:57.340Z13/30 Read block 235 len 28672
198182024-02-01T17:35:57.340Z14/30 Flush
198192024-02-01T17:35:57.340Z15/30 Flush
198202024-02-01T17:35:57.340Z16/30 Write block 181 len 36864 data: 16 19 18 18 20 17 16 11 13
198212024-02-01T17:35:57.340Z17/30 Read block 525 len 8192
198222024-02-01T17:35:57.340Z18/30 Read block 18 len 40960
198232024-02-01T17:35:57.341Z19/30 Read block 50 len 40960
198242024-02-01T17:35:57.341Z20/30 Write block 452 len 36864 data: 10 10 10 10 14 20 20 21 18
198252024-02-01T17:35:57.341Z21/30 Read block 10 len 4096
198262024-02-01T17:35:57.341Z22/30 Write block 222 len 28672 data: 17 19 18 12 10 8 8
198272024-02-01T17:35:57.341Z23/30 Read block 342 len 8192
198282024-02-01T17:35:57.341Z24/30 Read block 319 len 32768
198292024-02-01T17:35:57.341Z25/30 Read block 69 len 4096
198302024-02-01T17:35:57.341Z26/30 Flush
198312024-02-01T17:35:57.341Z27/30 Write block 284 len 16384 data: 16 19 18 14
198322024-02-01T17:35:57.341Z28/30 Write block 146 len 16384 data: 12 13 17 17
198332024-02-01T17:35:57.341Z29/30 Write block 578 len 4096 data: 13
198342024-02-01T17:35:57.341Z30/30 Read block 310 len 4096
198352024-02-01T17:35:57.341Z----------------------------------------------------------------
198362024-02-01T17:35:57.341Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30
198372024-02-01T17:35:57.341ZStates: Active Active Active
198382024-02-01T17:35:57.341ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198392024-02-01T17:35:57.341Z 1 Acked 1000 Write 3 Done Done Sent false
198402024-02-01T17:35:57.341Z 2 Acked 1001 Read 3 Done Done Sent false
198412024-02-01T17:35:57.341Z 3 Acked 1002 Write 3 Done Done Sent false
198422024-02-01T17:35:57.341Z 4 Acked 1003 Read 9 Done Done Sent false
198432024-02-01T17:35:57.346Z 5 Acked 1004 Write 10 Done Done Sent false
198442024-02-01T17:35:57.346Z 6 Acked 1005 Read 9 Done Done Sent false
198452024-02-01T17:35:57.346Z 7 Acked 1006 Read 1 Done Done Sent false
198462024-02-01T17:35:57.346Z 8 Acked 1007 Read 6 Done Done Sent false
198472024-02-01T17:35:57.346Z 9 Acked 1008 Read 8 Done Done Sent false
198482024-02-01T17:35:57.346Z 10 Acked 1009 Read 5 Done Done Sent false
198492024-02-01T17:35:57.346Z 11 Acked 1010 Read 5 Done Done Sent false
198502024-02-01T17:35:57.347Z 12 Acked 1011 Read 10 Done Done Sent false
198512024-02-01T17:35:57.347Z 13 Acked 1012 Read 7 Done Done Sent false
198522024-02-01T17:35:57.347Z 14 Acked 1013 Flush 0 Done Done Sent false
198532024-02-01T17:35:57.347Z 15 Acked 1014 Flush 0 Done Done Sent false
198542024-02-01T17:35:57.347Z 16 Acked 1015 Write 9 Done Done Sent false
198552024-02-01T17:35:57.347Z 17 Acked 1016 Read 2 Done Done Sent false
198562024-02-01T17:35:57.347Z 18 Acked 1017 Read 10 Done Done Sent false
198572024-02-01T17:35:57.347Z 19 Acked 1018 Read 10 Done Done Sent false
198582024-02-01T17:35:57.347Z 20 Acked 1019 Write 9 Done Done Sent false
198592024-02-01T17:35:57.347Z 21 Acked 1020 Read 1 Done Done Sent false
198602024-02-01T17:35:57.347Z 22 Acked 1021 Write 7 Done Done Sent false
198612024-02-01T17:35:57.347Z 23 Acked 1022 Read 2 Done Done Sent false
198622024-02-01T17:35:57.347Z 24 Acked 1023 Read 8 Done Done Sent false
198632024-02-01T17:35:57.347Z 25 Acked 1024 Read 1 Done Done Sent false
198642024-02-01T17:35:57.347Z 26 Acked 1025 Flush 0 Done Done Sent false
198652024-02-01T17:35:57.347Z 27 Acked 1026 Write 4 Done Done Sent false
198662024-02-01T17:35:57.348Z 28 Acked 1027 Write 4 Done Done Sent false
198672024-02-01T17:35:57.348Z 29 Acked 1028 Write 1 Done Done Sent false
198682024-02-01T17:35:57.349Z 30 Acked 1029 Read 1 Done Sent Sent false
198692024-02-01T17:35:57.349Z STATES DS:0 DS:1 DS:2 TOTAL
198702024-02-01T17:35:57.349Z New 0 0 0 0
198712024-02-01T17:35:57.349Z Sent 0 1 30 31
198722024-02-01T17:35:57.349Z Done 30 29 0 59
198732024-02-01T17:35:57.349Z Skipped 0 0 0 0
198742024-02-01T17:35:57.349Z Error 0 0 0 0
198752024-02-01T17:35:57.350ZLast Flush: 1025 1025 0
198762024-02-01T17:35:57.350ZDownstairs last five completed:
198772024-02-01T17:35:57.350ZUpstairs last five completed: 30 29 28 27 26
198782024-02-01T17:35:57.350Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:35:56.252711629Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1868}
198792024-02-01T17:35:57.350ZWrote out file "/tmp/test_repair_verify.data"
198802024-02-01T17:35:57.350Z
198812024-02-01T17:35:57.350ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198822024-02-01T17:35:57.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198832024-02-01T17:35:57.350Z 0 000-019 184 184 178 304 304 294 F F F <---
198842024-02-01T17:35:57.350Z 1 020-039 180 180 180 297 297 297 F F F
198852024-02-01T17:35:57.350Z 2 040-059 180 180 180 297 297 297 F F F
198862024-02-01T17:35:57.350Z 3 060-079 180 180 180 297 297 297 F F F
198872024-02-01T17:35:57.350Z 4 080-099 182 182 182 300 300 300 F F F
198882024-02-01T17:35:57.350Z 5 100-119 182 182 182 299 299 299 F F F
198892024-02-01T17:35:57.350Z 6 120-139 180 180 180 296 296 296 F F F
198902024-02-01T17:35:57.350Z 7 140-159 181 181 181 298 298 298 T T F <---
198912024-02-01T17:35:57.350Z 8 160-179 184 184 180 304 304 297 F F F <---
198922024-02-01T17:35:57.351Z 9 180-199 184 184 173 306 306 283 F F F <---
198932024-02-01T17:35:57.351Z 10 200-219 180 180 180 297 297 297 F F F
198942024-02-01T17:35:57.351Z 11 220-239 184 184 180 306 306 297 F F F <---
198952024-02-01T17:35:57.351Z 12 240-259 174 174 174 285 285 285 F F F
198962024-02-01T17:35:57.351Z 13 260-279 182 182 182 302 302 302 F F F
198972024-02-01T17:35:57.352Z 14 280-299 178 178 178 294 294 294 T T F <---
198982024-02-01T17:35:57.352Z 15 300-319 180 180 180 297 297 297 F F F
198992024-02-01T17:35:57.352Z 16 320-339 178 178 178 294 294 294 F F F
199002024-02-01T17:35:57.352Z 17 340-359 178 178 178 294 294 294 F F F
199012024-02-01T17:35:57.352Z 18 360-379 183 183 183 303 303 303 F F F
199022024-02-01T17:35:57.352Z 19 380-399 183 183 183 303 303 303 F F F
199032024-02-01T17:35:57.352Z 20 400-419 181 181 181 298 298 298 F F F
199042024-02-01T17:35:57.352Z 21 420-439 182 182 182 302 302 302 F F F
199052024-02-01T17:35:57.352Z 22 440-459 184 184 181 306 306 298 F F F <---
199062024-02-01T17:35:57.352Z 23 460-479 184 184 181 306 306 298 F F F <---
199072024-02-01T17:35:57.352Z 24 480-499 182 182 182 299 299 299 F F F
199082024-02-01T17:35:57.352Z 25 500-519 182 182 182 302 302 302 F F F
199092024-02-01T17:35:57.352Z 26 520-539 182 182 182 302 302 302 F F F
199102024-02-01T17:35:57.352Z 27 540-559 182 182 182 300 300 300 F F F
199112024-02-01T17:35:57.352Z 28 560-579 177 177 177 293 293 293 T T F <---
199122024-02-01T17:35:57.352Z 29 580-599 184 184 178 304 304 294 F F F <---
199132024-02-01T17:35:57.352ZMax gen: 184, Max flush: 306
199142024-02-01T17:35:57.352ZError: Difference in extent metadata found!
199152024-02-01T17:35:57.352ZA Difference in extent metadata is expected here
199162024-02-01T17:35:57.352ZOn loop 92
199172024-02-01T17:35:57.352Z
199182024-02-01T17:35:57.352ZVerifying data now
199192024-02-01T17:36:01.897ZNo change after verify
199202024-02-01T17:36:01.897ZLoop: 92 Downstairs dump after verify (and repair):
199212024-02-01T17:36:01.930ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199222024-02-01T17:36:01.931Z 0 000-019 184 184 184 304 304 304 F F F
199232024-02-01T17:36:01.931Z 1 020-039 180 180 180 297 297 297 F F F
199242024-02-01T17:36:01.931Z 2 040-059 180 180 180 297 297 297 F F F
199252024-02-01T17:36:01.931Z 3 060-079 180 180 180 297 297 297 F F F
199262024-02-01T17:36:01.931Z 4 080-099 182 182 182 300 300 300 F F F
199272024-02-01T17:36:01.931Z 5 100-119 182 182 182 299 299 299 F F F
199282024-02-01T17:36:01.931Z 6 120-139 180 180 180 296 296 296 F F F
199292024-02-01T17:36:01.948Z 7 140-159 185 185 185 307 307 307 F F F
199302024-02-01T17:36:01.948Z 8 160-179 184 184 184 304 304 304 F F F
199312024-02-01T17:36:01.949Z 9 180-199 184 184 184 306 306 306 F F F
199322024-02-01T17:36:01.949Z 10 200-219 180 180 180 297 297 297 F F F
199332024-02-01T17:36:01.949Z 11 220-239 184 184 184 306 306 306 F F F
199342024-02-01T17:36:01.949Z 12 240-259 174 174 174 285 285 285 F F F
199352024-02-01T17:36:01.950Z 13 260-279 182 182 182 302 302 302 F F F
199362024-02-01T17:36:01.950Z 14 280-299 185 185 185 307 307 307 F F F
199372024-02-01T17:36:01.950Z 15 300-319 180 180 180 297 297 297 F F F
199382024-02-01T17:36:01.950Z 16 320-339 178 178 178 294 294 294 F F F
199392024-02-01T17:36:01.950Z 17 340-359 178 178 178 294 294 294 F F F
199402024-02-01T17:36:01.950Z 18 360-379 183 183 183 303 303 303 F F F
199412024-02-01T17:36:01.951Z 19 380-399 183 183 183 303 303 303 F F F
199422024-02-01T17:36:01.951Z 20 400-419 181 181 181 298 298 298 F F F
199432024-02-01T17:36:01.951Z 21 420-439 182 182 182 302 302 302 F F F
199442024-02-01T17:36:01.951Z 22 440-459 184 184 184 306 306 306 F F F
199452024-02-01T17:36:01.952Z 23 460-479 184 184 184 306 306 306 F F F
199462024-02-01T17:36:01.953Z 24 480-499 182 182 182 299 299 299 F F F
199472024-02-01T17:36:01.953Z 25 500-519 182 182 182 302 302 302 F F F
199482024-02-01T17:36:01.953Z 26 520-539 182 182 182 302 302 302 F F F
199492024-02-01T17:36:01.954Z 27 540-559 182 182 182 300 300 300 F F F
199502024-02-01T17:36:01.954Z 28 560-579 185 185 185 307 307 307 F F F
199512024-02-01T17:36:01.954Z 29 580-599 184 184 184 304 304 304 F F F
199522024-02-01T17:36:01.954ZMax gen: 185, Max flush: 307
199532024-02-01T17:36:01.954Z
199542024-02-01T17:36:01.955ZBegin loop 93
199552024-02-01T17:36:01.956ZDownstairs to restart: 0
199562024-02-01T17:36:01.966Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.238411122Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
199572024-02-01T17:36:01.966Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.238947826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
199582024-02-01T17:36:01.966Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
199592024-02-01T17:36:01.966Z,"time":"2024-02-01T17:36:01.238993126Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
199602024-02-01T17:36:01.967Z{"msg":"Crucible e2e4535b-d12b-4a19-8aed-6b371c6a40c4 has session id: 14791cab-85de-4fd9-ad1d-0351726907b8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239277787Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199612024-02-01T17:36:01.967Z{"msg":"Upstairs opts: Upstairs UUID: e2e4535b-d12b-4a19-8aed-6b371c6a40c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239365617Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199622024-02-01T17:36:01.967Z{"msg":"Crucible stats registered with UUID: e2e4535b-d12b-4a19-8aed-6b371c6a40c4","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239438856Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199632024-02-01T17:36:01.967Z{"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239480957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
199642024-02-01T17:36:01.967Z{"msg":"e2e4535b-d12b-4a19-8aed-6b371c6a40c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239567376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199652024-02-01T17:36:01.967Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239639172Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199662024-02-01T17:36:01.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239696273Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199672024-02-01T17:36:01.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239790794Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199682024-02-01T17:36:01.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239839711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199692024-02-01T17:36:01.968Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239909456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199702024-02-01T17:36:01.968Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.239982914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199712024-02-01T17:36:01.968Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240036943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199722024-02-01T17:36:01.968Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240074983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199732024-02-01T17:36:01.969Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240112051Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199742024-02-01T17:36:01.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240158162Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199752024-02-01T17:36:01.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240190248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199762024-02-01T17:36:01.969Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.240239643Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"io task","client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199772024-02-01T17:36:01.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.241530023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199782024-02-01T17:36:01.969Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.241585974Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199792024-02-01T17:36:01.969Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.241693144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199802024-02-01T17:36:01.970Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.241780344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199812024-02-01T17:36:01.970Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.24242087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199822024-02-01T17:36:01.970Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.242653241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199832024-02-01T17:36:01.970Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.242728914Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199842024-02-01T17:36:01.970Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.242788282Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199852024-02-01T17:36:01.970Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243317542Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199862024-02-01T17:36:01.970Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243390711Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199872024-02-01T17:36:01.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243465256Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199882024-02-01T17:36:01.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243620267Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199892024-02-01T17:36:01.971Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243672742Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199902024-02-01T17:36:01.971Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243698324Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199912024-02-01T17:36:01.971Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.243739071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199922024-02-01T17:36:01.971Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244464449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199932024-02-01T17:36:01.971Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244549176Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199942024-02-01T17:36:01.972Z{"msg":"[0]R flush_numbers[0..12]: [304, 297, 297, 297, 300, 299, 296, 307, 304, 306, 297, 306]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244623604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199952024-02-01T17:36:01.972Z{"msg":"[0]R generation[0..12]: [184, 180, 180, 180, 182, 182, 180, 185, 184, 184, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244697128Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199962024-02-01T17:36:01.975Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.24476925Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199972024-02-01T17:36:01.975Z{"msg":"[1]R flush_numbers[0..12]: [304, 297, 297, 297, 300, 299, 296, 307, 304, 306, 297, 306]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244840587Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs"Wait for a query_work_queue command to finish before sending IO
199982024-02-01T17:36:01.975Z,"":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
199992024-02-01T17:36:01.975Z{"msg":"[1]R generation[0..12]: [184, 180, 180, 180, 182, 182, 180, 185, 184, 184, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.244977026Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200002024-02-01T17:36:01.975Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245079874Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200012024-02-01T17:36:01.975Z{"msg":"[2]R flush_numbers[0..12]: [304, 297, 297, 297, 300, 299, 296, 307, 304, 306, 297, 306]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245155177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200022024-02-01T17:36:01.975Z{"msg":"[2]R generation[0..12]: [184, 180, 180, 180, 182, 182, 180, 185, 184, 184, 180, 184]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245228544Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200032024-02-01T17:36:01.975Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245301382Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200042024-02-01T17:36:01.975Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245370075Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200052024-02-01T17:36:01.975Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245415387Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200062024-02-01T17:36:01.976Z{"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245476519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200072024-02-01T17:36:01.977Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.24552251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"":"downstairs","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200082024-02-01T17:36:01.977Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245595229Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200092024-02-01T17:36:01.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.24564117Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"0","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200102024-02-01T17:36:01.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245675969Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"1","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200112024-02-01T17:36:01.977Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245745085Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"client":"2","":"downstairs","session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200122024-02-01T17:36:01.977Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245816819Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200132024-02-01T17:36:01.977Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.24588967Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200142024-02-01T17:36:01.977Z{"msg":"e2e4535b-d12b-4a19-8aed-6b371c6a40c4 is now active with session: 14791cab-85de-4fd9-ad1d-0351726907b8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.245935332Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200152024-02-01T17:36:01.977ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200162024-02-01T17:36:01.977Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.246042729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876,"session_id":"14791cab-85de-4fd9-ad1d-0351726907b8"}
200172024-02-01T17:36:01.978Z{"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:01.246071822Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
200182024-02-01T17:36:01.978ZLoading write count information from file "/tmp/test_repair_verify.data"
200192024-02-01T17:36:01.979ZRun Repair workload
200202024-02-01T17:36:02.923Z01/30 Write block 211 len 12288 data: 16 16 14
200212024-02-01T17:36:02.923Z02/30 Flush
200222024-02-01T17:36:02.924Z03/30 Write block 170 len 40960 data: 20 22 23 21 19 20 23 21 19 19
200232024-02-01T17:36:02.924Z04/30 Write block 313 len 4096 data: 18
200242024-02-01T17:36:02.924Z05/30 Write block 93 len 16384 data: 10 10 8 10
200252024-02-01T17:36:02.924Z06/30 Read block 321 len 24576
200262024-02-01T17:36:02.924Z07/30 Write block 339 len 28672 data: 14 13 16 15 17 16 13
200272024-02-01T17:36:02.924Z08/30 Write block 398 len 32768 data: 12 10 12 16 15 17 16 14
200282024-02-01T17:36:02.924Z09/30 Write block 363 len 12288 data: 18 17 15
200292024-02-01T17:36:02.924Z10/30 Write block 441 len 16384 data: 15 11 9 12
200302024-02-01T17:36:02.924Z11/30 Write block 11 len 32768 data: 13 12 11 11 11 14 16 17
200312024-02-01T17:36:02.924Z12/30 Write block 22 len 8192 data: 12 11
200322024-02-01T17:36:02.924Z13/30 Write block 314 len 4096 data: 18
200332024-02-01T17:36:02.924Z14/30 Write block 335 len 28672 data: 18 18 17 14 15 14 17
200342024-02-01T17:36:02.924Z15/30 Read block 341 len 32768
200352024-02-01T17:36:02.924Z16/30 Read block 415 len 28672
200362024-02-01T17:36:02.924Z17/30 Read block 576 len 20480
200372024-02-01T17:36:02.926Z18/30 Write block 13 len 32768 data: 12 12 12 15 17 18 13 15
200382024-02-01T17:36:02.927Z19/30 Flush
200392024-02-01T17:36:03.495Z20/30 Write block 507 len 20480 data: 12 14 15 12 14
200402024-02-01T17:36:03.495Z21/30 Read block 139 len 28672
200412024-02-01T17:36:03.495Z22/30 Read block 596 len 16384
200422024-02-01T17:36:03.495Z23/30 Write block 232 len 36864 data: 14 12 11 10 12 12 12 12 11
200432024-02-01T17:36:03.495Z24/30 Write block 123 len 4096 data: 14
200442024-02-01T17:36:03.496Z25/30 Read block 87 len 8192
200452024-02-01T17:36:03.496Z26/30 Read block 369 len 16384
200462024-02-01T17:36:03.496Z27/30 Read block 187 len 36864
200472024-02-01T17:36:03.496Z28/30 Write block 387 len 28672 data: 10 10 12 12 13 15 14
200482024-02-01T17:36:03.496Z29/30 Read block 494 len 20480
200492024-02-01T17:36:03.497Z30/30 Read block 512 len 4096
200502024-02-01T17:36:03.497Z----------------------------------------------------------------
200512024-02-01T17:36:03.497Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
200522024-02-01T17:36:03.497ZStates: Active Active Active
200532024-02-01T17:36:03.497ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200542024-02-01T17:36:03.497Z 1 Acked 1000 Write 3 Sent Done Done false
200552024-02-01T17:36:03.498Z 2 Acked 1001 Flush 0 Sent Done Done false
200562024-02-01T17:36:03.498Z 3 Acked 1002 Write 10 Sent Done Done false
200572024-02-01T17:36:03.498Z 4 Acked 1003 Write 1 Sent Done Done false
200582024-02-01T17:36:03.498Z 5 Acked 1004 Write 4 Sent Done Done false
200592024-02-01T17:36:03.498Z 6 Acked 1005 Read 6 Sent Done Done false
200602024-02-01T17:36:03.499Z 7 Acked 1006 Write 7 Sent Done Done false
200612024-02-01T17:36:03.499Z 8 Acked 1007 Write 8 Sent Done Done false
200622024-02-01T17:36:03.499Z 9 Acked 1008 Write 3 Sent Done Done false
200632024-02-01T17:36:03.499Z 10 Acked 1009 Write 4 Sent Done Done false
200642024-02-01T17:36:03.499Z 11 Acked 1010 Write 8 Sent Done Done false
200652024-02-01T17:36:03.499Z 12 Acked 1011 Write 2 Sent Done Done false
200662024-02-01T17:36:03.500Z 13 Acked 1012 Write 1 Sent Done Done false
200672024-02-01T17:36:03.500Z 14 Acked 1013 Write 7 Sent Done Done false
200682024-02-01T17:36:03.500Z 15 Acked 1014 Read 8 Sent Done Done false
200692024-02-01T17:36:03.500Z 16 Acked 1015 Read 7 Sent Done Done false
200702024-02-01T17:36:03.500Z 17 Acked 1016 Read 5 Sent Done Done false
200712024-02-01T17:36:03.500Z 18 Acked 1017 Write 8 Sent Done Done false
200722024-02-01T17:36:03.501Z 19 Acked 1018 Flush 0 Sent Done Done false
200732024-02-01T17:36:03.501Z 20 Acked 1019 Write 5 Sent Done Done false
200742024-02-01T17:36:03.501Z 21 Acked 1020 Read 7 Sent Done Done false
200752024-02-01T17:36:03.501Z 22 Acked 1021 Read 4 Sent Done Done false
200762024-02-01T17:36:03.501Z 23 Acked 1022 Write 9 Sent Done Done false
200772024-02-01T17:36:03.501Z 24 Acked 1023 Write 1 Sent Done Done false
200782024-02-01T17:36:03.501Z 25 Acked 1024 Read 2 Sent Done Done false
200792024-02-01T17:36:03.501Z 26 Acked 1025 Read 4 Sent Done Done false
200802024-02-01T17:36:03.502Z 27 Acked 1026 Read 9 Sent Done Done false
200812024-02-01T17:36:03.502Z 28 Acked 1027 Write 7 Sent Done Done false
200822024-02-01T17:36:03.502Z 29 Acked 1028 Read 5 Sent Done Done false
200832024-02-01T17:36:03.502Z 30 Acked 1029 Read 1 Sent Done Done false
200842024-02-01T17:36:03.502Z STATES DS:0 DS:1 DS:2 TOTAL
200852024-02-01T17:36:03.502Z New 0 0 0 0
200862024-02-01T17:36:03.502Z Sent 30 0 0 30
200872024-02-01T17:36:03.502Z Done 0 30 30 60
200882024-02-01T17:36:03.503Z Skipped 0 0 0 0
200892024-02-01T17:36:03.503Z Error 0 0 0 0
200902024-02-01T17:36:03.503ZLast Flush: 0 1018 1018
200912024-02-01T17:36:03.503ZDownstairs last five completed:
200922024-02-01T17:36:03.503ZUpstairs last five completed: 30 29 28 27 26
200932024-02-01T17:36:03.503Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:01.293712792Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1876}
200942024-02-01T17:36:03.503ZWrote out file "/tmp/test_repair_verify.data"
200952024-02-01T17:36:03.504Z
200962024-02-01T17:36:03.504ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
200972024-02-01T17:36:03.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200982024-02-01T17:36:03.504Z 0 000-019 184 186 186 304 309 309 F F F <---
200992024-02-01T17:36:03.504Z 1 020-039 180 186 186 297 309 309 F F F <---
201002024-02-01T17:36:03.504Z 2 040-059 180 180 180 297 297 297 F F F
201012024-02-01T17:36:03.504Z 3 060-079 180 180 180 297 297 297 F F F
201022024-02-01T17:36:03.504Z 4 080-099 182 186 186 300 309 309 F F F <---
201032024-02-01T17:36:03.504Z 5 100-119 182 182 182 299 299 299 F F F
201042024-02-01T17:36:03.505Z 6 120-139 180 180 180 296 296 296 F T T <---
201052024-02-01T17:36:03.505Z 7 140-159 185 185 185 307 307 307 F F F
201062024-02-01T17:36:03.505Z 8 160-179 184 186 186 304 309 309 F F F <---
201072024-02-01T17:36:03.505Z 9 180-199 184 184 184 306 306 306 F F F
201082024-02-01T17:36:03.505Z 10 200-219 180 186 186 297 308 308 F F F <---
201092024-02-01T17:36:03.506Z 11 220-239 184 184 184 306 306 306 F T T <---
201102024-02-01T17:36:03.506Z 12 240-259 174 174 174 285 285 285 F T T <---
201112024-02-01T17:36:03.506Z 13 260-279 182 182 182 302 302 302 F F F
201122024-02-01T17:36:03.507Z 14 280-299 185 185 185 307 307 307 F F F
201132024-02-01T17:36:03.507Z 15 300-319 180 186 186 297 309 309 F F F <---
201142024-02-01T17:36:03.507Z 16 320-339 178 186 186 294 309 309 F F F <---
201152024-02-01T17:36:03.507Z 17 340-359 178 186 186 294 309 309 F F F <---
201162024-02-01T17:36:03.507Z 18 360-379 183 186 186 303 309 309 F F F <---
201172024-02-01T17:36:03.507Z 19 380-399 183 186 186 303 309 309 F T T <---
201182024-02-01T17:36:03.508Z 20 400-419 181 186 186 298 309 309 F F F <---
201192024-02-01T17:36:03.508Z 21 420-439 182 182 182 302 302 302 F F F
201202024-02-01T17:36:03.508Z 22 440-459 184 186 186 306 309 309 F F F <---
201212024-02-01T17:36:03.508Z 23 460-479 184 184 184 306 306 306 F F F
201222024-02-01T17:36:03.508Z 24 480-499 182 182 182 299 299 299 F F F
201232024-02-01T17:36:03.508Z 25 500-519 182 182 182 302 302 302 F T T <---
201242024-02-01T17:36:03.509Z 26 520-539 182 182 182 302 302 302 F F F
201252024-02-01T17:36:03.509Z 27 540-559 182 182 182 300 300 300 F F F
201262024-02-01T17:36:03.509Z 28 560-579 185 185 185 307 307 307 F F F
201272024-02-01T17:36:03.509Z 29 580-599 184 184 184 304 304 304 F F F
201282024-02-01T17:36:03.509ZMax gen: 186, Max flush: 309
201292024-02-01T17:36:03.509ZError: Difference in extent metadata found!
201302024-02-01T17:36:03.510ZA Difference in extent metadata is expected here
201312024-02-01T17:36:03.510ZOn loop 93
201322024-02-01T17:36:03.511Z
201332024-02-01T17:36:03.511ZVerifying data now
201342024-02-01T17:36:07.381ZNo change after verify
201352024-02-01T17:36:07.381ZLoop: 93 Downstairs dump after verify (and repair):
201362024-02-01T17:36:07.401ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201372024-02-01T17:36:07.401Z 0 000-019 186 186 186 309 309 309 F F F
201382024-02-01T17:36:07.401Z 1 020-039 186 186 186 309 309 309 F F F
201392024-02-01T17:36:07.401Z 2 040-059 180 180 180 297 297 297 F F F
201402024-02-01T17:36:07.407Z 3 060-079 180 180 180 297 297 297 F F F
201412024-02-01T17:36:07.407Z 4 080-099 186 186 186 309 309 309 F F F
201422024-02-01T17:36:07.407Z 5 100-119 182 182 182 299 299 299 F F F
201432024-02-01T17:36:07.407Z 6 120-139 187 187 187 310 310 310 F F F
201442024-02-01T17:36:07.407Z 7 140-159 185 185 185 307 307 307 F F F
201452024-02-01T17:36:07.407Z 8 160-179 186 186 186 309 309 309 F F F
201462024-02-01T17:36:07.407Z 9 180-199 184 184 184 306 306 306 F F F
201472024-02-01T17:36:07.407Z 10 200-219 186 186 186 308 308 308 F F F
201482024-02-01T17:36:07.408Z 11 220-239 187 187 187 310 310 310 F F F
201492024-02-01T17:36:07.408Z 12 240-259 187 187 187 310 310 310 F F F
201502024-02-01T17:36:07.409Z 13 260-279 182 182 182 302 302 302 F F F
201512024-02-01T17:36:07.409Z 14 280-299 185 185 185 307 307 307 F F F
201522024-02-01T17:36:07.409Z 15 300-319 186 186 186 309 309 309 F F F
201532024-02-01T17:36:07.409Z 16 320-339 186 186 186 309 309 309 F F F
201542024-02-01T17:36:07.409Z 17 340-359 186 186 186 309 309 309 F F F
201552024-02-01T17:36:07.409Z 18 360-379 186 186 186 309 309 309 F F F
201562024-02-01T17:36:07.409Z 19 380-399 187 187 187 310 310 310 F F F
201572024-02-01T17:36:07.409Z 20 400-419 186 186 186 309 309 309 F F F
201582024-02-01T17:36:07.409Z 21 420-439 182 182 182 302 302 302 F F F
201592024-02-01T17:36:07.409Z 22 440-459 186 186 186 309 309 309 F F F
201602024-02-01T17:36:07.409Z 23 460-479 184 184 184 306 306 306 F F F
201612024-02-01T17:36:07.409Z 24 480-499 182 182 182 299 299 299 F F F
201622024-02-01T17:36:07.409Z 25 500-519 187 187 187 310 310 310 F F F
201632024-02-01T17:36:07.410Z 26 520-539 182 182 182 302 302 302 F F F
201642024-02-01T17:36:07.410Z 27 540-559 182 182 182 300 300 300 F F F
201652024-02-01T17:36:07.410Z 28 560-579 185 185 185 307 307 307 F F F
201662024-02-01T17:36:07.410Z 29 580-599 184 184 184 304 304 304 F F F
201672024-02-01T17:36:07.410ZMax gen: 187, Max flush: 310
201682024-02-01T17:36:07.410Z
201692024-02-01T17:36:07.410ZBegin loop 94
201702024-02-01T17:36:07.410ZDownstairs to restart: 1
201712024-02-01T17:36:07.437Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.722456002Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884}
201722024-02-01T17:36:07.437Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723065022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884}
201732024-02-01T17:36:07.444Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723138694Z","hostname":"Crucible runtime is spawned
201742024-02-01T17:36:07.444Zip-10-150-1-139.us-west-2.compute.internal","pid":1884}
201752024-02-01T17:36:07.444Z{"msg":"Crucible 11af86f9-a4c0-4f48-ba10-f2221d437ab2 has session id: dcaa2191-1a92-42a4-9712-a8bb7756d3c8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72329106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201762024-02-01T17:36:07.444Z{"msg":"Upstairs opts: Upstairs UUID: 11af86f9-a4c0-4f48-ba10-f2221d437ab2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723382084Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201772024-02-01T17:36:07.444Z{"msg":"Crucible stats registered with UUID: 11af86f9-a4c0-4f48-ba10-f2221d437ab2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723454596Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201782024-02-01T17:36:07.445Z{"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72352293Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884}
201792024-02-01T17:36:07.445Z{"msg":"11af86f9-a4c0-4f48-ba10-f2221d437ab2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723589728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201802024-02-01T17:36:07.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723666686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201812024-02-01T17:36:07.445Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723769068Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201822024-02-01T17:36:07.445Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723816497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201832024-02-01T17:36:07.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72388861Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201842024-02-01T17:36:07.446Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723930917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201852024-02-01T17:36:07.446Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.723982012Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201862024-02-01T17:36:07.446Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72402381Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201872024-02-01T17:36:07.446Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.724246208Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201882024-02-01T17:36:07.446Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.724342811Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201892024-02-01T17:36:07.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.724378995Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201902024-02-01T17:36:07.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72440527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201912024-02-01T17:36:07.446Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72447722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"io task","client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201922024-02-01T17:36:07.446Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.725992676Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201932024-02-01T17:36:07.446Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.726089716Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201942024-02-01T17:36:07.447Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.726155396Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201952024-02-01T17:36:07.447Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72628289Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201962024-02-01T17:36:07.447Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.726365247Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201972024-02-01T17:36:07.447Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.726414031Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201982024-02-01T17:36:07.447Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72715571Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
201992024-02-01T17:36:07.447Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.727218528Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202002024-02-01T17:36:07.447Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.727321397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202012024-02-01T17:36:07.447Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72800499Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202022024-02-01T17:36:07.447Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728083078Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202032024-02-01T17:36:07.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728128899Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202042024-02-01T17:36:07.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728333453Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202052024-02-01T17:36:07.454Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728432768Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202062024-02-01T17:36:07.454Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728504825Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202072024-02-01T17:36:07.454Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728537464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202082024-02-01T17:36:07.454Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728592896Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202092024-02-01T17:36:07.454Z{"msg":"[0]R flush_numbers[0..12]: [309, 309, 297, 297, 309, 299, 310, 307, 309, 306, 308, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728627584Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202102024-02-01T17:36:07.454Z{"msg":"[0]R generation[0..12]: [186, 186, 180, 180, 186, 182, 187, 185, 186, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728682817Z","hostnameWait for a query_work_queue command to finish before sending IO
202112024-02-01T17:36:07.455Z":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202122024-02-01T17:36:07.455Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728760435Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202132024-02-01T17:36:07.455Z{"msg":"[1]R flush_numbers[0..12]: [309, 309, 297, 297, 309, 299, 310, 307, 309, 306, 308, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728794981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202142024-02-01T17:36:07.455Z{"msg":"[1]R generation[0..12]: [186, 186, 180, 180, 186, 182, 187, 185, 186, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728850753Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202152024-02-01T17:36:07.455Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728878855Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202162024-02-01T17:36:07.455Z{"msg":"[2]R flush_numbers[0..12]: [309, 309, 297, 297, 309, 299, 310, 307, 309, 306, 308, 310]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728940477Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202172024-02-01T17:36:07.455Z{"msg":"[2]R generation[0..12]: [186, 186, 180, 180, 186, 182, 187, 185, 186, 184, 186, 187]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.728983976Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202182024-02-01T17:36:07.455Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729060431Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202192024-02-01T17:36:07.455Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729114449Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202202024-02-01T17:36:07.455Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729140286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202212024-02-01T17:36:07.455Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729201635Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202222024-02-01T17:36:07.458Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.72922824Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"":"downstairs","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202232024-02-01T17:36:07.458Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729285672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202242024-02-01T17:36:07.458Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729313374Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"0","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202252024-02-01T17:36:07.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729374881Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"1","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202262024-02-01T17:36:07.459Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729401008Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"client":"2","":"downstairs","session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
202272024-02-01T17:36:07.460Z
202282024-02-01T17:36:07.460Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729525003Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202292024-02-01T17:36:07.460Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729586927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202302024-02-01T17:36:07.460Z{"msg":"11af86f9-a4c0-4f48-ba10-f2221d437ab2 is now active with session: dcaa2191-1a92-42a4-9712-a8bb7756d3c8","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729620852Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202312024-02-01T17:36:07.460Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729683769Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884,"session_id":"dcaa2191-1a92-42a4-9712-a8bb7756d3c8"}
202322024-02-01T17:36:07.460Z{"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:06.729751084Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884}
202332024-02-01T17:36:07.465ZLoading write count information from file "/tmp/test_repair_verify.data"
202342024-02-01T17:36:07.465ZRun Repair workload
202352024-02-01T17:36:07.465Z01/30 Write block 226 len 16384 data: 11 9 9 12
202362024-02-01T17:36:07.465Z02/30 Flush
202372024-02-01T17:36:07.465Z03/30 Write block 223 len 12288 data: 20 19 13
202382024-02-01T17:36:07.465Z04/30 Read block 463 len 36864
202392024-02-01T17:36:07.475Z05/30 Write block 57 len 32768 data: 14 10 13 14 12 13 13 12
202402024-02-01T17:36:07.475Z06/30 Read block 1 len 40960
202412024-02-01T17:36:07.475Z07/30 Read block 484 len 20480
202422024-02-01T17:36:07.475Z08/30 Write block 259 len 20480 data: 17 15 14 14 15
202432024-02-01T17:36:07.476Z09/30 Write block 113 len 20480 data: 12 13 13 12 12
202442024-02-01T17:36:07.476Z10/30 Write block 12 len 28672 data: 13 13 13 13 16 18 19
202452024-02-01T17:36:07.476Z11/30 Read block 269 len 4096
202462024-02-01T17:36:07.476Z12/30 Write block 264 len 32768 data: 15 16 16 16 17 15 15 11
202472024-02-01T17:36:07.476Z13/30 Read block 345 len 40960
202482024-02-01T17:36:07.476Z14/30 Flush
202492024-02-01T17:36:07.476Z15/30 Write block 98 len 12288 data: 11 12 14
202502024-02-01T17:36:07.476Z16/30 Write block 258 len 4096 data: 19
202512024-02-01T17:36:07.476Z17/30 Write block 186 len 16384 data: 18 17 12 14
202522024-02-01T17:36:07.476Z18/30 Flush
202532024-02-01T17:36:07.483Z19/30 Write block 285 len 16384 data: 20 19 15 17
202542024-02-01T17:36:07.483Z20/30 Read block 79 len 20480
202552024-02-01T17:36:07.483Z21/30 Write block 564 len 4096 data: 10
202562024-02-01T17:36:07.483Z22/30 Read block 192 len 20480
202572024-02-01T17:36:07.483Z23/30 Read block 480 len 24576
202582024-02-01T17:36:07.483Z24/30 Read block 65 len 12288
202592024-02-01T17:36:07.483Z25/30 Read block 502 len 24576
202602024-02-01T17:36:07.493Z26/30 Write block 241 len 24576 data: 10 14 14 14 14 14
202612024-02-01T17:36:07.493Z27/30 Write block 446 len 24576 data: 11 9 9 11 9 7
202622024-02-01T17:36:07.493Z28/30 Write block 303 len 12288 data: 12 12 12
202632024-02-01T17:36:07.493Z29/30 Write block 568 len 8192 data: 10 11
202642024-02-01T17:36:07.494Z30/30 Read block 244 len 28672
202652024-02-01T17:36:07.494Z----------------------------------------------------------------
202662024-02-01T17:36:07.494Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
202672024-02-01T17:36:07.494ZStates: Active Active Active
202682024-02-01T17:36:07.494ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
202692024-02-01T17:36:07.494Z 1 Acked 1000 Write 4 Done Done Done false
202702024-02-01T17:36:07.494Z 2 Acked 1001 Flush 0 Done Sent Done false
202712024-02-01T17:36:07.494Z 3 Acked 1002 Write 3 Done Sent Done false
202722024-02-01T17:36:07.494Z 4 Acked 1003 Read 9 Done Sent Done false
202732024-02-01T17:36:07.494Z 5 Acked 1004 Write 8 Done Sent Done false
202742024-02-01T17:36:07.494Z 6 Acked 1005 Read 10 Done Sent Done false
202752024-02-01T17:36:07.494Z 7 Acked 1006 Read 5 Done Sent Done false
202762024-02-01T17:36:07.494Z 8 Acked 1007 Write 5 Done Sent Done false
202772024-02-01T17:36:07.495Z 9 Acked 1008 Write 5 Done Sent Done false
202782024-02-01T17:36:07.495Z 10 Acked 1009 Write 7 Done Sent Done false
202792024-02-01T17:36:07.495Z 11 Acked 1010 Read 1 Done Sent Done false
202802024-02-01T17:36:07.496Z 12 Acked 1011 Write 8 Done Sent Done false
202812024-02-01T17:36:07.496Z 13 Acked 1012 Read 10 Done Sent Done false
202822024-02-01T17:36:07.504Z 14 Acked 1013 Flush 0 Done Sent Done false
202832024-02-01T17:36:07.505Z 15 Acked 1014 Write 3 Done Sent Done false
202842024-02-01T17:36:07.505Z 16 Acked 1015 Write 1 Done Sent Done false
202852024-02-01T17:36:07.505Z 17 Acked 1016 Write 4 Done Sent Done false
202862024-02-01T17:36:07.505Z 18 Acked 1017 Flush 0 Done Sent Done false
202872024-02-01T17:36:07.505Z 19 Acked 1018 Write 4 Done Sent Done false
202882024-02-01T17:36:07.505Z 20 Acked 1019 Read 5 Done Sent Done false
202892024-02-01T17:36:07.505Z 21 Acked 1020 Write 1 Done Sent Done false
202902024-02-01T17:36:07.505Z 22 Acked 1021 Read 5 Done Sent Done false
202912024-02-01T17:36:07.505Z 23 Acked 1022 Read 6 Done Sent Done false
202922024-02-01T17:36:07.505Z 24 Acked 1023 Read 3 Done Sent Done false
202932024-02-01T17:36:07.505Z 25 Acked 1024 Read 6 Done Sent Done false
202942024-02-01T17:36:07.505Z 26 Acked 1025 Write 6 Done Sent Done false
202952024-02-01T17:36:07.505Z 27 Acked 1026 Write 6 Done Sent Done false
202962024-02-01T17:36:07.505Z 28 Acked 1027 Write 3 Done Sent Done false
202972024-02-01T17:36:07.506Z 29 Acked 1028 Write 2 Done Sent Done false
202982024-02-01T17:36:07.506Z 30 Acked 1029 Read 7 Done Sent Sent false
202992024-02-01T17:36:07.506Z STATES DS:0 DS:1 DS:2 TOTAL
203002024-02-01T17:36:07.506Z New 0 0 0 0
203012024-02-01T17:36:07.506Z Sent 0 29 1 30
203022024-02-01T17:36:07.506Z Done 30 1 29 60
203032024-02-01T17:36:07.506Z Skipped 0 0 0 0
203042024-02-01T17:36:07.506Z Error 0 0 0 0
203052024-02-01T17:36:07.506ZLast Flush: 1017 0 1017
203062024-02-01T17:36:07.509ZDownstairs last five completed:
203072024-02-01T17:36:07.509ZUpstairs last five completed: 30 29 28 27 26
203082024-02-01T17:36:07.509Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:06.776334984Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1884}
203092024-02-01T17:36:07.509ZWrote out file "/tmp/test_repair_verify.data"
203102024-02-01T17:36:07.532Z
203112024-02-01T17:36:07.532ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203122024-02-01T17:36:07.532ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203132024-02-01T17:36:07.532Z 0 000-019 188 186 188 312 309 312 F F F <---
203142024-02-01T17:36:07.532Z 1 020-039 186 186 186 309 309 309 F F F
203152024-02-01T17:36:07.533Z 2 040-059 188 180 188 312 297 312 F F F <---
203162024-02-01T17:36:07.533Z 3 060-079 188 180 188 312 297 312 F F F <---
203172024-02-01T17:36:07.533Z 4 080-099 188 186 188 313 309 313 F F F <---
203182024-02-01T17:36:07.533Z 5 100-119 188 182 188 313 299 313 F F F <---
203192024-02-01T17:36:07.533Z 6 120-139 187 187 187 310 310 310 F F F
203202024-02-01T17:36:07.533Z 7 140-159 185 185 185 307 307 307 F F F
203212024-02-01T17:36:07.533Z 8 160-179 186 186 186 309 309 309 F F F
203222024-02-01T17:36:07.533Z 9 180-199 188 184 188 313 306 313 F F F <---
203232024-02-01T17:36:07.533Z 10 200-219 186 186 186 308 308 308 F F F
203242024-02-01T17:36:07.534Z 11 220-239 188 187 188 312 310 312 F T F <---
203252024-02-01T17:36:07.534Z 12 240-259 188 187 188 313 310 313 T F T <---
203262024-02-01T17:36:07.534Z 13 260-279 188 182 188 312 302 312 F F F <---
203272024-02-01T17:36:07.534Z 14 280-299 185 185 185 307 307 307 T F T <---
203282024-02-01T17:36:07.534Z 15 300-319 186 186 186 309 309 309 T F T <---
203292024-02-01T17:36:07.534Z 16 320-339 186 186 186 309 309 309 F F F
203302024-02-01T17:36:07.534Z 17 340-359 186 186 186 309 309 309 F F F
203312024-02-01T17:36:07.534Z 18 360-379 186 186 186 309 309 309 F F F
203322024-02-01T17:36:07.534Z 19 380-399 187 187 187 310 310 310 F F F
203332024-02-01T17:36:07.534Z 20 400-419 186 186 186 309 309 309 F F F
203342024-02-01T17:36:07.534Z 21 420-439 182 182 182 302 302 302 F F F
203352024-02-01T17:36:07.535Z 22 440-459 186 186 186 309 309 309 T F T <---
203362024-02-01T17:36:07.535Z 23 460-479 184 184 184 306 306 306 F F F
203372024-02-01T17:36:07.535Z 24 480-499 182 182 182 299 299 299 F F F
203382024-02-01T17:36:07.535Z 25 500-519 187 187 187 310 310 310 F F F
203392024-02-01T17:36:07.535Z 26 520-539 182 182 182 302 302 302 F F F
203402024-02-01T17:36:07.535Z 27 540-559 182 182 182 300 300 300 F F F
203412024-02-01T17:36:07.535Z 28 560-579 185 185 185 307 307 307 T F T <---
203422024-02-01T17:36:07.535Z 29 580-599 184 184 184 304 304 304 F F F
203432024-02-01T17:36:07.535ZMax gen: 188, Max flush: 313
203442024-02-01T17:36:07.535ZError: Difference in extent metadata found!
203452024-02-01T17:36:07.541ZA Difference in extent metadata is expected here
203462024-02-01T17:36:07.541ZOn loop 94
203472024-02-01T17:36:07.542Z
203482024-02-01T17:36:07.542ZVerifying data now
203492024-02-01T17:36:20.266ZNo change after verify
203502024-02-01T17:36:20.267ZLoop: 94 Downstairs dump after verify (and repair):
203512024-02-01T17:36:20.267ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
203522024-02-01T17:36:20.267Z 0 000-019 188 188 188 312 312 312 F F F
203532024-02-01T17:36:20.267Z 1 020-039 186 186 186 309 309 309 F F F
203542024-02-01T17:36:20.267Z 2 040-059 188 188 188 312 312 312 F F F
203552024-02-01T17:36:20.267Z 3 060-079 188 188 188 312 312 312 F F F
203562024-02-01T17:36:20.267Z 4 080-099 188 188 188 313 313 313 F F F
203572024-02-01T17:36:20.267Z 5 100-119 188 188 188 313 313 313 F F F
203582024-02-01T17:36:20.267Z 6 120-139 187 187 187 310 310 310 F F F
203592024-02-01T17:36:20.267Z 7 140-159 185 185 185 307 307 307 F F F
203602024-02-01T17:36:20.267Z 8 160-179 186 186 186 309 309 309 F F F
203612024-02-01T17:36:20.267Z 9 180-199 188 188 188 313 313 313 F F F
203622024-02-01T17:36:20.268Z 10 200-219 186 186 186 308 308 308 F F F
203632024-02-01T17:36:20.268Z 11 220-239 188 188 188 312 312 312 F F F
203642024-02-01T17:36:20.268Z 12 240-259 189 189 189 314 314 314 F F F
203652024-02-01T17:36:20.268Z 13 260-279 188 188 188 312 312 312 F F F
203662024-02-01T17:36:20.268Z 14 280-299 189 189 189 314 314 314 F F F
203672024-02-01T17:36:20.268Z 15 300-319 189 189 189 314 314 314 F F F
203682024-02-01T17:36:20.268Z 16 320-339 186 186 186 309 309 309 F F F
203692024-02-01T17:36:20.268Z 17 340-359 186 186 186 309 309 309 F F F
203702024-02-01T17:36:20.268Z 18 360-379 186 186 186 309 309 309 F F F
203712024-02-01T17:36:20.268Z 19 380-399 187 187 187 310 310 310 F F F
203722024-02-01T17:36:20.268Z 20 400-419 186 186 186 309 309 309 F F F
203732024-02-01T17:36:20.268Z 21 420-439 182 182 182 302 302 302 F F F
203742024-02-01T17:36:20.269Z 22 440-459 189 189 189 314 314 314 F F F
203752024-02-01T17:36:20.269Z 23 460-479 184 184 184 306 306 306 F F F
203762024-02-01T17:36:20.269Z 24 480-499 182 182 182 299 299 299 F F F
203772024-02-01T17:36:20.269Z 25 500-519 187 187 187 310 310 310 F F F
203782024-02-01T17:36:20.269Z 26 520-539 182 182 182 302 302 302 F F F
203792024-02-01T17:36:20.269Z 27 540-559 182 182 182 300 300 300 F F F
203802024-02-01T17:36:20.269Z 28 560-579 189 189 189 314 314 314 F F F
203812024-02-01T17:36:20.269Z 29 580-599 184 184 184 304 304 304 F F F
203822024-02-01T17:36:20.269ZMax gen: 189, Max flush: 314
203832024-02-01T17:36:20.269Z
203842024-02-01T17:36:20.269ZBegin loop 95
203852024-02-01T17:36:20.269ZDownstairs to restart: 2
203862024-02-01T17:36:20.270Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.992562419Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
203872024-02-01T17:36:20.270Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993082163Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
203882024-02-01T17:36:20.270Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993140026Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
203892024-02-01T17:36:20.270Z{"msg":"Crucible e05f88b9-8646-47cb-a8e5-b18ec5650b53 has session id: ae79d5f7-3082-4585-82fa-87401f98606e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993165035Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203902024-02-01T17:36:20.270Z{"msg":"Upstairs opts: Upstairs UUID: e05f88b9-8646-47cb-a8e5-b18ec5650b53, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993223829Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203912024-02-01T17:36:20.270Z{"msg":"Crucible stats registered with UUID: e05f88b9-8646-47cb-a8e5-b18ec5650b53","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
203922024-02-01T17:36:20.270Z2024-02-01T17:36:11.993290044Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203932024-02-01T17:36:20.270Z{"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993461943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
203942024-02-01T17:36:20.270Z{"msg":"e05f88b9-8646-47cb-a8e5-b18ec5650b53 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993594607Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203952024-02-01T17:36:20.270Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993645567Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203962024-02-01T17:36:20.270Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993748579Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203972024-02-01T17:36:20.270Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993812447Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203982024-02-01T17:36:20.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993872311Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
203992024-02-01T17:36:20.271Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993940023Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204002024-02-01T17:36:20.271Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.993997939Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204012024-02-01T17:36:20.272Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.994068538Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204022024-02-01T17:36:20.272Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.994177519Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204032024-02-01T17:36:20.272Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.99421712Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204042024-02-01T17:36:20.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.994266989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204052024-02-01T17:36:20.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.994442878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204062024-02-01T17:36:20.272Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.994514873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"io task","client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204072024-02-01T17:36:20.272Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.995633464Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204082024-02-01T17:36:20.272Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.995714315Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204092024-02-01T17:36:20.273Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.995829889Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204102024-02-01T17:36:20.273Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.995885438Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204112024-02-01T17:36:20.274Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.996126153Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204122024-02-01T17:36:20.274Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.996274481Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204132024-02-01T17:36:20.274Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.99662589Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204142024-02-01T17:36:20.274Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.996692375Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204152024-02-01T17:36:20.274Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997365365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204162024-02-01T17:36:20.274Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997717921Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204172024-02-01T17:36:20.274Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997817174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204182024-02-01T17:36:20.274Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997888981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204192024-02-01T17:36:20.274Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997933185Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204202024-02-01T17:36:20.275Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.997977648Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204212024-02-01T17:36:20.275Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998008087Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204222024-02-01T17:36:20.275Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.99816761Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204232024-02-01T17:36:20.275Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998230195Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204242024-02-01T17:36:20.275Z{"msg":"[0]R flush_numbers[0..12]: [312, 309, 312, 312, 313, 313, 310, 307, 309, 313, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998276319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204252024-02-01T17:36:20.275Z{"msg":"[0]R generation[0..12]: [188, 186, 188, 188, 188, 188, 187, 185, 186, 188, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998320556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204262024-02-01T17:36:20.275Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998364107Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204272024-02-01T17:36:20.275Z{"msg":"[1]R flush_numbers[0..12]: [312, 309, 312, 312, 313, 313, 310, 307, 309, 313, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998408983Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204282024-02-01T17:36:20.275Z{"msg":"[1]R generation[0..12]: [188, 186, 188, 188, 188, 188, 187, 185, 186, 188, 186, 188]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
204292024-02-01T17:36:20.275Z,"time":"2024-02-01T17:36:11.998452285Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204302024-02-01T17:36:20.275Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998573417Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204312024-02-01T17:36:20.277Z{"msg":"[2]R flush_numbers[0..12]: [312, 309, 312, 312, 313, 313, 310, 307, 309, 313, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998626837Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204322024-02-01T17:36:20.277Z{"msg":"[2]R generation[0..12]: [188, 186, 188, 188, 188, 188, 187, 185, 186, 188, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998671623Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204332024-02-01T17:36:20.277Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998715414Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204342024-02-01T17:36:20.277Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998757024Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204352024-02-01T17:36:20.277Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998803218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204362024-02-01T17:36:20.277Z{"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998844736Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204372024-02-01T17:36:20.277Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998886978Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"":"downstairs","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204382024-02-01T17:36:20.277Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998937356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204392024-02-01T17:36:20.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.998981326Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"0","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204402024-02-01T17:36:20.277Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.99902489Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"1","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204412024-02-01T17:36:20.278Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.99906737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"client":"2","":"downstairs","session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204422024-02-01T17:36:20.278Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.999122697Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204432024-02-01T17:36:20.278Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.999146933Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204442024-02-01T17:36:20.278Z{"msg":"e05f88b9-8646-47cb-a8e5-b18ec5650b53 is now active with session: ae79d5f7-3082-4585-82fa-87401f98606e","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.999190739Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204452024-02-01T17:36:20.278Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:11.999227027Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892,"session_id":"ae79d5f7-3082-4585-82fa-87401f98606e"}
204462024-02-01T17:36:20.278Z{"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
204472024-02-01T17:36:20.279Z,"time":"2024-02-01T17:36:11.999286779Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
204482024-02-01T17:36:20.279ZLoading write count information from file "/tmp/test_repair_verify.data"
204492024-02-01T17:36:20.371ZRun Repair workload
204502024-02-01T17:36:20.371ZError: Difference in extent metadata found!
204512024-02-01T17:36:20.372Z01/30 Write block 412 len 32768 data: 12 11 14 16 20 14 13 17
204522024-02-01T17:36:20.372Z02/30 Write block 172 len 20480 data: 24 22 20 21 24
204532024-02-01T17:36:20.372Z03/30 Flush
204542024-02-01T17:36:20.372Z04/30 Write block 151 len 8192 data: 13 15
204552024-02-01T17:36:20.372Z05/30 Flush
204562024-02-01T17:36:20.373Z06/30 Write block 185 len 24576 data: 21 19 18 13 15 15
204572024-02-01T17:36:20.373Z07/30 Write block 420 len 12288 data: 13 13 13
204582024-02-01T17:36:20.373Z08/30 Read block 82 len 8192
204592024-02-01T17:36:20.373Z09/30 Flush
204602024-02-01T17:36:20.373Z10/30 Write block 95 len 32768 data: 9 11 10 12 13 15 15 18
204612024-02-01T17:36:20.373Z11/30 Read block 462 len 32768
204622024-02-01T17:36:20.374Z12/30 Write block 364 len 32768 data: 18 16 16 12 10 9 12 15
204632024-02-01T17:36:20.374Z13/30 Read block 466 len 32768
204642024-02-01T17:36:20.374Z14/30 Flush
204652024-02-01T17:36:20.374Z15/30 Flush
204662024-02-01T17:36:20.374Z16/30 Write block 498 len 12288 data: 16 16 16
204672024-02-01T17:36:20.374Z17/30 Read block 40 len 28672
204682024-02-01T17:36:20.374Z18/30 Read block 143 len 4096
204692024-02-01T17:36:20.374Z19/30 Write block 492 len 40960 data: 24 19 17 19 16 15 17 17 17 14
204702024-02-01T17:36:20.374Z20/30 Write block 105 len 40960 data: 20 19 20 18 15 12 11 10 13 14
204712024-02-01T17:36:20.374Z21/30 Read block 560 len 24576
204722024-02-01T17:36:20.374Z22/30 Read block 280 len 36864
204732024-02-01T17:36:20.374Z23/30 Write block 100 len 12288 data: 16 16 19
204742024-02-01T17:36:20.374Z24/30 Flush
204752024-02-01T17:36:20.374Z25/30 Write block 264 len 20480 data: 16 17 17 17 18
204762024-02-01T17:36:20.374Z26/30 Flush
204772024-02-01T17:36:20.374Z27/30 Write block 454 len 16384 data: 11 11 15 21
204782024-02-01T17:36:20.374Z28/30 Read block 544 len 16384
204792024-02-01T17:36:20.374Z29/30 Write block 282 len 32768 data: 13 15 17 21 20 16 18 20
204802024-02-01T17:36:20.375Z30/30 Write block 517 len 32768 data: 9 13 11 11 11 10 11 8
204812024-02-01T17:36:20.375Z----------------------------------------------------------------
204822024-02-01T17:36:20.375Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
204832024-02-01T17:36:20.375ZStates: Active Active Active
204842024-02-01T17:36:20.375ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204852024-02-01T17:36:20.375Z 1 Acked 1000 Write 8 Done Done Sent false
204862024-02-01T17:36:20.375Z 2 Acked 1001 Write 5 Done Done Sent false
204872024-02-01T17:36:20.375Z 3 Acked 1002 Flush 0 Done Done Sent false
204882024-02-01T17:36:20.375Z 4 Acked 1003 Write 2 Done Done Sent false
204892024-02-01T17:36:20.375Z 5 Acked 1004 Flush 0 Done Done Sent false
204902024-02-01T17:36:20.375Z 6 Acked 1005 Write 6 Done Done Sent false
204912024-02-01T17:36:20.375Z 7 Acked 1006 Write 3 Done Done Sent false
204922024-02-01T17:36:20.375Z 8 Acked 1007 Read 2 Done Done Sent false
204932024-02-01T17:36:20.375Z 9 Acked 1008 Flush 0 Done Done Sent false
204942024-02-01T17:36:20.375Z 10 Acked 1009 Write 8 Done Done Sent false
204952024-02-01T17:36:20.375Z 11 Acked 1010 Read 8 Done Done Sent false
204962024-02-01T17:36:20.375Z 12 Acked 1011 Write 8 Done Done Sent false
204972024-02-01T17:36:20.375Z 13 Acked 1012 Read 8 Done Done Sent false
204982024-02-01T17:36:20.375Z 14 Acked 1013 Flush 0 Done Done Sent false
204992024-02-01T17:36:20.375Z 15 Acked 1014 Flush 0 Done Done Sent false
205002024-02-01T17:36:20.375Z 16 Acked 1015 Write 3 Done Done Sent false
205012024-02-01T17:36:20.375Z 17 Acked 1016 Read 7 Done Done Sent false
205022024-02-01T17:36:20.375Z 18 Acked 1017 Read 1 Done Done Sent false
205032024-02-01T17:36:20.376Z 19 Acked 1018 Write 10 Done Done Sent false
205042024-02-01T17:36:20.377Z 20 Acked 1019 Write 10 Done Done Sent false
205052024-02-01T17:36:20.377Z 21 Acked 1020 Read 6 Done Done Sent false
205062024-02-01T17:36:20.377Z 22 Acked 1021 Read 9 Done Done Sent false
205072024-02-01T17:36:20.377Z 23 Acked 1022 Write 3 Done Done Sent false
205082024-02-01T17:36:20.377Z 24 Acked 1023 Flush 0 Done Done Sent false
205092024-02-01T17:36:20.377Z 25 Acked 1024 Write 5 Done Done Sent false
205102024-02-01T17:36:20.377Z 26 Acked 1025 Flush 0 Done Done Sent false
205112024-02-01T17:36:20.377Z 27 Acked 1026 Write 4 Done Done Sent false
205122024-02-01T17:36:20.377Z 28 Acked 1027 Read 4 Done Done Sent false
205132024-02-01T17:36:20.378Z 29 Acked 1028 Write 8 Done Done Sent false
205142024-02-01T17:36:20.378Z 30 Acked 1029 Write 8 Sent Sent Sent false
205152024-02-01T17:36:20.378Z STATES DS:0 DS:1 DS:2 TOTAL
205162024-02-01T17:36:20.378Z New 0 0 0 0
205172024-02-01T17:36:20.378Z Sent 1 1 30 32
205182024-02-01T17:36:20.378Z Done 29 29 0 58
205192024-02-01T17:36:20.378Z Skipped 0 0 0 0
205202024-02-01T17:36:20.378Z Error 0 0 0 0
205212024-02-01T17:36:20.378ZLast Flush: 1025 1025 0
205222024-02-01T17:36:20.378ZDownstairs last five completed:
205232024-02-01T17:36:20.378ZUpstairs last five completed: 30 29 28 27 26
205242024-02-01T17:36:20.378Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:12.048758915Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1892}
205252024-02-01T17:36:20.378ZWrote out file "/tmp/test_repair_verify.data"
205262024-02-01T17:36:20.378Z
205272024-02-01T17:36:20.379ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
205282024-02-01T17:36:20.379ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205292024-02-01T17:36:20.379Z 0 000-019 188 188 188 312 312 312 F F F
205302024-02-01T17:36:20.379Z 1 020-039 186 186 186 309 309 309 F F F
205312024-02-01T17:36:20.379Z 2 040-059 188 188 188 312 312 312 F F F
205322024-02-01T17:36:20.379Z 3 060-079 188 188 188 312 312 312 F F F
205332024-02-01T17:36:20.379Z 4 080-099 190 190 188 318 318 313 F F F <---
205342024-02-01T17:36:20.379Z 5 100-119 190 190 188 320 320 313 F F F <---
205352024-02-01T17:36:20.379Z 6 120-139 187 187 187 310 310 310 F F F
205362024-02-01T17:36:20.379Z 7 140-159 190 190 185 316 316 307 F F F <---
205372024-02-01T17:36:20.379Z 8 160-179 190 190 186 315 315 309 F F F <---
205382024-02-01T17:36:20.379Z 9 180-199 190 190 188 317 317 313 F F F <---
205392024-02-01T17:36:20.379Z 10 200-219 186 186 186 308 308 308 F F F
205402024-02-01T17:36:20.379Z 11 220-239 188 188 188 312 312 312 F F F
205412024-02-01T17:36:20.379Z 12 240-259 189 189 189 314 314 314 F F F
205422024-02-01T17:36:20.379Z 13 260-279 190 190 188 321 321 312 F F F <---
205432024-02-01T17:36:20.379Z 14 280-299 189 189 189 314 314 314 T T F <---
205442024-02-01T17:36:20.380Z 15 300-319 189 189 189 314 314 314 F F F
205452024-02-01T17:36:20.380Z 16 320-339 186 186 186 309 309 309 F F F
205462024-02-01T17:36:20.380Z 17 340-359 186 186 186 309 309 309 F F F
205472024-02-01T17:36:20.380Z 18 360-379 190 190 186 318 318 309 F F F <---
205482024-02-01T17:36:20.380Z 19 380-399 187 187 187 310 310 310 F F F
205492024-02-01T17:36:20.504Z 20 400-419 190 190 186 315 315 309 F F F <---
205502024-02-01T17:36:20.504Z 21 420-439 190 190 182 317 317 302 F F F <---
205512024-02-01T17:36:20.504Z 22 440-459 189 189 189 314 314 314 T T F <---
205522024-02-01T17:36:20.505Z 23 460-479 184 184 184 306 306 306 F F F
205532024-02-01T17:36:20.505Z 24 480-499 190 190 182 320 320 299 F F F <---
205542024-02-01T17:36:20.505Z 25 500-519 190 190 187 320 320 310 T T F <---
205552024-02-01T17:36:20.505Z 26 520-539 182 182 182 302 302 302 T T F <---
205562024-02-01T17:36:20.505Z 27 540-559 182 182 182 300 300 300 F F F
205572024-02-01T17:36:20.505Z 28 560-579 189 189 189 314 314 314 F F F
205582024-02-01T17:36:20.506Z 29 580-599 184 184 184 304 304 304 F F F
205592024-02-01T17:36:20.506ZMax gen: 190, Max flush: 321
205602024-02-01T17:36:20.506ZA Difference in extent metadata is expected here
205612024-02-01T17:36:20.506ZOn loop 95
205622024-02-01T17:36:20.506Z
205632024-02-01T17:36:20.507ZVerifying data now
205642024-02-01T17:36:20.507ZNo change after verify
205652024-02-01T17:36:20.508ZLoop: 95 Downstairs dump after verify (and repair):
205662024-02-01T17:36:20.508ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
205672024-02-01T17:36:20.508Z 0 000-019 188 188 188 312 312 312 F F F
205682024-02-01T17:36:20.508Z 1 020-039 186 186 186 309 309 309 F F F
205692024-02-01T17:36:20.508Z 2 040-059 188 188 188 312 312 312 F F F
205702024-02-01T17:36:20.509Z 3 060-079 188 188 188 312 312 312 F F F
205712024-02-01T17:36:20.509Z 4 080-099 190 190 190 318 318 318 F F F
205722024-02-01T17:36:20.509Z 5 100-119 190 190 190 320 320 320 F F F
205732024-02-01T17:36:20.509Z 6 120-139 187 187 187 310 310 310 F F F
205742024-02-01T17:36:20.509Z 7 140-159 190 190 190 316 316 316 F F F
205752024-02-01T17:36:20.509Z 8 160-179 190 190 190 315 315 315 F F F
205762024-02-01T17:36:20.509Z 9 180-199 190 190 190 317 317 317 F F F
205772024-02-01T17:36:20.510Z 10 200-219 186 186 186 308 308 308 F F F
205782024-02-01T17:36:20.510Z 11 220-239 188 188 188 312 312 312 F F F
205792024-02-01T17:36:20.510Z 12 240-259 189 189 189 314 314 314 F F F
205802024-02-01T17:36:20.510Z 13 260-279 190 190 190 321 321 321 F F F
205812024-02-01T17:36:20.510Z 14 280-299 191 191 191 322 322 322 F F F
205822024-02-01T17:36:20.511Z 15 300-319 189 189 189 314 314 314 F F F
205832024-02-01T17:36:20.511Z 16 320-339 186 186 186 309 309 309 F F F
205842024-02-01T17:36:20.511Z 17 340-359 186 186 186 309 309 309 F F F
205852024-02-01T17:36:20.511Z 18 360-379 190 190 190 318 318 318 F F F
205862024-02-01T17:36:20.511Z 19 380-399 187 187 187 310 310 310 F F F
205872024-02-01T17:36:20.511Z 20 400-419 190 190 190 315 315 315 F F F
205882024-02-01T17:36:20.511Z 21 420-439 190 190 190 317 317 317 F F F
205892024-02-01T17:36:20.511Z 22 440-459 191 191 191 322 322 322 F F F
205902024-02-01T17:36:20.512Z 23 460-479 184 184 184 306 306 306 F F F
205912024-02-01T17:36:20.512Z 24 480-499 190 190 190 320 320 320 F F F
205922024-02-01T17:36:20.512Z 25 500-519 191 191 191 322 322 322 F F F
205932024-02-01T17:36:20.512Z 26 520-539 191 191 191 322 322 322 F F F
205942024-02-01T17:36:20.512Z 27 540-559 182 182 182 300 300 300 F F F
205952024-02-01T17:36:20.512Z 28 560-579 189 189 189 314 314 314 F F F
205962024-02-01T17:36:20.512Z 29 580-599 184 184 184 304 304 304 F F F
205972024-02-01T17:36:20.513ZMax gen: 191, Max flush: 322
205982024-02-01T17:36:20.513Z
205992024-02-01T17:36:20.513ZBegin loop 96
206002024-02-01T17:36:20.513ZDownstairs to restart: 0
206012024-02-01T17:36:20.513Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.331525151Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
206022024-02-01T17:36:20.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33201699Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
206032024-02-01T17:36:20.513Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332050726Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
206042024-02-01T17:36:20.514Z{"msg":"Crucible 03942ded-80bb-45de-a29f-c89397cf3685 has session id: fce1b184-bad6-4499-acc2-cc6d093ab7dd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332072987Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206052024-02-01T17:36:20.514Z{"msg":"Upstairs opts: Upstairs UUID: 03942ded-80bb-45de-a29f-c89397cf3685, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332102517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206062024-02-01T17:36:20.514Z{"msg":"Crucible stats registered with UUID: 03942ded-80bb-45de-a29f-c89397cf3685","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332124767Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206072024-02-01T17:36:20.514ZCrucible runtime is spawned
206082024-02-01T17:36:20.514Z{"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332361241Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
206092024-02-01T17:36:20.514Z{"msg":"03942ded-80bb-45de-a29f-c89397cf3685 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332572425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206102024-02-01T17:36:20.515Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33261375Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206112024-02-01T17:36:20.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332667422Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206122024-02-01T17:36:20.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332712515Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206132024-02-01T17:36:20.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332755Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206142024-02-01T17:36:20.516Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33279672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206152024-02-01T17:36:20.516Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33284222Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206162024-02-01T17:36:20.516Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332884729Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206172024-02-01T17:36:20.516Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33292875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206182024-02-01T17:36:20.516Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.332968728Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206192024-02-01T17:36:20.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.333011176Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206202024-02-01T17:36:20.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.333051875Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206212024-02-01T17:36:20.517Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.333093973Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"io task","client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206222024-02-01T17:36:20.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.334239707Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206232024-02-01T17:36:20.517Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.334278437Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206242024-02-01T17:36:20.517Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33430261Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206252024-02-01T17:36:20.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.334330564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206262024-02-01T17:36:20.517Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.335182964Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206272024-02-01T17:36:20.518Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.335384169Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206282024-02-01T17:36:20.518Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.335676359Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206292024-02-01T17:36:20.518Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.335736416Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206302024-02-01T17:36:20.518Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.336065774Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206312024-02-01T17:36:20.520Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.336112071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206322024-02-01T17:36:20.520Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:17.33613696Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206332024-02-01T17:36:20.520ZWait for a query_work_queue command to finish before sending IO
206342024-02-01T17:36:20.520Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073430873Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206352024-02-01T17:36:20.520Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073520556Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206362024-02-01T17:36:20.520Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073551145Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206372024-02-01T17:36:20.521Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073574104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206382024-02-01T17:36:20.521Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073597168Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206392024-02-01T17:36:20.521Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073619878Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206402024-02-01T17:36:20.521Z{"msg":"[0]R flush_numbers[0..12]: [312, 309, 312, 312, 318, 320, 310, 316, 315, 317, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073643314Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206412024-02-01T17:36:20.522Z{"msg":"[0]R generation[0..12]: [188, 186, 188, 188, 190, 190, 187, 190, 190, 190, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073667348Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206422024-02-01T17:36:20.522Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073691532Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206432024-02-01T17:36:20.522Z{"msg":"[1]R flush_numbers[0..12]: [312, 309, 312, 312, 318, 320, 310, 316, 315, 317, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073769491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206442024-02-01T17:36:20.522Z{"msg":"[1]R generation[0..12]: [188, 186, 188, 188, 190, 190, 187, 190, 190, 190, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073804144Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206452024-02-01T17:36:20.522Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073833171Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206462024-02-01T17:36:20.523Z{"msg":"[2]R flush_numbers[0..12]: [312, 309, 312, 312, 318, 320, 310, 316, 315, 317, 308, 312]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073877517Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206472024-02-01T17:36:20.523Z{"msg":"[2]R generation[0..12]: [188, 186, 188, 188, 190, 190, 187, 190, 190, 190, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073946444Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206482024-02-01T17:36:20.523Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.073975961Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206492024-02-01T17:36:20.561Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074033376Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206502024-02-01T17:36:20.561Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074106471Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206512024-02-01T17:36:20.561Z{"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074135672Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206522024-02-01T17:36:20.561Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074200841Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"":"downstairs","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206532024-02-01T17:36:20.561Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074236903Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206542024-02-01T17:36:20.562Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074303184Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"0","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206552024-02-01T17:36:20.562Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074366239Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"1","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206562024-02-01T17:36:20.562Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206572024-02-01T17:36:20.562Ztime":"2024-02-01T17:36:19.074402668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"client":"2","":"downstairs","session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206582024-02-01T17:36:20.562Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074593432Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206592024-02-01T17:36:20.562Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074664303Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206602024-02-01T17:36:20.563Z{"msg":"03942ded-80bb-45de-a29f-c89397cf3685 is now active with session: fce1b184-bad6-4499-acc2-cc6d093ab7dd","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.07473398Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206612024-02-01T17:36:20.563Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.074799869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900,"session_id":"fce1b184-bad6-4499-acc2-cc6d093ab7dd"}
206622024-02-01T17:36:20.563Z{"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:19.07486759Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
206632024-02-01T17:36:20.563ZLoading write count information from file "/tmp/test_repair_verify.data"
206642024-02-01T17:36:20.563ZRun Repair workload
206652024-02-01T17:36:20.563Z01/30 Write block 433 len 32768 data: 21 19 21 18 18 16 14 14
206662024-02-01T17:36:20.563Z02/30 Read block 540 len 20480
206672024-02-01T17:36:20.564Z03/30 Read block 245 len 12288
206682024-02-01T17:36:20.564Z04/30 Read block 474 len 40960
206692024-02-01T17:36:20.564Z05/30 Read block 46 len 12288
206702024-02-01T17:36:20.564Z06/30 Read block 308 len 8192
206712024-02-01T17:36:20.564Z07/30 Write block 307 len 32768 data: 9 13 11 11 10 15 19 19
206722024-02-01T17:36:20.564Z08/30 Read block 29 len 4096
206732024-02-01T17:36:20.564Z09/30 Read block 245 len 36864
206742024-02-01T17:36:20.564Z10/30 Write block 419 len 16384 data: 18 14 14 14
206752024-02-01T17:36:20.564Z11/30 Read block 352 len 40960
206762024-02-01T17:36:20.564Z12/30 Read block 511 len 32768
206772024-02-01T17:36:20.565Z13/30 Write block 319 len 20480 data: 19 18 15 17 15
206782024-02-01T17:36:20.565Z14/30 Write block 213 len 36864 data: 15 13 13 9 10 16 19 19 17
206792024-02-01T17:36:20.565Z15/30 Read block 348 len 20480
206802024-02-01T17:36:20.565Z16/30 Write block 52 len 20480 data: 17 18 21 20 16
206812024-02-01T17:36:20.565Z17/30 Read block 60 len 28672
206822024-02-01T17:36:20.565Z18/30 Write block 35 len 16384 data: 13 15 16 18
206832024-02-01T17:36:20.565Z19/30 Write block 482 len 12288 data: 12 12 13
206842024-02-01T17:36:20.565Z20/30 Flush
206852024-02-01T17:36:20.566Z21/30 Write block 144 len 8192 data: 12 12
206862024-02-01T17:36:20.566Z22/30 Read block 45 len 4096
206872024-02-01T17:36:20.566Z23/30 Read block 479 len 24576
206882024-02-01T17:36:20.566Z24/30 Write block 530 len 12288 data: 8 9 11
206892024-02-01T17:36:20.566Z25/30 Read block 414 len 36864
206902024-02-01T17:36:20.566Z26/30 Read block 53 len 36864
206912024-02-01T17:36:20.566Z27/30 Write block 485 len 20480 data: 12 14 17 19 22
206922024-02-01T17:36:20.566Z28/30 Read block 549 len 28672
206932024-02-01T17:36:20.566Z29/30 Write block 439 len 40960 data: 15 15 16 12 10 13 13 12 10 10
206942024-02-01T17:36:20.567Z30/30 Read block 216 len 8192
206952024-02-01T17:36:20.567Z----------------------------------------------------------------
206962024-02-01T17:36:20.567Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
206972024-02-01T17:36:20.567ZStates: Active Active Active
206982024-02-01T17:36:20.567ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206992024-02-01T17:36:20.567Z 1 Acked 1000 Write 8 Done Done Done false
207002024-02-01T17:36:20.567Z 2 Acked 1001 Read 5 Done Done Done false
207012024-02-01T17:36:20.567Z 3 Acked 1002 Read 3 Done Done Done false
207022024-02-01T17:36:20.568Z 4 Acked 1003 Read 10 Done Done Done false
207032024-02-01T17:36:20.568Z 5 Acked 1004 Read 3 Done Done Done false
207042024-02-01T17:36:20.568Z 6 Acked 1005 Read 2 Sent Done Done false
207052024-02-01T17:36:20.568Z 7 Acked 1006 Write 8 Sent Done Done false
207062024-02-01T17:36:20.568Z 8 Acked 1007 Read 1 Sent Done Done false
207072024-02-01T17:36:20.568Z 9 Acked 1008 Read 9 Sent Done Done false
207082024-02-01T17:36:20.568Z 10 Acked 1009 Write 4 Sent Done Done false
207092024-02-01T17:36:20.568Z 11 Acked 1010 Read 10 Sent Done Done false
207102024-02-01T17:36:20.568Z 12 Acked 1011 Read 8 Sent Done Done false
207112024-02-01T17:36:20.568Z 13 Acked 1012 Write 5 Sent Done Done false
207122024-02-01T17:36:20.568Z 14 Acked 1013 Write 9 Sent Done Done false
207132024-02-01T17:36:20.569Z 15 Acked 1014 Read 5 Sent Done Done false
207142024-02-01T17:36:20.569Z 16 Acked 1015 Write 5 Sent Done Done false
207152024-02-01T17:36:20.569Z 17 Acked 1016 Read 7 Sent Done Done false
207162024-02-01T17:36:20.569Z 18 Acked 1017 Write 4 Sent Done Done false
207172024-02-01T17:36:20.569Z 19 Acked 1018 Write 3 Sent Done Done false
207182024-02-01T17:36:20.569Z 20 Acked 1019 Flush 0 Sent Done Done false
207192024-02-01T17:36:20.569Z 21 Acked 1020 Write 2 Sent Done Done false
207202024-02-01T17:36:20.569Z 22 Acked 1021 Read 1 Sent Done Done false
207212024-02-01T17:36:20.570Z 23 Acked 1022 Read 6 Sent Done Done false
207222024-02-01T17:36:20.570Z 24 Acked 1023 Write 3 Sent Done Done false
207232024-02-01T17:36:20.570Z 25 Acked 1024 Read 9 Sent Done Done false
207242024-02-01T17:36:20.570Z 26 Acked 1025 Read 9 Sent Done Done false
207252024-02-01T17:36:20.570Z 27 Acked 1026 Write 5 Sent Done Done false
207262024-02-01T17:36:20.570Z 28 Acked 1027 Read 7 Sent Done Done false
207272024-02-01T17:36:20.570Z 29 Acked 1028 Write 10 Sent Done Done false
207282024-02-01T17:36:20.571Z 30 Acked 1029 Read 2 Sent Done Sent false
207292024-02-01T17:36:20.571Z STATES DS:0 DS:1 DS:2 TOTAL
207302024-02-01T17:36:20.571Z New 0 0 0 0
207312024-02-01T17:36:20.571Z Sent 25 0 1 26
207322024-02-01T17:36:20.571Z Done 5 30 29 64
207332024-02-01T17:36:20.571Z Skipped 0 0 0 0
207342024-02-01T17:36:20.571Z Error 0 0 0 0
207352024-02-01T17:36:20.572ZLast Flush: 0 1019 1019
207362024-02-01T17:36:20.572ZDownstairs last five completed:
207372024-02-01T17:36:20.572ZUpstairs last five completed: 30 29 28 27 26
207382024-02-01T17:36:20.572Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:19.122543073Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1900}
207392024-02-01T17:36:20.572ZWrote out file "/tmp/test_repair_verify.data"
207402024-02-01T17:36:20.572Z
207412024-02-01T17:36:20.572ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
207422024-02-01T17:36:20.572ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207432024-02-01T17:36:20.572Z 0 000-019 188 188 188 312 312 312 F F F
207442024-02-01T17:36:20.572Z 1 020-039 186 192 192 309 323 323 F F F <---
207452024-02-01T17:36:20.573Z 2 040-059 188 192 192 312 323 323 F F F <---
207462024-02-01T17:36:20.573Z 3 060-079 188 188 188 312 312 312 F F F
207472024-02-01T17:36:20.573Z 4 080-099 190 190 190 318 318 318 F F F
207482024-02-01T17:36:20.573Z 5 100-119 190 190 190 320 320 320 F F F
207492024-02-01T17:36:20.632Z 6 120-139 187 187 187 310 310 310 F F F
207502024-02-01T17:36:20.632Z 7 140-159 190 190 190 316 316 316 F T T <---
207512024-02-01T17:36:20.632Z 8 160-179 190 190 190 315 315 315 F F F
207522024-02-01T17:36:20.632Z 9 180-199 190 190 190 317 317 317 F F F
207532024-02-01T17:36:20.632Z 10 200-219 186 192 192 308 323 323 F F F <---
207542024-02-01T17:36:20.632Z 11 220-239 188 192 192 312 323 323 F F F <---
207552024-02-01T17:36:20.632Z 12 240-259 189 189 189 314 314 314 F F F
207562024-02-01T17:36:20.632Z 13 260-279 190 190 190 321 321 321 F F F
207572024-02-01T17:36:20.633Z 14 280-299 191 191 191 322 322 322 F F F
207582024-02-01T17:36:20.633Z 15 300-319 189 192 192 314 323 323 F F F <---
207592024-02-01T17:36:20.633Z 16 320-339 186 192 192 309 323 323 F F F <---
207602024-02-01T17:36:20.633Z 17 340-359 186 186 186 309 309 309 F F F
207612024-02-01T17:36:20.633Z 18 360-379 190 190 190 318 318 318 F F F
207622024-02-01T17:36:20.633Z 19 380-399 187 187 187 310 310 310 F F F
207632024-02-01T17:36:20.633Z 20 400-419 190 192 192 315 323 323 F F F <---
207642024-02-01T17:36:20.634Z 21 420-439 190 192 192 317 323 323 T T T <---
207652024-02-01T17:36:20.634Z 22 440-459 191 192 192 322 323 323 T T T <---
207662024-02-01T17:36:20.634Z 23 460-479 184 184 184 306 306 306 F F F
207672024-02-01T17:36:20.634Z 24 480-499 190 192 192 320 323 323 F T T <---
207682024-02-01T17:36:20.634Z 25 500-519 191 191 191 322 322 322 F F F
207692024-02-01T17:36:20.634Z 26 520-539 191 191 191 322 322 322 F T T <---
207702024-02-01T17:36:20.634Z 27 540-559 182 182 182 300 300 300 F F F
207712024-02-01T17:36:20.635Z 28 560-579 189 189 189 314 314 314 F F F
207722024-02-01T17:36:20.635Z 29 580-599 184 184 184 304 304 304 F F F
207732024-02-01T17:36:20.635ZMax gen: 192, Max flush: 323
207742024-02-01T17:36:20.635ZError: Difference in extent metadata found!
207752024-02-01T17:36:20.635ZA Difference in extent metadata is expected here
207762024-02-01T17:36:20.635ZOn loop 96
207772024-02-01T17:36:20.636Z
207782024-02-01T17:36:20.636ZVerifying data now
207792024-02-01T17:36:24.924ZNo change after verify
207802024-02-01T17:36:24.924ZLoop: 96 Downstairs dump after verify (and repair):
207812024-02-01T17:36:24.950ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207822024-02-01T17:36:24.950Z 0 000-019 188 188 188 312 312 312 F F F
207832024-02-01T17:36:24.952Z 1 020-039 192 192 192 323 323 323 F F F
207842024-02-01T17:36:24.952Z 2 040-059 192 192 192 323 323 323 F F F
207852024-02-01T17:36:24.952Z 3 060-079 188 188 188 312 312 312 F F F
207862024-02-01T17:36:24.952Z 4 080-099 190 190 190 318 318 318 F F F
207872024-02-01T17:36:24.952Z 5 100-119 190 190 190 320 320 320 F F F
207882024-02-01T17:36:24.952Z 6 120-139 187 187 187 310 310 310 F F F
207892024-02-01T17:36:24.952Z 7 140-159 193 193 193 324 324 324 F F F
207902024-02-01T17:36:24.952Z 8 160-179 190 190 190 315 315 315 F F F
207912024-02-01T17:36:24.953Z 9 180-199 190 190 190 317 317 317 F F F
207922024-02-01T17:36:24.953Z 10 200-219 192 192 192 323 323 323 F F F
207932024-02-01T17:36:24.953Z 11 220-239 192 192 192 323 323 323 F F F
207942024-02-01T17:36:24.953Z 12 240-259 189 189 189 314 314 314 F F F
207952024-02-01T17:36:24.953Z 13 260-279 190 190 190 321 321 321 F F F
207962024-02-01T17:36:24.953Z 14 280-299 191 191 191 322 322 322 F F F
207972024-02-01T17:36:24.953Z 15 300-319 192 192 192 323 323 323 F F F
207982024-02-01T17:36:24.953Z 16 320-339 192 192 192 323 323 323 F F F
207992024-02-01T17:36:24.953Z 17 340-359 186 186 186 309 309 309 F F F
208002024-02-01T17:36:24.954Z 18 360-379 190 190 190 318 318 318 F F F
208012024-02-01T17:36:24.954Z 19 380-399 187 187 187 310 310 310 F F F
208022024-02-01T17:36:24.954Z 20 400-419 192 192 192 323 323 323 F F F
208032024-02-01T17:36:24.954Z 21 420-439 193 193 193 324 324 324 F F F
208042024-02-01T17:36:24.955Z 22 440-459 193 193 193 324 324 324 F F F
208052024-02-01T17:36:24.955Z 23 460-479 184 184 184 306 306 306 F F F
208062024-02-01T17:36:24.955Z 24 480-499 193 193 193 324 324 324 F F F
208072024-02-01T17:36:24.955Z 25 500-519 191 191 191 322 322 322 F F F
208082024-02-01T17:36:24.955Z 26 520-539 193 193 193 324 324 324 F F F
208092024-02-01T17:36:24.955Z 27 540-559 182 182 182 300 300 300 F F F
208102024-02-01T17:36:24.955Z 28 560-579 189 189 189 314 314 314 F F F
208112024-02-01T17:36:24.956Z 29 580-599 184 184 184 304 304 304 F F F
208122024-02-01T17:36:24.956ZMax gen: 193, Max flush: 324
208132024-02-01T17:36:24.974Z
208142024-02-01T17:36:24.975ZBegin loop 97
208152024-02-01T17:36:24.975ZDownstairs to restart: 0
208162024-02-01T17:36:24.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.256834866Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
208172024-02-01T17:36:24.996Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257360201Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
208182024-02-01T17:36:24.996Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257432022Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
208192024-02-01T17:36:24.996Z{"msg":"Crucible 7d980e34-0544-45e4-9656-b8bbb1637080 has session id: 45075578-61b6-4a33-9620-6e0a79c79003","v":0,"name":"crucible","level":30Crucible runtime is spawned
208202024-02-01T17:36:24.996Z,"time":"2024-02-01T17:36:24.257470927Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208212024-02-01T17:36:24.996Z{"msg":"Upstairs opts: Upstairs UUID: 7d980e34-0544-45e4-9656-b8bbb1637080, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257582683Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208222024-02-01T17:36:24.999Z{"msg":"Crucible stats registered with UUID: 7d980e34-0544-45e4-9656-b8bbb1637080","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257640756Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208232024-02-01T17:36:24.999Z{"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257665871Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
208242024-02-01T17:36:24.999Z{"msg":"7d980e34-0544-45e4-9656-b8bbb1637080 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257842218Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208252024-02-01T17:36:25.000Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257910959Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208262024-02-01T17:36:25.000Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.257990808Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208272024-02-01T17:36:25.000Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258082722Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208282024-02-01T17:36:25.000Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258128102Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208292024-02-01T17:36:25.000Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258163725Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208302024-02-01T17:36:25.000Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258199595Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208312024-02-01T17:36:25.000Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258238846Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208322024-02-01T17:36:25.001Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258281056Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208332024-02-01T17:36:25.001Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258475401Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208342024-02-01T17:36:25.001Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258588802Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208352024-02-01T17:36:25.001Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258673988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208362024-02-01T17:36:25.001Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.258787423Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"io task","client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208372024-02-01T17:36:25.001Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.259732686Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208382024-02-01T17:36:25.001Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.259933076Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208392024-02-01T17:36:25.002Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.26008618Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208402024-02-01T17:36:25.002Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.260123851Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208412024-02-01T17:36:25.002Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.260274402Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208422024-02-01T17:36:25.002Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.260364716Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208432024-02-01T17:36:25.003Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.260909928Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208442024-02-01T17:36:25.003Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.260980643Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208452024-02-01T17:36:25.003Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.261598399Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208462024-02-01T17:36:25.003Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.261990584Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208472024-02-01T17:36:25.003Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262077684Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208482024-02-01T17:36:25.003Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262110582Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208492024-02-01T17:36:25.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262166426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208502024-02-01T17:36:25.004Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262197847Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208512024-02-01T17:36:25.004Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262253758Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208522024-02-01T17:36:25.004Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262439853Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208532024-02-01T17:36:25.004Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262534441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208542024-02-01T17:36:25.005Z{"msg":"[0]R flush_numbers[0..12]: [312, 323, 323, 312, 318, 320, 310, 324, 315, 317, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262582358Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208552024-02-01T17:36:25.005Z{"msg":"[0]R generation[0..12]: [188, 192, 192, 188, 190, 190, 187, 193, 190, 190, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262695248Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208562024-02-01T17:36:25.005Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262742891Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208572024-02-01T17:36:25.005Z{"msg":"[1]R flush_numbers[0..12]: [312, 323, 323, 312, 318, 320, 310, 324, 315, 317, 323, 323]","Wait for a query_work_queue command to finish before sending IO
208582024-02-01T17:36:25.005Zv":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262806146Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208592024-02-01T17:36:25.005Z{"msg":"[1]R generation[0..12]: [188, 192, 192, 188, 190, 190, 187, 193, 190, 190, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262896988Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208602024-02-01T17:36:25.006Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.262942214Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208612024-02-01T17:36:25.006Z{"msg":"[2]R flush_numbers[0..12]: [312, 323, 323, 312, 318, 320, 310, 324, 315, 317, 323, 323]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263006492Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208622024-02-01T17:36:25.006Z{"msg":"[2]R generation[0..12]: [188, 192, 192, 188, 190, 190, 187, 193, 190, 190, 192, 192]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263050456Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208632024-02-01T17:36:25.006Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.26311981Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208642024-02-01T17:36:25.006Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263178019Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208652024-02-01T17:36:25.007Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.26320341Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208662024-02-01T17:36:25.007Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263265807Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208672024-02-01T17:36:25.007Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263296104Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"":"downstairs","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208682024-02-01T17:36:25.007Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263334052Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208692024-02-01T17:36:25.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263392344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"0","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208702024-02-01T17:36:25.007Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208712024-02-01T17:36:25.008Z2024-02-01T17:36:24.263467575Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"1","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208722024-02-01T17:36:25.009Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263504189Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"client":"2","":"downstairs","session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208732024-02-01T17:36:25.009Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263564594Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208742024-02-01T17:36:25.009Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263590498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208752024-02-01T17:36:25.010Z{"msg":"7d980e34-0544-45e4-9656-b8bbb1637080 is now active with session: 45075578-61b6-4a33-9620-6e0a79c79003","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.263631832Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208762024-02-01T17:36:25.011Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.26365397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908,"session_id":"45075578-61b6-4a33-9620-6e0a79c79003"}
208772024-02-01T17:36:25.011Z{"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:24.26369319Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
208782024-02-01T17:36:25.011ZLoading write count information from file "/tmp/test_repair_verify.data"
208792024-02-01T17:36:25.011ZRun Repair workload
208802024-02-01T17:36:25.011Z01/30 Write block 385 len 24576 data: 10 9 11 11 13 13
208812024-02-01T17:36:25.012Z02/30 Write block 249 len 24576 data: 11 9 11 13 12 14
208822024-02-01T17:36:25.012Z03/30 Read block 577 len 20480
208832024-02-01T17:36:25.012Z04/30 Write block 117 len 24576 data: 13 13 11 11 11 14
208842024-02-01T17:36:25.024Z05/30 Read block 440 len 28672
208852024-02-01T17:36:25.025Z06/30 Write block 164 len 28672 data: 15 13 11 12 15 18 21
208862024-02-01T17:36:25.025Z07/30 Write block 422 len 8192 data: 15 12
208872024-02-01T17:36:25.025Z08/30 Read block 244 len 24576
208882024-02-01T17:36:25.025Z09/30 Read block 329 len 16384
208892024-02-01T17:36:25.026Z10/30 Write block 17 len 28672 data: 19 20 14 16 14 13 12
208902024-02-01T17:36:25.026Z11/30 Read block 180 len 12288
208912024-02-01T17:36:25.026Z12/30 Write block 546 len 20480 data: 17 17 13 11 11
208922024-02-01T17:36:25.027Z13/30 Read block 569 len 28672
208932024-02-01T17:36:25.027Z14/30 Read block 230 len 20480
208942024-02-01T17:36:25.027Z15/30 Write block 396 len 28672 data: 9 11 13 11 13 17 16
208952024-02-01T17:36:25.027Z16/30 Read block 224 len 12288
208962024-02-01T17:36:25.027Z17/30 Write block 214 len 36864 data: 14 14 10 11 17 20 20 18 18
208972024-02-01T17:36:25.027Z18/30 Read block 361 len 24576
208982024-02-01T17:36:25.028Z19/30 Read block 550 len 20480
208992024-02-01T17:36:25.028Z20/30 Write block 2 len 32768 data: 9 12 14 13 14 12 12 13
209002024-02-01T17:36:25.028Z21/30 Write block 302 len 24576 data: 9 13 13 13 10 10
209012024-02-01T17:36:25.028Z22/30 Read block 0 len 36864
209022024-02-01T17:36:25.028Z23/30 Read block 560 len 40960
209032024-02-01T17:36:25.029Z24/30 Write block 506 len 8192 data: 10 13
209042024-02-01T17:36:25.030Z25/30 Read block 265 len 20480
209052024-02-01T17:36:25.030Z26/30 Read block 436 len 28672
209062024-02-01T17:36:25.030Z27/30 Write block 437 len 40960 data: 19 17 16 16 17 13 11 14 14 13
209072024-02-01T17:36:25.030Z28/30 Write block 530 len 32768 data: 9 10 12 12 18 18 12 16
209082024-02-01T17:36:25.030Z29/30 Read block 195 len 40960
209092024-02-01T17:36:25.038Z30/30 Write block 275 len 4096 data: 13
209102024-02-01T17:36:25.038Z----------------------------------------------------------------
209112024-02-01T17:36:25.038Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
209122024-02-01T17:36:25.039ZStates: Active Active Active
209132024-02-01T17:36:25.040ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
209142024-02-01T17:36:25.040Z 1 Acked 1000 Write 6 Done Done Done false
209152024-02-01T17:36:25.040Z 2 Acked 1001 Write 6 Sent Done Done false
209162024-02-01T17:36:25.040Z 3 Acked 1002 Read 5 Sent Done Done false
209172024-02-01T17:36:25.040Z 4 Acked 1003 Write 6 Sent Done Done false
209182024-02-01T17:36:25.041Z 5 Acked 1004 Read 7 Sent Done Done false
209192024-02-01T17:36:25.041Z 6 Acked 1005 Write 7 Sent Done Done false
209202024-02-01T17:36:25.041Z 7 Acked 1006 Write 2 Sent Done Done false
209212024-02-01T17:36:25.041Z 8 Acked 1007 Read 6 Sent Done Done false
209222024-02-01T17:36:25.041Z 9 Acked 1008 Read 4 Sent Done Done false
209232024-02-01T17:36:25.041Z 10 Acked 1009 Write 7 Sent Done Done false
209242024-02-01T17:36:25.042Z 11 Acked 1010 Read 3 Sent Done Done false
209252024-02-01T17:36:25.042Z 12 Acked 1011 Write 5 Sent Done Done false
209262024-02-01T17:36:25.042Z 13 Acked 1012 Read 7 Sent Done Done false
209272024-02-01T17:36:25.042Z 14 Acked 1013 Read 5 Sent Done Done false
209282024-02-01T17:36:25.042Z 15 Acked 1014 Write 7 Sent Done Done false
209292024-02-01T17:36:25.042Z 16 Acked 1015 Read 3 Sent Done Done false
209302024-02-01T17:36:25.042Z 17 Acked 1016 Write 9 Sent Done Done false
209312024-02-01T17:36:25.043Z 18 Acked 1017 Read 6 Sent Done Done false
209322024-02-01T17:36:25.043Z 19 Acked 1018 Read 5 Sent Done Done false
209332024-02-01T17:36:25.043Z 20 Acked 1019 Write 8 Sent Done Done false
209342024-02-01T17:36:25.043Z 21 Acked 1020 Write 6 Sent Done Done false
209352024-02-01T17:36:25.043Z 22 Acked 1021 Read 9 Sent Done Done false
209362024-02-01T17:36:25.043Z 23 Acked 1022 Read 10 Sent Done Done false
209372024-02-01T17:36:25.044Z 24 Acked 1023 Write 2 Sent Done Done false
209382024-02-01T17:36:25.044Z 25 Acked 1024 Read 5 Sent Done Done false
209392024-02-01T17:36:25.044Z 26 Acked 1025 Read 7 Sent Done Done false
209402024-02-01T17:36:25.044Z 27 Acked 1026 Write 10 Sent Done Done false
209412024-02-01T17:36:25.045Z 28 Acked 1027 Write 8 Sent Done Done false
209422024-02-01T17:36:25.045Z 29 Acked 1028 Read 10 Sent Done Done false
209432024-02-01T17:36:25.045Z 30 Acked 1029 Write 1 Sent Sent Sent false
209442024-02-01T17:36:25.045Z STATES DS:0 DS:1 DS:2 TOTAL
209452024-02-01T17:36:25.045Z New 0 0 0 0
209462024-02-01T17:36:25.045Z Sent 29 1 1 31
209472024-02-01T17:36:25.045Z Done 1 29 29 59
209482024-02-01T17:36:25.046Z Skipped 0 0 0 0
209492024-02-01T17:36:25.046Z Error 0 0 0 0
209502024-02-01T17:36:25.046ZLast Flush: 0 0 0
209512024-02-01T17:36:25.046ZDownstairs last five completed:
209522024-02-01T17:36:25.046ZUpstairs last five completed: 30 29 28 27 26
209532024-02-01T17:36:25.046Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:24.311075196Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1908}
209542024-02-01T17:36:25.046ZWrote out file "/tmp/test_repair_verify.data"
209552024-02-01T17:36:25.046Z
209562024-02-01T17:36:25.055ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209572024-02-01T17:36:25.076ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209582024-02-01T17:36:25.076Z 0 000-019 188 188 188 312 312 312 F T T <---
209592024-02-01T17:36:25.076Z 1 020-039 192 192 192 323 323 323 F T T <---
209602024-02-01T17:36:25.077Z 2 040-059 192 192 192 323 323 323 F F F
209612024-02-01T17:36:25.077Z 3 060-079 188 188 188 312 312 312 F F F
209622024-02-01T17:36:25.519Z 4 080-099 190 190 190 318 318 318 F F F
209632024-02-01T17:36:25.519Z 5 100-119 190 190 190 320 320 320 F T T <---
209642024-02-01T17:36:25.519Z 6 120-139 187 187 187 310 310 310 F T T <---
209652024-02-01T17:36:25.519Z 7 140-159 193 193 193 324 324 324 F F F
209662024-02-01T17:36:25.519Z 8 160-179 190 190 190 315 315 315 F T T <---
209672024-02-01T17:36:25.519Z 9 180-199 190 190 190 317 317 317 F F F
209682024-02-01T17:36:25.520Z 10 200-219 192 192 192 323 323 323 F T T <---
209692024-02-01T17:36:25.521Z 11 220-239 192 192 192 323 323 323 F T T <---
209702024-02-01T17:36:25.521Z 12 240-259 189 189 189 314 314 314 F T T <---
209712024-02-01T17:36:25.521Z 13 260-279 190 190 190 321 321 321 F T T <---
209722024-02-01T17:36:25.521Z 14 280-299 191 191 191 322 322 322 F F F
209732024-02-01T17:36:25.521Z 15 300-319 192 192 192 323 323 323 F T T <---
209742024-02-01T17:36:25.521Z 16 320-339 192 192 192 323 323 323 F F F
209752024-02-01T17:36:25.521Z 17 340-359 186 186 186 309 309 309 F F F
209762024-02-01T17:36:25.521Z 18 360-379 190 190 190 318 318 318 F F F
209772024-02-01T17:36:25.521Z 19 380-399 187 187 187 310 310 310 T T T
209782024-02-01T17:36:25.521Z 20 400-419 192 192 192 323 323 323 F T T <---
209792024-02-01T17:36:25.521Z 21 420-439 193 193 193 324 324 324 F T T <---
209802024-02-01T17:36:25.521Z 22 440-459 193 193 193 324 324 324 F T T <---
209812024-02-01T17:36:25.521Z 23 460-479 184 184 184 306 306 306 F F F
209822024-02-01T17:36:25.521Z 24 480-499 193 193 193 324 324 324 F F F
209832024-02-01T17:36:25.521Z 25 500-519 191 191 191 322 322 322 F T T <---
209842024-02-01T17:36:25.522Z 26 520-539 193 193 193 324 324 324 F T T <---
209852024-02-01T17:36:25.522Z 27 540-559 182 182 182 300 300 300 F T T <---
209862024-02-01T17:36:25.522Z 28 560-579 189 189 189 314 314 314 F F F
209872024-02-01T17:36:25.522Z 29 580-599 184 184 184 304 304 304 F F F
209882024-02-01T17:36:25.522ZMax gen: 193, Max flush: 324
209892024-02-01T17:36:25.522ZError: Difference in extent metadata found!
209902024-02-01T17:36:25.522ZA Difference in extent metadata is expected here
209912024-02-01T17:36:25.522ZOn loop 97
209922024-02-01T17:36:25.522Z
209932024-02-01T17:36:25.522ZVerifying data now
209942024-02-01T17:36:30.554ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
209952024-02-01T17:36:30.564Z399,402c399,400
209962024-02-01T17:36:30.564Z< 8,
209972024-02-01T17:36:30.564Z< 10,
209982024-02-01T17:36:30.565Z< 12,
209992024-02-01T17:36:30.565Z< 10,
210002024-02-01T17:36:30.565Z---
210012024-02-01T17:36:30.565Z> 9,
210022024-02-01T17:36:30.566Z> 11,
210032024-02-01T17:36:30.566Z403a402,403
210042024-02-01T17:36:30.566Z> 11,
210052024-02-01T17:36:30.566Z> 13,
210062024-02-01T17:36:30.567Zdiff found after verify
210072024-02-01T17:36:30.567ZLoop: 97 Downstairs dump after verify (and repair):
210082024-02-01T17:36:30.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
210092024-02-01T17:36:30.588Z 0 000-019 194 194 194 325 325 325 F F F
210102024-02-01T17:36:30.588Z 1 020-039 194 194 194 325 325 325 F F F
210112024-02-01T17:36:30.588Z 2 040-059 192 192 192 323 323 323 F F F
210122024-02-01T17:36:30.589Z 3 060-079 188 188 188 312 312 312 F F F
210132024-02-01T17:36:30.589Z 4 080-099 190 190 190 318 318 318 F F F
210142024-02-01T17:36:30.589Z 5 100-119 194 194 194 325 325 325 F F F
210152024-02-01T17:36:30.601Z 6 120-139 194 194 194 325 325 325 F F F
210162024-02-01T17:36:30.602Z 7 140-159 193 193 193 324 324 324 F F F
210172024-02-01T17:36:30.602Z 8 160-179 194 194 194 325 325 325 F F F
210182024-02-01T17:36:30.602Z 9 180-199 190 190 190 317 317 317 F F F
210192024-02-01T17:36:30.602Z 10 200-219 194 194 194 325 325 325 F F F
210202024-02-01T17:36:30.602Z 11 220-239 194 194 194 325 325 325 F F F
210212024-02-01T17:36:30.602Z 12 240-259 194 194 194 325 325 325 F F F
210222024-02-01T17:36:30.603Z 13 260-279 194 194 194 325 325 325 F F F
210232024-02-01T17:36:30.603Z 14 280-299 191 191 191 322 322 322 F F F
210242024-02-01T17:36:30.603Z 15 300-319 194 194 194 325 325 325 F F F
210252024-02-01T17:36:30.603Z 16 320-339 192 192 192 323 323 323 F F F
210262024-02-01T17:36:30.603Z 17 340-359 186 186 186 309 309 309 F F F
210272024-02-01T17:36:30.603Z 18 360-379 190 190 190 318 318 318 F F F
210282024-02-01T17:36:30.604Z 19 380-399 194 194 194 325 325 325 F F F
210292024-02-01T17:36:30.604Z 20 400-419 194 194 194 325 325 325 F F F
210302024-02-01T17:36:30.604Z 21 420-439 194 194 194 325 325 325 F F F
210312024-02-01T17:36:30.605Z 22 440-459 194 194 194 325 325 325 F F F
210322024-02-01T17:36:30.605Z 23 460-479 184 184 184 306 306 306 F F F
210332024-02-01T17:36:30.606Z 24 480-499 193 193 193 324 324 324 F F F
210342024-02-01T17:36:30.606Z 25 500-519 194 194 194 325 325 325 F F F
210352024-02-01T17:36:30.606Z 26 520-539 194 194 194 325 325 325 F F F
210362024-02-01T17:36:30.606Z 27 540-559 194 194 194 325 325 325 F F F
210372024-02-01T17:36:30.607Z 28 560-579 189 189 189 314 314 314 F F F
210382024-02-01T17:36:30.607Z 29 580-599 184 184 184 304 304 304 F F F
210392024-02-01T17:36:30.607ZMax gen: 194, Max flush: 325
210402024-02-01T17:36:30.607Z
210412024-02-01T17:36:30.607ZBegin loop 98
210422024-02-01T17:36:30.607ZDownstairs to restart: 2
210432024-02-01T17:36:30.626Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.904656356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
210442024-02-01T17:36:30.626Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905154286Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
210452024-02-01T17:36:30.626Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905183777Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
210462024-02-01T17:36:30.664Z{"msg":"Crucible 934d10af-6c73-4588-bd81-f53d04e876c2 has session id: a0303110-0847-41f4-a31e-5ab3483c3bf3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905248063Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210472024-02-01T17:36:30.664Z{"msg":"Upstairs opts: Upstairs UUID: 934d10af-6c73-4588-bd81-f53d04e876c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
210482024-02-01T17:36:30.665Z,"time":"2024-02-01T17:36:29.905302546Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210492024-02-01T17:36:30.665Z{"msg":"Crucible stats registered with UUID: 934d10af-6c73-4588-bd81-f53d04e876c2","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905417189Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210502024-02-01T17:36:30.665Z{"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905475652Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
210512024-02-01T17:36:30.665Z{"msg":"934d10af-6c73-4588-bd81-f53d04e876c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.90568559Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210522024-02-01T17:36:30.665Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905741951Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210532024-02-01T17:36:30.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905774389Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210542024-02-01T17:36:30.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905854497Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210552024-02-01T17:36:30.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905899714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210562024-02-01T17:36:30.666Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.905942251Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210572024-02-01T17:36:30.666Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906145879Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210582024-02-01T17:36:30.667Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906196354Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210592024-02-01T17:36:30.667Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906244397Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210602024-02-01T17:36:30.667Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906324407Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210612024-02-01T17:36:30.667Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906375919Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210622024-02-01T17:36:30.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.90651297Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210632024-02-01T17:36:30.669Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.906563284Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210642024-02-01T17:36:30.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.907348714Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210652024-02-01T17:36:30.669Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.907441448Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210662024-02-01T17:36:30.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.907574657Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210672024-02-01T17:36:30.669Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.907634812Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210682024-02-01T17:36:30.669Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.908136574Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210692024-02-01T17:36:30.669Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.908343461Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210702024-02-01T17:36:30.669Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.90841225Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210712024-02-01T17:36:30.669Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.908438426Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210722024-02-01T17:36:30.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909057365Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210732024-02-01T17:36:30.670Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909142495Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210742024-02-01T17:36:30.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909242624Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210752024-02-01T17:36:30.670Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.90928897Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210762024-02-01T17:36:30.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909360174Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210772024-02-01T17:36:30.670Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909430551Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210782024-02-01T17:36:30.670Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.909805425Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210792024-02-01T17:36:30.671Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.91051763Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210802024-02-01T17:36:30.671Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910582317Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210812024-02-01T17:36:30.671Z{"msg":"[0]R flush_numbers[0..12]: [325, 325, 323, 312, 318, 325, 325, 324, 325, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910677691Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210822024-02-01T17:36:30.671Z{"msg":"[0]R generation[0..12]: [194, 194, 192, 188, 190, 194, 194, 193, 194, 190, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910833257Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210832024-02-01T17:36:30.671Z{Wait for a query_work_queue command to finish before sending IO
210842024-02-01T17:36:30.671Z"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910907764Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210852024-02-01T17:36:30.671Z{"msg":"[1]R flush_numbers[0..12]: [325, 325, 323, 312, 318, 325, 325, 324, 325, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910937547Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210862024-02-01T17:36:30.671Z{"msg":"[1]R generation[0..12]: [194, 194, 192, 188, 190, 194, 194, 193, 194, 190, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.910981177Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210872024-02-01T17:36:30.672Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911007705Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210882024-02-01T17:36:30.672Z{"msg":"[2]R flush_numbers[0..12]: [325, 325, 323, 312, 318, 325, 325, 324, 325, 317, 325, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911051292Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210892024-02-01T17:36:30.672Z{"msg":"[2]R generation[0..12]: [194, 194, 192, 188, 190, 194, 194, 193, 194, 190, 194, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911077545Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210902024-02-01T17:36:30.672Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911119936Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210912024-02-01T17:36:30.672Z{"msg":"Max found gen is 195","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911144236Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210922024-02-01T17:36:30.672Z{"msg":"Generation requested: 196 >= found:195","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911166641Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210932024-02-01T17:36:30.672Z{"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911206637Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210942024-02-01T17:36:30.672Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911230799Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"":"downstairs","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210952024-02-01T17:36:30.672Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911276436Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210962024-02-01T17:36:30.672Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.91129943Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"0","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210972024-02-01T17:36:30.673Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911348192Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"1","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210982024-02-01T17:36:30.673Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911372231Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"client":"2","":"downstairs","session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
210992024-02-01T17:36:30.674Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911412826Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
211002024-02-01T17:36:30.674Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911435937Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
211012024-02-01T17:36:30.674Z{"msg":"934d10af-6c73-4588-bd81-f53d04e876c2 is now active with session: a0303110-0847-41f4-a31e-5ab3483c3bf3","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911458009Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
211022024-02-01T17:36:30.674Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911500604Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919,"session_id":"a0303110-0847-41f4-a31e-5ab3483c3bf3"}
211032024-02-01T17:36:30.674Z{"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:29.911523611Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
211042024-02-01T17:36:30.674ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211052024-02-01T17:36:30.674ZLoading write count information from file "/tmp/test_repair_verify.data"
211062024-02-01T17:36:30.674ZRun Repair workload
211072024-02-01T17:36:30.675Z01/30 Write block 195 len 28672 data: 13 14 13 9 10 12 13
211082024-02-01T17:36:30.675Z02/30 Write block 300 len 12288 data: 15 11 10
211092024-02-01T17:36:30.675Z03/30 Read block 528 len 40960
211102024-02-01T17:36:30.697Z04/30 Read block 374 len 24576
211112024-02-01T17:36:30.698Z05/30 Write block 445 len 4096 data: 15
211122024-02-01T17:36:30.698Z06/30 Read block 137 len 28672
211132024-02-01T17:36:30.698Z07/30 Read block 308 len 28672
211142024-02-01T17:36:30.699Z08/30 Write block 325 len 12288 data: 9 10 12
211152024-02-01T17:36:30.700Z09/30 Write block 454 len 32768 data: 12 12 16 22 21 22 19 17
211162024-02-01T17:36:30.700Z10/30 Write block 31 len 28672 data: 10 11 11 11 14 16 17
211172024-02-01T17:36:30.700Z11/30 Read block 53 len 36864
211182024-02-01T17:36:30.700Z12/30 Read block 594 len 24576
211192024-02-01T17:36:30.701Z13/30 Read block 265 len 32768
211202024-02-01T17:36:30.701Z14/30 Read block 599 len 4096
211212024-02-01T17:36:30.701Z15/30 Read block 26 len 40960
211222024-02-01T17:36:30.701Z16/30 Read block 509 len 32768
211232024-02-01T17:36:30.701Z17/30 Write block 115 len 24576 data: 14 13 14 14 12 12
211242024-02-01T17:36:30.701Z18/30 Write block 261 len 16384 data: 15 15 16 17
211252024-02-01T17:36:30.702Z19/30 Write block 92 len 8192 data: 11 11
211262024-02-01T17:36:30.702Z20/30 Flush
211272024-02-01T17:36:30.702Z21/30 Flush
211282024-02-01T17:36:30.702Z22/30 Write block 265 len 28672 data: 18 18 18 19 16 16 12
211292024-02-01T17:36:30.702Z23/30 Flush
211302024-02-01T17:36:30.702Z24/30 Write block 267 len 16384 data: 19 20 17 17
211312024-02-01T17:36:30.703Z25/30 Write block 480 len 32768 data: 9 10 13 13 14 13 15 18
211322024-02-01T17:36:30.703Z26/30 Read block 173 len 4096
211332024-02-01T17:36:30.703Z27/30 Write block 102 len 40960 data: 20 16 18 21 20 21 19 16 13 12
211342024-02-01T17:36:30.703Z28/30 Read block 1 len 12288
211352024-02-01T17:36:30.703Z29/30 Write block 141 len 36864 data: 14 12 12 13 13 13 14 18 18
211362024-02-01T17:36:30.704Z30/30 Read block 206 len 12288
211372024-02-01T17:36:30.704Z----------------------------------------------------------------
211382024-02-01T17:36:30.704Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
211392024-02-01T17:36:30.704ZStates: Active Active Active
211402024-02-01T17:36:30.705ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211412024-02-01T17:36:30.705Z 1 Acked 1000 Write 7 Done Done Done false
211422024-02-01T17:36:30.705Z 2 Acked 1001 Write 3 Done Done Done false
211432024-02-01T17:36:30.705Z 3 Acked 1002 Read 10 Done Done Done false
211442024-02-01T17:36:30.705Z 4 Acked 1003 Read 6 Done Done Sent false
211452024-02-01T17:36:30.705Z 5 Acked 1004 Write 1 Done Done Sent false
211462024-02-01T17:36:30.706Z 6 Acked 1005 Read 7 Done Done Sent false
211472024-02-01T17:36:30.706Z 7 Acked 1006 Read 7 Done Done Sent false
211482024-02-01T17:36:30.706Z 8 Acked 1007 Write 3 Done Done Sent false
211492024-02-01T17:36:30.706Z 9 Acked 1008 Write 8 Done Done Sent false
211502024-02-01T17:36:30.706Z 10 Acked 1009 Write 7 Done Done Sent false
211512024-02-01T17:36:30.707Z 11 Acked 1010 Read 9 Done Done Sent false
211522024-02-01T17:36:30.707Z 12 Acked 1011 Read 6 Done Done Sent false
211532024-02-01T17:36:30.707Z 13 Acked 1012 Read 8 Done Done Sent false
211542024-02-01T17:36:30.707Z 14 Acked 1013 Read 1 Done Done Sent false
211552024-02-01T17:36:30.709Z 15 Acked 1014 Read 10 Done Done Sent false
211562024-02-01T17:36:30.709Z 16 Acked 1015 Read 8 Done Done Sent false
211572024-02-01T17:36:30.709Z 17 Acked 1016 Write 6 Done Done Sent false
211582024-02-01T17:36:30.709Z 18 Acked 1017 Write 4 Done Done Sent false
211592024-02-01T17:36:30.710Z 19 Acked 1018 Write 2 Done Done Sent false
211602024-02-01T17:36:30.710Z 20 Acked 1019 Flush 0 Done Done Sent false
211612024-02-01T17:36:30.710Z 21 Acked 1020 Flush 0 Done Done Sent false
211622024-02-01T17:36:30.710Z 22 Acked 1021 Write 7 Done Done Sent false
211632024-02-01T17:36:30.710Z 23 Acked 1022 Flush 0 Done Done Sent false
211642024-02-01T17:36:30.711Z 24 Acked 1023 Write 4 Done Done Sent false
211652024-02-01T17:36:30.711Z 25 Acked 1024 Write 8 Done Done Sent false
211662024-02-01T17:36:30.711Z 26 Acked 1025 Read 1 Done Done Sent false
211672024-02-01T17:36:30.711Z 27 Acked 1026 Write 10 Done Done Sent false
211682024-02-01T17:36:30.712Z 28 Acked 1027 Read 3 Done Done Sent false
211692024-02-01T17:36:30.712Z 29 Acked 1028 Write 9 Done Done Sent false
211702024-02-01T17:36:30.712Z 30 Acked 1029 Read 3 Done Done Sent false
211712024-02-01T17:36:30.712Z STATES DS:0 DS:1 DS:2 TOTAL
211722024-02-01T17:36:30.712Z New 0 0 0 0
211732024-02-01T17:36:30.712Z Sent 0 0 27 27
211742024-02-01T17:36:30.713Z Done 30 30 3 63
211752024-02-01T17:36:30.713Z Skipped 0 0 0 0
211762024-02-01T17:36:30.713Z Error 0 0 0 0
211772024-02-01T17:36:30.713ZLast Flush: 1022 1022 0
211782024-02-01T17:36:30.713ZDownstairs last five completed:
211792024-02-01T17:36:30.714ZUpstairs last five completed: 30 29 28 27 26
211802024-02-01T17:36:30.714Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:29.962985217Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1919}
211812024-02-01T17:36:30.714ZWrote out file "/tmp/test_repair_verify.data"
211822024-02-01T17:36:31.146Z
211832024-02-01T17:36:31.146ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211842024-02-01T17:36:31.152ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211852024-02-01T17:36:31.152Z 0 000-019 194 194 194 325 325 325 F F F
211862024-02-01T17:36:31.152Z 1 020-039 196 196 194 326 326 325 F F F <---
211872024-02-01T17:36:31.152Z 2 040-059 192 192 192 323 323 323 F F F
211882024-02-01T17:36:31.152Z 3 060-079 188 188 188 312 312 312 F F F
211892024-02-01T17:36:31.152Z 4 080-099 196 196 190 326 326 318 F F F <---
211902024-02-01T17:36:31.152Z 5 100-119 196 196 194 326 326 325 T T F <---
211912024-02-01T17:36:31.152Z 6 120-139 196 196 194 326 326 325 F F F <---
211922024-02-01T17:36:31.152Z 7 140-159 193 193 193 324 324 324 T T F <---
211932024-02-01T17:36:31.152Z 8 160-179 194 194 194 325 325 325 F F F
211942024-02-01T17:36:31.152Z 9 180-199 196 196 190 326 326 317 F F T <---
211952024-02-01T17:36:31.152Z 10 200-219 196 196 194 326 326 325 F F T <---
211962024-02-01T17:36:31.152Z 11 220-239 194 194 194 325 325 325 F F F
211972024-02-01T17:36:31.152Z 12 240-259 194 194 194 325 325 325 F F F
211982024-02-01T17:36:31.153Z 13 260-279 196 196 194 328 328 325 T T F <---
211992024-02-01T17:36:31.153Z 14 280-299 191 191 191 322 322 322 F F F
212002024-02-01T17:36:31.153Z 15 300-319 196 196 194 326 326 325 F F T <---
212012024-02-01T17:36:31.153Z 16 320-339 196 196 192 326 326 323 F F F <---
212022024-02-01T17:36:31.153Z 17 340-359 186 186 186 309 309 309 F F F
212032024-02-01T17:36:31.153Z 18 360-379 190 190 190 318 318 318 F F F
212042024-02-01T17:36:31.153Z 19 380-399 194 194 194 325 325 325 F F F
212052024-02-01T17:36:31.153Z 20 400-419 194 194 194 325 325 325 F F F
212062024-02-01T17:36:31.153Z 21 420-439 194 194 194 325 325 325 F F F
212072024-02-01T17:36:31.153Z 22 440-459 196 196 194 326 326 325 F F F <---
212082024-02-01T17:36:31.154Z 23 460-479 196 196 184 326 326 306 F F F <---
212092024-02-01T17:36:31.154Z 24 480-499 193 193 193 324 324 324 T T F <---
212102024-02-01T17:36:31.155Z 25 500-519 194 194 194 325 325 325 F F F
212112024-02-01T17:36:31.155Z 26 520-539 194 194 194 325 325 325 F F F
212122024-02-01T17:36:31.155Z 27 540-559 194 194 194 325 325 325 F F F
212132024-02-01T17:36:31.155Z 28 560-579 189 189 189 314 314 314 F F F
212142024-02-01T17:36:31.155Z 29 580-599 184 184 184 304 304 304 F F F
212152024-02-01T17:36:31.155ZMax gen: 196, Max flush: 328
212162024-02-01T17:36:31.155ZError: Difference in extent metadata found!
212172024-02-01T17:36:31.155ZA Difference in extent metadata is expected here
212182024-02-01T17:36:31.155ZOn loop 98
212192024-02-01T17:36:31.155Z
212202024-02-01T17:36:31.155ZVerifying data now
212212024-02-01T17:36:36.828ZNo change after verify
212222024-02-01T17:36:36.828ZLoop: 98 Downstairs dump after verify (and repair):
212232024-02-01T17:36:36.849ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212242024-02-01T17:36:36.849Z 0 000-019 194 194 194 325 325 325 F F F
212252024-02-01T17:36:36.849Z 1 020-039 196 196 196 326 326 326 F F F
212262024-02-01T17:36:36.849Z 2 040-059 192 192 192 323 323 323 F F F
212272024-02-01T17:36:36.855Z 3 060-079 188 188 188 312 312 312 F F F
212282024-02-01T17:36:36.856Z 4 080-099 196 196 196 326 326 326 F F F
212292024-02-01T17:36:36.856Z 5 100-119 197 197 197 329 329 329 F F F
212302024-02-01T17:36:36.856Z 6 120-139 196 196 196 326 326 326 F F F
212312024-02-01T17:36:36.857Z 7 140-159 197 197 197 329 329 329 F F F
212322024-02-01T17:36:36.857Z 8 160-179 194 194 194 325 325 325 F F F
212332024-02-01T17:36:36.857Z 9 180-199 196 196 196 326 326 326 F F F
212342024-02-01T17:36:36.857Z 10 200-219 196 196 196 326 326 326 F F F
212352024-02-01T17:36:36.857Z 11 220-239 194 194 194 325 325 325 F F F
212362024-02-01T17:36:36.857Z 12 240-259 194 194 194 325 325 325 F F F
212372024-02-01T17:36:36.857Z 13 260-279 197 197 197 329 329 329 F F F
212382024-02-01T17:36:36.857Z 14 280-299 191 191 191 322 322 322 F F F
212392024-02-01T17:36:36.857Z 15 300-319 196 196 196 326 326 326 F F F
212402024-02-01T17:36:36.857Z 16 320-339 196 196 196 326 326 326 F F F
212412024-02-01T17:36:36.857Z 17 340-359 186 186 186 309 309 309 F F F
212422024-02-01T17:36:36.857Z 18 360-379 190 190 190 318 318 318 F F F
212432024-02-01T17:36:36.857Z 19 380-399 194 194 194 325 325 325 F F F
212442024-02-01T17:36:36.857Z 20 400-419 194 194 194 325 325 325 F F F
212452024-02-01T17:36:36.857Z 21 420-439 194 194 194 325 325 325 F F F
212462024-02-01T17:36:36.857Z 22 440-459 196 196 196 326 326 326 F F F
212472024-02-01T17:36:36.857Z 23 460-479 196 196 196 326 326 326 F F F
212482024-02-01T17:36:36.858Z 24 480-499 197 197 197 329 329 329 F F F
212492024-02-01T17:36:36.859Z 25 500-519 194 194 194 325 325 325 F F F
212502024-02-01T17:36:36.859Z 26 520-539 194 194 194 325 325 325 F F F
212512024-02-01T17:36:36.859Z 27 540-559 194 194 194 325 325 325 F F F
212522024-02-01T17:36:36.859Z 28 560-579 189 189 189 314 314 314 F F F
212532024-02-01T17:36:36.859Z 29 580-599 184 184 184 304 304 304 F F F
212542024-02-01T17:36:36.859ZMax gen: 197, Max flush: 329
212552024-02-01T17:36:36.859Z
212562024-02-01T17:36:36.859ZBegin loop 99
212572024-02-01T17:36:36.860ZDownstairs to restart: 1
212582024-02-01T17:36:36.890Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.169640071Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
212592024-02-01T17:36:36.890Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"c49269318fbfc7a408561fd872515f115027bbe7\",\n git_commit_timestamp: \"2024-02-01T16:48:08.000000000Z\",\n git_branch: \"alan/more-info\",\n rustc_semver: \"1.75.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"82e1608dfa6e0b5569232559e3d385fea5a93112\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170168106Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
212602024-02-01T17:36:36.890Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170205364Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
212612024-02-01T17:36:36.902Z{"msg":"Crucible 1373347f-2430-4530-97c0-682370689d23 has session id: 4b2d88a9-87b6-4eae-a749-973159885e42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170260527Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212622024-02-01T17:36:36.902Z{"msg":"Upstairs opts: Upstairs UUID: 1373347f-2430-4530-97c0-682370689d23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170304334Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212632024-02-01T17:36:36.902Z{"msg":"Crucible stats registered with UUID: 1373347f-2430-4530-97c0-682370689d23","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170453249Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212642024-02-01T17:36:36.902ZCrucible runtime is spawned
212652024-02-01T17:36:36.902Z{"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170760242Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
212662024-02-01T17:36:36.902Z{"msg":"1373347f-2430-4530-97c0-682370689d23 active request set","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.170882313Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212672024-02-01T17:36:36.902Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171007991Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212682024-02-01T17:36:36.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171140097Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212692024-02-01T17:36:36.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171368074Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212702024-02-01T17:36:36.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.17142975Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212712024-02-01T17:36:36.903Z{"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171567441Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212722024-02-01T17:36:36.903Z{"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171725946Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212732024-02-01T17:36:36.903Z{"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171771972Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212742024-02-01T17:36:36.903Z{"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171842034Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212752024-02-01T17:36:36.903Z{"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171876234Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212762024-02-01T17:36:36.903Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171901806Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212772024-02-01T17:36:36.903Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171925136Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212782024-02-01T17:36:36.904Z{"msg":"ds_connection connected","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.171965344Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212792024-02-01T17:36:36.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.172543592Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212802024-02-01T17:36:36.904Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.172608133Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212812024-02-01T17:36:36.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.172822496Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212822024-02-01T17:36:36.904Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.172979668Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212832024-02-01T17:36:36.904Z{"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.173034312Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212842024-02-01T17:36:36.904Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.173068831Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212852024-02-01T17:36:36.904Z{"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.173464491Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212862024-02-01T17:36:36.904Z{"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.173874228Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212872024-02-01T17:36:36.904Z{"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174094962Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212882024-02-01T17:36:36.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174156942Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212892024-02-01T17:36:36.904Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174194148Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212902024-02-01T17:36:36.904Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174251296Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212912024-02-01T17:36:36.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174753805Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212922024-02-01T17:36:36.904Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174826198Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212932024-02-01T17:36:36.904Z{"msg":"Waiting for true more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174859727Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212942024-02-01T17:36:36.904Z{"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174894804Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212952024-02-01T17:36:36.904Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.174940724Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212962024-02-01T17:36:36.904Z{"msg":"[0]R flush_numbers[0..12]: [325, 326, 323, 312, 326, 329, 326, 329, 325, 326, 326, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.17498704Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212972024-02-01T17:36:36.904Z{"msg":"[0]R generation[0..12]: [194, 196, 192, 188, 196, 197, 196, 197, 194, 196, 196, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175030764Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212982024-02-01T17:36:36.904Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175069617Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
212992024-02-01T17:36:36.904Z{"msg":"[1]R flush_numbers[0..12]: [325, 326, 323, 312, 326, 329, 326, 329, 325, 326, 326, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175098252Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213002024-02-01T17:36:36.904Z{"msg":"[1]R generation[0..12]: [194, 196, 192, 188, 196, 197, 196, 197, 194, 196, 196, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.17512242Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213012024-02-01T17:36:36.905Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175169869Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213022024-02-01T17:36:36.905Z{"msg":"[2]R flush_numbers[0..12]: [325, 326, 323, 312, 326, 329, 326, 329, 325, 326, 326, 325]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175196356Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213032024-02-01T17:36:36.905Z{"msg":"[2]R generation[0..12]: [194, 196, 192, 188, 196, 197, 196, 197, 194, 196, 196, 194]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175219737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213042024-02-01T17:36:36.905ZWait for a query_work_queue command to finish before sending IO
213052024-02-01T17:36:36.905Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175263957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213062024-02-01T17:36:36.905Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175288089Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213072024-02-01T17:36:36.905Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175310412Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213082024-02-01T17:36:36.905Z{"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175349917Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213092024-02-01T17:36:36.905Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175373775Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"":"downstairs","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213102024-02-01T17:36:36.905Z{"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175395989Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213112024-02-01T17:36:36.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175435564Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"0","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213122024-02-01T17:36:36.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175459465Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"1","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213132024-02-01T17:36:36.905Z{"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175481737Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"client":"2","":"downstairs","session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213142024-02-01T17:36:36.905Z{"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175522459Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213152024-02-01T17:36:36.905Z{"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175545253Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213162024-02-01T17:36:36.905Z{"msg":"1373347f-2430-4530-97c0-682370689d23 is now active with session: 4b2d88a9-87b6-4eae-a749-973159885e42","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175566957Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213172024-02-01T17:36:36.905Z{"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175614819Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927,"session_id":"4b2d88a9-87b6-4eae-a749-973159885e42"}
213182024-02-01T17:36:36.906Z{"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-02-01T17:36:36.175635945Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
213192024-02-01T17:36:36.906ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213202024-02-01T17:36:36.906ZLoading write count information from file "/tmp/test_repair_verify.data"
213212024-02-01T17:36:36.906ZRun Repair workload
213222024-02-01T17:36:36.906Z01/30 Write block 405 len 12288 data: 15 13 15
213232024-02-01T17:36:36.906Z02/30 Write block 477 len 4096 data: 8
213242024-02-01T17:36:36.906Z03/30 Write block 8 len 16384 data: 13 14 15 14
213252024-02-01T17:36:36.906Z04/30 Flush
213262024-02-01T17:36:36.906Z05/30 Flush
213272024-02-01T17:36:36.914Z06/30 Write block 114 len 28672 data: 15 15 14 15 15 13 13
213282024-02-01T17:36:36.914Z07/30 Flush
213292024-02-01T17:36:36.914Z08/30 Write block 179 len 28672 data: 20 16 17 20 19 19 22
213302024-02-01T17:36:36.914Z09/30 Read block 367 len 16384
213312024-02-01T17:36:36.914Z10/30 Write block 579 len 12288 data: 14 16 16
213322024-02-01T17:36:36.914Z11/30 Write block 355 len 20480 data: 14 13 13 15 14
213332024-02-01T17:36:36.914Z12/30 Write block 331 len 32768 data: 18 17 17 19 19 19 18 15
213342024-02-01T17:36:36.914Z13/30 Read block 525 len 36864
213352024-02-01T17:36:36.914Z14/30 Write block 67 len 36864 data: 14 15 18 14 16 18 18 18 17
213362024-02-01T17:36:36.914Z15/30 Read block 174 len 28672
213372024-02-01T17:36:36.921Z16/30 Write block 357 len 24576 data: 14 16 15 17 19 21
213382024-02-01T17:36:36.921Z17/30 Write block 506 len 28672 data: 11 14 15 16 13 15 13
213392024-02-01T17:36:36.921Z18/30 Write block 280 len 36864 data: 15 14 14 16 18 22 21 17 19
213402024-02-01T17:36:36.921Z19/30 Read block 381 len 12288
213412024-02-01T17:36:36.938Z20/30 Write block 228 len 20480 data: 10 13 13 13 15
213422024-02-01T17:36:36.938Z21/30 Read block 403 len 32768
213432024-02-01T17:36:36.938Z22/30 Write block 577 len 28672 data: 11 14 15 17 17 16 14
213442024-02-01T17:36:36.938Z23/30 Read block 298 len 16384
213452024-02-01T17:36:36.938Z24/30 Write block 518 len 12288 data: 14 12 12
213462024-02-01T17:36:36.938Z25/30 Read block 459 len 24576
213472024-02-01T17:36:36.938Z26/30 Write block 243 len 4096 data: 15
213482024-02-01T17:36:36.938Z27/30 Read block 21 len 24576
213492024-02-01T17:36:36.938Z28/30 Read block 448 len 8192
213502024-02-01T17:36:36.938Z29/30 Write block 512 len 16384 data: 14 14 11 9
213512024-02-01T17:36:36.938Z30/30 Write block 93 len 28672 data: 12 11 10 12 11 13 14
213522024-02-01T17:36:36.938Z----------------------------------------------------------------
213532024-02-01T17:36:36.938Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
213542024-02-01T17:36:36.938ZStates: Active Active Active
213552024-02-01T17:36:36.938ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213562024-02-01T17:36:36.938Z 1 Acked 1000 Write 3 Done Sent Done false
213572024-02-01T17:36:36.938Z 2 Acked 1001 Write 1 Done Sent Done false
213582024-02-01T17:36:36.938Z 3 Acked 1002 Write 4 Done Sent Done false
213592024-02-01T17:36:36.938Z 4 Acked 1003 Flush 0 Done Sent Done false
213602024-02-01T17:36:36.938Z 5 Acked 1004 Flush 0 Done Sent Done false
213612024-02-01T17:36:36.938Z 6 Acked 1005 Write 7 Done Sent Done false
213622024-02-01T17:36:36.939Z 7 Acked 1006 Flush 0 Done Sent Done false
213632024-02-01T17:36:36.939Z 8 Acked 1007 Write 7 Done Sent Done false
213642024-02-01T17:36:36.939Z 9 Acked 1008 Read 4 Done Sent Done false
213652024-02-01T17:36:36.939Z 10 Acked 1009 Write 3 Done Sent Done false
213662024-02-01T17:36:36.939Z 11 Acked 1010 Write 5 Done Sent Done false
213672024-02-01T17:36:36.939Z 12 Acked 1011 Write 8 Done Sent Done false
213682024-02-01T17:36:36.939Z 13 Acked 1012 Read 9 Done Sent Done false
213692024-02-01T17:36:36.939Z 14 Acked 1013 Write 9 Done Sent Done false
213702024-02-01T17:36:36.939Z 15 Acked 1014 Read 7 Done Sent Done false
213712024-02-01T17:36:36.939Z 16 Acked 1015 Write 6 Done Sent Done false
213722024-02-01T17:36:36.939Z 17 Acked 1016 Write 7 Done Sent Done false
213732024-02-01T17:36:36.939Z 18 Acked 1017 Write 9 Done Sent Done false
213742024-02-01T17:36:36.939Z 19 Acked 1018 Read 3 Done Sent Done false
213752024-02-01T17:36:36.939Z 20 Acked 1019 Write 5 Done Sent Done false
213762024-02-01T17:36:36.939Z 21 Acked 1020 Read 8 Done Sent Done false
213772024-02-01T17:36:36.939Z 22 Acked 1021 Write 7 Done Sent Done false
213782024-02-01T17:36:36.939Z 23 Acked 1022 Read 4 Done Sent Done false
213792024-02-01T17:36:36.939Z 24 Acked 1023 Write 3 Done Sent Done false
213802024-02-01T17:36:36.939Z 25 Acked 1024 Read 6 Done Sent Done false
213812024-02-01T17:36:36.939Z 26 Acked 1025 Write 1 Done Sent Done false
213822024-02-01T17:36:36.939Z 27 Acked 1026 Read 6 Done Sent Done false
213832024-02-01T17:36:36.939Z 28 Acked 1027 Read 2 Done Sent Done false
213842024-02-01T17:36:36.939Z 29 Acked 1028 Write 4 Done Sent Done false
213852024-02-01T17:36:36.939Z 30 Acked 1029 Write 7 Sent Sent Sent false
213862024-02-01T17:36:36.939Z STATES DS:0 DS:1 DS:2 TOTAL
213872024-02-01T17:36:36.940Z New 0 0 0 0
213882024-02-01T17:36:36.940Z Sent 1 30 1 32
213892024-02-01T17:36:36.940Z Done 29 0 29 58
213902024-02-01T17:36:36.940Z Skipped 0 0 0 0
213912024-02-01T17:36:36.940Z Error 0 0 0 0
213922024-02-01T17:36:36.940ZLast Flush: 1006 0 1006
213932024-02-01T17:36:36.940ZDownstairs last five completed:
213942024-02-01T17:36:36.940ZUpstairs last five completed: 30 29 28 27 26
213952024-02-01T17:36:36.940Z{"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-02-01T17:36:36.221247498Z","hostname":"ip-10-150-1-139.us-west-2.compute.internal","pid":1927}
213962024-02-01T17:36:36.948ZWrote out file "/tmp/test_repair_verify.data"
213972024-02-01T17:36:36.948Z
213982024-02-01T17:36:36.948ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
213992024-02-01T17:36:36.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214002024-02-01T17:36:36.978Z 0 000-019 198 194 198 330 325 330 F F F <---
214012024-02-01T17:36:36.978Z 1 020-039 196 196 196 326 326 326 F F F
214022024-02-01T17:36:36.979Z 2 040-059 192 192 192 323 323 323 F F F
214032024-02-01T17:36:36.979Z 3 060-079 188 188 188 312 312 312 T F T <---
214042024-02-01T17:36:36.979Z 4 080-099 196 196 196 326 326 326 T F T <---
214052024-02-01T17:36:36.979Z 5 100-119 198 197 198 332 329 332 F F F <---
214062024-02-01T17:36:36.979Z 6 120-139 198 196 198 332 326 332 F F F <---
214072024-02-01T17:36:36.979Z 7 140-159 197 197 197 329 329 329 F F F
214082024-02-01T17:36:36.989Z 8 160-179 194 194 194 325 325 325 T F T <---
214092024-02-01T17:36:36.989Z 9 180-199 196 196 196 326 326 326 T F T <---
214102024-02-01T17:36:36.989Z 10 200-219 196 196 196 326 326 326 F F F
214112024-02-01T17:36:36.989Z 11 220-239 194 194 194 325 325 325 T F T <---
214122024-02-01T17:36:36.989Z 12 240-259 194 194 194 325 325 325 T F T <---
214132024-02-01T17:36:36.989Z 13 260-279 197 197 197 329 329 329 F F F
214142024-02-01T17:36:36.989Z 14 280-299 191 191 191 322 322 322 T F T <---
214152024-02-01T17:36:36.989Z 15 300-319 196 196 196 326 326 326 F F F
214162024-02-01T17:36:36.989Z 16 320-339 196 196 196 326 326 326 T F T <---
214172024-02-01T17:36:36.989Z 17 340-359 186 186 186 309 309 309 T F T <---
214182024-02-01T17:36:36.989Z 18 360-379 190 190 190 318 318 318 T F T <---
214192024-02-01T17:36:36.989Z 19 380-399 194 194 194 325 325 325 F F F
214202024-02-01T17:36:36.989Z 20 400-419 198 194 198 330 325 330 F F F <---
214212024-02-01T17:36:36.989Z 21 420-439 194 194 194 325 325 325 F F F
214222024-02-01T17:36:36.989Z 22 440-459 196 196 196 326 326 326 F F F
214232024-02-01T17:36:36.989Z 23 460-479 198 196 198 330 326 330 F F F <---
214242024-02-01T17:36:36.990Z 24 480-499 197 197 197 329 329 329 F F F
214252024-02-01T17:36:36.990Z 25 500-519 194 194 194 325 325 325 T F T <---
214262024-02-01T17:36:36.990Z 26 520-539 194 194 194 325 325 325 T F T <---
214272024-02-01T17:36:36.990Z 27 540-559 194 194 194 325 325 325 F F F
214282024-02-01T17:36:36.990Z 28 560-579 189 189 189 314 314 314 T F T <---
214292024-02-01T17:36:36.990Z 29 580-599 184 184 184 304 304 304 T F T <---
214302024-02-01T17:36:36.990ZMax gen: 198, Max flush: 332
214312024-02-01T17:36:36.990ZError: Difference in extent metadata found!
214322024-02-01T17:36:36.990ZA Difference in extent metadata is expected here
214332024-02-01T17:36:36.990ZOn loop 99
214342024-02-01T17:36:36.990Z
214352024-02-01T17:36:36.990ZVerifying data now
214362024-02-01T17:36:42.562ZNo change after verify
214372024-02-01T17:36:42.562ZLoop: 99 Downstairs dump after verify (and repair):
214382024-02-01T17:36:42.579ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214392024-02-01T17:36:42.579Z 0 000-019 198 198 198 330 330 330 F F F
214402024-02-01T17:36:42.579Z 1 020-039 196 196 196 326 326 326 F F F
214412024-02-01T17:36:42.579Z 2 040-059 192 192 192 323 323 323 F F F
214422024-02-01T17:36:42.580Z 3 060-079 199 199 199 333 333 333 F F F
214432024-02-01T17:36:42.580Z 4 080-099 199 199 199 333 333 333 F F F
214442024-02-01T17:36:42.580Z 5 100-119 198 198 198 332 332 332 F F F
214452024-02-01T17:36:42.580Z 6 120-139 198 198 198 332 332 332 F F F
214462024-02-01T17:36:42.580Z 7 140-159 197 197 197 329 329 329 F F F
214472024-02-01T17:36:42.581Z 8 160-179 199 199 199 333 333 333 F F F
214482024-02-01T17:36:42.582Z 9 180-199 199 199 199 333 333 333 F F F
214492024-02-01T17:36:42.582Z 10 200-219 196 196 196 326 326 326 F F F
214502024-02-01T17:36:42.582Z 11 220-239 199 199 199 333 333 333 F F F
214512024-02-01T17:36:42.583Z 12 240-259 199 199 199 333 333 333 F F F
214522024-02-01T17:36:42.583Z 13 260-279 197 197 197 329 329 329 F F F
214532024-02-01T17:36:42.583Z 14 280-299 199 199 199 333 333 333 F F F
214542024-02-01T17:36:42.583Z 15 300-319 196 196 196 326 326 326 F F F
214552024-02-01T17:36:42.583Z 16 320-339 199 199 199 333 333 333 F F F
214562024-02-01T17:36:42.583Z 17 340-359 199 199 199 333 333 333 F F F
214572024-02-01T17:36:42.584Z 18 360-379 199 199 199 333 333 333 F F F
214582024-02-01T17:36:42.584Z 19 380-399 194 194 194 325 325 325 F F F
214592024-02-01T17:36:42.584Z 20 400-419 198 198 198 330 330 330 F F F
214602024-02-01T17:36:42.584Z 21 420-439 194 194 194 325 325 325 F F F
214612024-02-01T17:36:42.584Z 22 440-459 196 196 196 326 326 326 F F F
214622024-02-01T17:36:42.585Z 23 460-479 198 198 198 330 330 330 F F F
214632024-02-01T17:36:42.585Z 24 480-499 197 197 197 329 329 329 F F F
214642024-02-01T17:36:42.585Z 25 500-519 199 199 199 333 333 333 F F F
214652024-02-01T17:36:42.585Z 26 520-539 199 199 199 333 333 333 F F F
214662024-02-01T17:36:42.585Z 27 540-559 194 194 194 325 325 325 F F F
214672024-02-01T17:36:42.586Z 28 560-579 199 199 199 333 333 333 F F F
214682024-02-01T17:36:42.586Z 29 580-599 199 199 199 333 333 333 F F F
214692024-02-01T17:36:42.586ZMax gen: 199, Max flush: 333
214702024-02-01T17:36:42.586Z8:43 Test duration
214712024-02-01T17:36:42.586ZTest completed
214722024-02-01T17:36:42.586Z
214732024-02-01T17:36:42.587Zreal 8:43.553446477
214742024-02-01T17:36:42.587Zuser 2:52.755458615
214752024-02-01T17:36:42.587Zsys 28.435248924
214762024-02-01T17:36:42.588Ztrap 0.148468260
214772024-02-01T17:36:42.588Ztflt 0.079704367
214782024-02-01T17:36:42.588Zdflt 0.160295694
214792024-02-01T17:36:42.588Zkflt 0.068112847
214802024-02-01T17:36:42.588Zlock 6:48:57.202854801
214812024-02-01T17:36:42.589Zslp 40:58.137953977
214822024-02-01T17:36:42.589Zlat 10.180163112
214832024-02-01T17:36:42.589Zstop 0.063667264
214842024-02-01T17:36:43.010Zprocess exited: duration 537502 ms, exit code 0
 
214852024-02-01T17:36:43.028Zfound 4 output files
214862024-02-01T17:36:43.029Zuploading: /tmp/test_repair_ds8810.txt (86105 bytes)
214872024-02-01T17:36:44.080Zuploaded: /tmp/test_repair_ds8810.txt
214882024-02-01T17:36:44.083Zuploading: /tmp/test_repair_ds8820.txt (28216 bytes)
214892024-02-01T17:36:45.103Zuploaded: /tmp/test_repair_ds8820.txt
214902024-02-01T17:36:45.103Zuploading: /tmp/test_repair_ds8830.txt (43471 bytes)
214912024-02-01T17:36:46.155Zuploaded: /tmp/test_repair_ds8830.txt
214922024-02-01T17:36:46.156Zuploading: /tmp/test_repair_out.txt (118571 bytes)
214932024-02-01T17:36:47.231Zuploaded: /tmp/test_repair_out.txt